commit f13deaa139ec45ce4cd722300061fec2e022393c Author: zhouganqing Date: Fri Nov 25 16:30:36 2022 +0800 Import Upstream version 1.4.0 diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..877149c --- /dev/null +++ b/.coveragerc @@ -0,0 +1,49 @@ +[run] +# In coverage 4.0b3, concurrency=gevent is exactly equivalent to +# concurrency=greenlet, except it causes coverage itself to import +# gevent. That messes up our coverage numbers for top-level +# statements, so we use greenlet instead. See https://github.com/gevent/gevent/pull/655#issuecomment-141198002 +# See also .coveragerc-pypy +concurrency = greenlet +parallel = True +source = gevent +omit = + # This is for <= 2.7.8, which we don't test + src/gevent/_ssl2.py + src/gevent/libev/_corecffi_build.py + src/gevent/libuv/_corecffi_build.py + src/gevent/win32util.py + # having concurrency=greenlet means that the Queue class + # which is used from multiple real threads doesn't + # properly get covered. + src/gevent/_threading.py + # local.so sometimes gets included, and it can't be parsed + # as source, so it fails the whole process. + *.so + src/gevent/libev/*.so + src/gevent/libuv/*.so + src/gevent/resolver/*.so + + +[report] +# Coverage is run on Linux under cPython 2/3 and pypy +exclude_lines = + pragma: no cover + def __repr__ + raise AssertionError + raise NotImplementedError + except ImportError: + if __name__ == .__main__.: + if sys.platform == 'win32': + if mswindows: + if is_windows: + if WIN: + self.fail +omit = + # local.so sometimes gets included, and it can't be parsed + # as source, so it fails the whole process. + # coverage 4.5 needs this specified here, 4.4.2 needed it in [run] + *.so + /tmp/test_* + # Third-party vendored code + src/gevent/_tblib.py diff --git a/.coveragerc-pypy b/.coveragerc-pypy new file mode 100644 index 0000000..bc162e9 --- /dev/null +++ b/.coveragerc-pypy @@ -0,0 +1,35 @@ +[run] +# This is just like .coveragerc, but +# used for PyPy running. pypy doesn't support concurrency=greenlet +parallel = True +source = gevent +omit = + # This is for <= 2.7.8, which we don't test + src/gevent/_ssl2.py + src/gevent/libev/_corecffi_build.py + src/gevent/libuv/_corecffi_build.py + src/gevent/win32util.py + # having concurrency=greenlet means that the Queue class + # which is used from multiple real threads doesn't + # properly get covered. + src/gevent/_threading.py + test_* + # local.so sometimes gets included, and it can't be parsed + # as source, so it fails the whole process. + *.so + + +[report] +# Coverage is run on Linux under cPython 2/3 and pypy, so +# exclude branches that are windows specific or pypy +# specific +exclude_lines = + pragma: no cover + def __repr__ + raise AssertionError + raise NotImplementedError + except ImportError: + if __name__ == .__main__.: + if sys.platform == 'win32': + if mswindows: + if is_windows: diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..1688349 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,20 @@ +* gevent version: +* Python version: Please be as specific as possible: "cPython 2.7.9 downloaded from python.org" +* Operating System: Please be as specific as possible: "Raspbian (Debian Linux 8.0 Linux 4.9.35-v7+ armv7l)" + +### Description: + +**REPLACE ME**: What are you trying to get done, what has happened, what went wrong, and what did you expect? + +``` +Remember to put tracebacks in literal blocks +``` + +### What I've run: + +**REPLACE ME**: Paste short, self contained, correct example code (sscce.org), tracebacks, etc, here + + +```python +"Put python code in python blocks" +``` diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..02e01bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,98 @@ +*.py[cod] +build/ +.runtimes +.tox/ +*.so +*.o +*.egg-info +gevent.*.[ch] +src/gevent/__pycache__ +src/gevent/_semaphore.c +src/gevent/local.c +src/gevent/greenlet.c +src/gevent/_ident.c +src/gevent/_imap.c +src/gevent/event.c +src/gevent/_hub_local.c +src/gevent/_waiter.c +src/gevent/_tracer.c +src/gevent/queue.c +src/gevent/_hub_primitives.c +src/gevent/_greenlet_primitives.c +src/gevent/_abstract_linkable.c +src/gevent/libev/corecext.c +src/gevent/libev/corecext.h +src/gevent/libev/_corecffi.c +src/gevent/libev/_corecffi.o +src/gevent/resolver/cares.c + +# Cython annotations +src/gevent/*.html +src/gevent/libev/*.html +src/gevent/resolver/*.html + +Makefile.ext +MANIFEST +*_flymake.py + +.coverage\.* +htmlcov/ +.coverage + +doc/_build +doc/__pycache__ + +# Artifacts of configuring in place +deps/c-ares/config.log +deps/c-ares/config.status +deps/c-ares/stamp-h1 +deps/c-ares/stamp-h2 +deps/c-ares/ares_build.h.orig +deps/c-ares/ares_config.h +deps/c-ares/ares_build.h +deps/c-ares/.libs +deps/c-ares/*.o +deps/c-ares/*.lo +deps/c-ares/*.la +deps/c-ares/.deps +deps/c-ares/acountry +deps/c-ares/adig +deps/c-ares/ahost +deps/c-ares/Makefile +deps/c-ares/libtool +deps/c-ares/libcares.pc +deps/c-ares/test/.deps +deps/c-ares/test/Makefile +deps/c-ares/test/config.log +deps/c-ares/test/config.status +deps/c-ares/test/libtool +deps/c-ares/test/stamp-h1 + +deps/libev/.deps +deps/libev/Makefile +deps/libev/config.log +deps/libev/config.h +deps/libev/config.status +deps/libev/libtool +deps/libev/stamp-h1 +deps/libev/.libs +deps/libev/*.lo +deps/libev/*.la +deps/libev/*.o + +deps/libuv/.deps +deps/libuv/Makefile +deps/libuv/config.log +deps/libuv/config.h +deps/libuv/config.status +deps/libduv/libtool +deps/libuv/stamp-h1 +deps/libuv/.libs +deps/libuv/*.lo +deps/libuv/*.la +deps/libuv/*.o + + +# running setup.py on PyPy +config.h +configure-output.txt diff --git a/.landscape.yml b/.landscape.yml new file mode 100644 index 0000000..3c70d5d --- /dev/null +++ b/.landscape.yml @@ -0,0 +1,125 @@ +doc-warnings: no # experimental, raises an exception +test-warnings: no +strictness: veryhigh +max-line-length: 160 +# We don't use any of the auto-detected things, and +# auto-detection slows down startup +autodetect: false +requirements: + - dev-requirements.txt + +python-targets: + - 2 +# - 3 # landscape.io seems to fail if we run both py2 and py3? +ignore-paths: + - examples/webchat/ + - deps/libuv/ + - doc/ + - build/ + - deps/ + - dist + - .eggs + # util creates lots of warnings. ideally they should be fixed, + # but that code doesn't change often + - util + # likewise with scripts + - scripts/ + # This file has invalid syntax for Python 3, which is how + # landscape.io runs things... + - src/gevent/_util_py2.py + # ...and this file has invalid syntax for Python 2, which is how + # travis currently runs things. sigh. + - src/gevent/_socket3.py + # This is vendored with minimal changes + - src/gevent/_tblib.py + # likewise + - src/greentest/_six.py + # This triggers https://github.com/PyCQA/pylint/issues/846 on Travis, + # but the file is really small, so it's better to skip this one + # file than disable that whole check. + - src/gevent/core.py + # sadly, this one is complicated + - setup.py + # This crashes (infinite recursion) trying to get the mro() of a class that extends + # build_ext + - _setuputils.py + - src/greentest/getaddrinfo_module.py +ignore-patterns: + # disabled code + - ^src/greentest/xtest_.*py + # standard library code + - ^src/greentest/2.* + - ^src/greentest/3.* + # benchmarks that aren't used/changed much + - ^src/greentest/bench_.*py + +pyroma: + run: true + +mccabe: + # We have way too many violations of the complexity measure. + # We should enable this and fix them one at a time, but that's + # more refactoring than I want to do initially. + run: false + +pyflakes: + disable: + # F821: undefined name; caught better by pylint, where it can be + # controlled for the whole file/per-line + - F821 + # F401: unused import; same story + - F401 + # F811: redefined function; same story + - F811 + # F403: wildcard import; same story + - F403 + +pep8: + disable: + # N805: first arg should be self; fails on metaclasses and + # classmethods; pylint does a better job + - N805 + # N802: function names should be lower-case; comes from Windows + # funcs and unittest-style asserts and factory funcs + - N802 + # N801: class names should use CapWords + - N801 + # N803: argument name should be lower-case; comes up with using + # the class name as a keyword-argument + - N803 + # N813: camelCase imported as lowercase; socketcommon + - N813 + # N806: variable in function should be lowercase; but sometimes we + # want constant-looking names, especially for closures + - N806 + # N812: lowercase imported as non-lowercase; from greenlet import + # greenlet as RawGreenlet + - N812 + # E261: at least two spaces before inline comment. Really? Who does + # that? + - E261 + # E265: Block comment should start with "# ". This arises from + # commenting out individual lines of code. + - E265 + # N806: variable in function should be lowercase; but sometimes we + # want constant-looking names, especially for closures + - N806 + # W503 line break before binary operator (I like and/or on the + # next line, it makes more sense) + - W503 + # E266: too many leading '#' for block comment. (Multiple # can + # set off blocks) + - E266 + # E402 module level import not at top of file. (happens in + # setup.py, some test cases) + - E402 + # E702: multiple expressions on one line semicolon + # (happens for monkey-patch)) + - E702 + # E731: do not assign a lambda expression, use a def + # simpler than a def sometimes, and prevents redefinition warnings + - E731 + # E302/303: Too many/too few blank lines (between classes, etc) + # This is *really* nitpicky. + - E302 + - E303 diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..080f0b4 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,112 @@ +[MASTER] +extension-pkg-whitelist=gevent.libuv._corecffi,gevent.libev._corecffi,gevent.local,gevent._ident + +[MESSAGES CONTROL] + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). +# NOTE: comments must go ABOVE the statement. In Python 2, mixing in +# comments disables all directives that follow, while in Python 3, putting +# comments at the end of the line does the same thing (though Py3 supports +# mixing) + + +# invalid-name, ; We get lots of these, especially in scripts. should fix many of them +# protected-access, ; We have many cases of this; legit ones need to be examinid and commented, then this removed +# no-self-use, ; common in superclasses with extension points +# too-few-public-methods, ; Exception and marker classes get tagged with this +# exec-used, ; should tag individual instances with this, there are some but not too many +# global-statement, ; should tag individual instances +# multiple-statements, ; "from gevent import monkey; monkey.patch_all()" +# locally-disabled, ; yes, we know we're doing this. don't replace one warning with another +# cyclic-import, ; most of these are deferred imports +# too-many-arguments, ; these are almost always because that's what the stdlib does +# redefined-builtin, ; likewise: these tend to be keyword arguments like len= in the stdlib +# undefined-all-variable, ; XXX: This crashes with pylint 1.5.4 on Travis (but not locally on Py2/3 +# ; or landscape.io on Py3). The file causing the problem is unclear. UPDATE: identified and disabled +# that file. +# see https://github.com/PyCQA/pylint/issues/846 +# useless-suppression: the only way to avoid repeating it for specific statements everywhere that we +# do Py2/Py3 stuff is to put it here. Sadly this means that we might get better but not realize it. +# duplicate-code: Yeah, the compatibility ssl modules are much the same +# In pylint 1.8.0, inconsistent-return-statements are created for the wrong reasons. +# This code raises it, even though there's only one return (the implicit 'return None' is presumably +# what triggers it): +# def foo(): +# if baz: +# return 1 +# In Pylint 2dev1, needed for Python 3.7, we get spurious 'useless return' errors: +# @property +# def foo(self): +# return None # generates useless-return +disable=wrong-import-position, + wrong-import-order, + missing-docstring, + ungrouped-imports, + invalid-name, + protected-access, + no-self-use, + too-few-public-methods, + exec-used, + global-statement, + multiple-statements, + locally-disabled, + cyclic-import, + too-many-arguments, + redefined-builtin, + useless-suppression, + duplicate-code, + undefined-all-variable, + inconsistent-return-statements, + useless-return, + useless-object-inheritance + + +[FORMAT] +# duplicated from setup.cfg +max-line-length=160 +max-module-lines=1100 + +[MISCELLANEOUS] +# List of note tags to take in consideration, separated by a comma. +#notes=FIXME,XXX,TODO +# Disable that, we don't want them in the report (???) +notes= + +[VARIABLES] + +dummy-variables-rgx=_.* + +[TYPECHECK] + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +# gevent: this is helpful for py3/py2 code. +generated-members=exc_clear + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). This supports can work +# with qualified names. +# greenlet, Greenlet, parent, dead: all attempts to fix issues in greenlet.py +# only seen on the service, e.g., self.parent.loop: class parent has no loop +ignored-classes=SSLContext, SSLSocket, greenlet, Greenlet, parent, dead + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=gevent._corecffi,gevent.os,os,greenlet,threading,gevent.libev.corecffi,gevent.socket,gevent.core + +[DESIGN] +max-attributes=12 +max-parents=10 + +[BASIC] +bad-functions=input +# Prospector turns ot unsafe-load-any-extension by default, but +# pylint leaves it off. This is the proximal cause of the +# undefined-all-variable crash. +#unsafe-load-any-extension = no diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2ad40da --- /dev/null +++ b/.travis.yml @@ -0,0 +1,49 @@ +# .travis.yml based on https://github.com/DRMacIver/hypothesis/blob/master/.travis.yml +language: python +dist: xenial +group: travis_latest +python: + - 3.7-dev + +env: + global: + - BUILD_RUNTIMES=$HOME/.runtimes + - PYTHONHASHSEED=8675309 + - CC="ccache gcc" + - CCACHE_NOCPP2=true + - CCACHE_SLOPPINESS=file_macro,time_macros,include_file_ctime,include_file_mtime + - CCACHE_NOHASHDIR=true + - CFLAGS="-g -pipe" + + matrix: + # These are ordered to get as much diversity in the + # first group of parallel runs (4) as possible + - TASK=test-py37 + - TASK=test-py27 + - TASK=test-pypy + - TASK=test-py36 + - TASK=test-py27-noembed + - TASK=test-pypy3 + - TASK=test-py35 + - TASK=test-py34 + +matrix: + fast_finish: true + +script: + - make $TASK + +notifications: + email: false + + +cache: + pip: true + directories: + - $HOME/.venv + - $HOME/.runtimes + - $HOME/.wheelhouse + - $HOME/.ccache + +before_cache: + - rm -f $HOME/.cache/pip/log/debug.log diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..05892fb --- /dev/null +++ b/AUTHORS @@ -0,0 +1,62 @@ +Gevent is written and maintained by + + Denis Bilenko + Matt Iversen + Steffen Prince + Jason Madden + +and the contributors (ordered by the date of first contribution): + + Jason Toffaletti + Mike Barton + Ludvig Ericson + Marcus Cavanaugh + Matt Goodall + Ralf Schmitt + Daniele Varrazzo + Nicholas Piël + Örjan Persson + Uriel Katz + Ted Suzman + Randall Leeds + Erik Näslund + Alexey Borzenkov + David Hain + Dmitry Chechik + Ned Rockson + Tommie Gannert + Shaun Lindsay + Andreas Blixt + Nick Barkas + Galfy Pundee + Alexander Boudkar + Damien Churchill + Tom Lynn + Shaun Cutts + David LaBissoniere + Alexandre Kandalintsev + Geert Jansen + Vitaly Kruglikov + Saúl Ibarra Corretgé + Oliver Beattie + Bobby Powers + Anton Patrushev + Jan-Philip Gehrcke + Alex Gaynor + 陈小玉 + Philip Conrad + Heungsub Lee + Ron Rothman + + See https://github.com/gevent/gevent/graphs/contributors for more info. + +Gevent is inspired by and uses some code from eventlet which was written by + + Bob Ipollito + Donovan Preston + +The win32util module is taken from Twisted. The tblib module is taken from python-tblib by Ionel Cristian Mărieș. + +Some modules (local, ssl) contain code from the Python standard library. + +If your code is used in gevent and you are not mentioned above, please contact the maintainer. diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..e0b13dc --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,815 @@ +=========== + Changelog +=========== + +.. currentmodule:: gevent + +1.4.0 (2019-01-04) +================== + +- Build with Cython 0.29 in '3str' mode. + +- Test with PyPy 6.0 on Windows. + +- Add support for application-wide callbacks when ``Greenlet`` objects + are started. See :pr:`1289`, provided by Yury Selivanov. + +- Fix consuming a single ready object using + ``next(gevent.iwait(objs))``. Previously such a construction would + hang because `iter` was not called. See :pr:`1288`, provided by Josh + Snyder. This is not recommended, though, as unwaited objects will + have dangling links (but see next item). + +- Make `gevent.iwait` return an iterator that can now also be used as + a context manager. If you'll only be consuming part of the iterator, + use it in a ``with`` block to avoid leaking resources. See + :pr:`1290`, provided by Josh Snyder. + +- Fix semaphores to immediately notify links if they are ready and + ``rawlink()`` is called. This behaves like ``Event`` and + ``AsyncEvent``. Note that the order in which semaphore links are + called is not specified. See :issue:`1287`, reported by Dan Milon. + +- Improve safety of handling exceptions during interpreter shutdown. + See :issue:`1295` reported by BobDenar1212. + +- Remove the deprecated ability to specify ``GEVENT_RESOLVER`` and + other importable settings as a ``path/to/a/package.module.item``. + This had race conditions and didn't work with complicated resolver + implementations. Place the required package or module on `sys.path` + first. + +- Reduce the chances that using the blocking monitor functionality + could result in apparently random ``SystemError: + Objects/tupleobject.c: bad argument to internal function``. Reported + in :issue:`1302` by Ulrich Petri. + +- Refactored the gevent test runner and test suite to make them more + reusable. In particular, the tests are now run with ``python -m + gevent.tests``. See :issue:`1293`. + +- Make a monkey-patched ``socket.getaddrinfo`` return socket module + enums instead of plain integers for the socket type and address + family on Python 3. See :issue:`1310` reported by TheYOSH. + +- Make gevent's pywsgi server set the non-standard environment value + ``wsgi.input_terminated`` to True. See :issue:`1308`. + +- Make `gevent.util.assert_switches` produce more informative messages + when the assertion fails. + +- Python 2: If a `gevent.socket` was closed asynchronously (in a + different greenlet or a hub callback), `AttributeError` could result + if the socket was already in use. Now the correct socket.error + should be raised. + +- Fix :meth:`gevent.threadpool.ThreadPool.join` raising a + `UserWarning` when using the libuv backend. Reported in + :issue:`1321` by ZeroNet. + +- Fix ``FileObjectPosix.seek`` raising `OSError` when it should have + been `IOError` on Python 2. Reported by, and PR by, Ricardo Kirkner. + See :issue:`1323`. + +- Upgrade libuv from 1.23.2 to 1.24.0. + +1.3.7 (2018-10-12) +================== + +- Formatting run info no longer includes ``gevent.local.local`` + objects that have no value in the greenlet. See :issue:`1275`. + +- Fixed negative length in pywsgi's Input read functions for non chunked body. + Reported in :issue:`1274` by tzickel. + +- Upgrade libuv from 1.22.0 to 1.23.2. + +- Fix opening files in text mode in CPython 2 on Windows by patching + libuv. See :issue:`1282` reported by wiggin15. + +1.3.6 (2018-08-17) +================== + +- gevent now depends on greenlet 0.4.14 or above. gevent binary wheels + for 1.3.5 and below must have greenlet 0.4.13 installed on Python + 3.7 or they will crash. Reported by Alexey Stepanov in :issue:`1260` + and pkittenis in :issue:`1261`. + +- :class:`gevent.local.local` subclasses correctly supports + ``@staticmethod`` functions. Reported by Brendan Powers in + :issue:`1266`. + + +1.3.5 (2018-07-16) +================== + +- Update the bundled libuv from 1.20.1 to 1.22.0. + +- Test Python 3.7 on Appveyor. Fix the handling of Popen's + ``close_fds`` argument on 3.7. + +- Update Python versions tested on Travis, including PyPy to 6.0. See :issue:`1195`. + +- :mod:`gevent.queue` imports ``_PySimpleQueue`` instead of + ``SimpleQueue`` so that it doesn't block the event loop. + :func:`gevent.monkey.patch_all` makes this same substitution in + :mod:`queue`. This fixes issues with + :class:`concurrent.futures.ThreadPoolExecutor` as well. Reported in + :issue:`1248` by wwqgtxx and :issue:`1251` by pyld. + +- :meth:`gevent.socket.socket.connect` doesn't pass the port (service) + to :func:`socket.getaddrinfo` when it resolves an ``AF_INET`` or + ``AF_INET6`` address. (The standard library doesn't either.) This + fixes an issue on Solaris. Reported in :issue:`1252` by wiggin15. + +- :meth:`gevent.socket.socket.connect` works with more address + families, notably AF_TIPC, AF_NETLINK, AF_BLUETOOTH, AF_ALG and AF_VSOCK. + + +1.3.4 (2018-06-20) +================== + +- Be more careful about issuing ``MonkeyPatchWarning`` for ssl + imports. Now, we only issue it if we detect the one specific + condition that is known to lead to RecursionError. This may produce + false negatives, but should reduce or eliminate false positives. + +- Based on measurements and discussion in :issue:`1233`, adjust the + way :mod:`gevent.pywsgi` generates HTTP chunks. This is intended to + reduce network overhead, especially for smaller chunk sizes. + +- Additional slight performance improvements in :mod:`gevent.pywsgi`. + See :pr:`1241`. + +1.3.3 (2018-06-08) +================== + +- :func:`gevent.sleep` updates the loop's notion of the current time + before sleeping so that sleep duration corresponds more closely to + elapsed (wall clock) time. :class:`gevent.Timeout` does the same. + Reported by champax and FoP in :issue:`1227`. + +- Fix an ``UnboundLocalError`` in SSL servers when wrapping a socket + throws an error. Reported in :issue:`1236` by kochelmonster. + +1.3.2.post0 (2018-05-30) +======================== + +- Fix a packaging error in manylinux binary wheels that prevented some + imports from working. See :issue:`1219`. + + +1.3.2 (2018-05-29) +================== + +- Allow weak refeneces to :class:`gevent.queue.Queue`. Reported in + :issue:`1217` by githrdw. + + +1.3.1 (2018-05-18) +================== + +- Allow weak references to :class:`gevent.event.Event`. Reported in + :issue:`1211` by Matias Guijarro. + +- Fix embedded uses of :func:`gevent.Greenlet.spawn`, especially under + uwsgi. Reported in :issue:`1212` by Kunal Gangakhedkar. + +- Fix :func:`gevent.os.nb_write` and :func:`gevent.os.nb_read` not + always closing the IO event they opened in the event of an + exception. This would be a problem especially for libuv. + +1.3.0 (2018-05-11) +================== + +- Python 3.7 passes the automated memory leak checks. See :issue:`1197`. + +- Update autoconf's config.guess and config.sub to the latest versions + for c-ares and libev. + +- :class:`gevent.local.local` subclasses that mix-in ABCs can be instantiated. + Reported in :issue:`1201` by Bob Jordan. + +1.3b2 (2018-05-03) +================== + +- On Windows, CFFI is now a dependency so that the libuv backend + really can be used by default. + +- Fix a bug detecting whether we can use the memory monitoring + features when psutil is not installed. + +- `gevent.subprocess.Popen` uses ``/proc/self/fd`` (on Linux) or + ``/dev/fd`` (on BSD, including macOS) to find the file descriptors + to close when ``close_fds`` is true. This matches an optimization + added to Python 3 (and backports it to Python 2.7), making process + spawning up to 9 times faster. Also, on Python 3, since Python 3.3 + is no longer supported, we can also optimize the case where + ``close_fds`` is false (not the default), making process spawning up + to 38 times faster. Initially reported in :issue:`1172` by Ofer Koren. + +- The bundled libuv is now 1.20.1, up from 1.19.2. See :issue:`1177`. + +- The long-deprecated and undocumented module ``gevent.wsgi`` was removed. + +- Add `gevent.util.assert_switches` to build on the monitoring + functions. Fixes :issue:`1182`. + +- A started monitor thread for the active hub now survives a fork. See + :issue:`1185`. + +- The greenlet tracer functions used for the various monitoring + capabilities are now compiled with Cython for substantially lower + overhead. See :pr:`1190`. + +- libuv now collects all pending watchers and runs their callbacks at + the end of the loop iteration using UV_RUN_ONCE. This eliminates the + need to patch libuv to be greenlet-safe. It also means that + zero-duration timer watchers are actual timer watchers again + (instead of being turned into check watchers); newly added + zero-duration timers cannot block the event loop because they won't + be run until a safe time. + +- Python 3.7.0b4 is now the tested and supported version of Python + 3.7. PyPy 6.0 has been tested, although CI continues to use 5.10. + +1.3b1 (2018-04-13) +================== + +Dependencies +------------ + +- Cython 0.28.2 is now used to build gevent from a source checkout. + +- The bundled libuv is now 1.19.2, up from 1.18.0. + +Platform Support +---------------- + +- Travis CI tests on Python 3.7.0b3. + +- Windows now defaults to the libuv backend if CFFI is installed. See + :issue:`1163`. + +Bug Fixes +--------- + +- On Python 2, when monkey-patching `threading.Event`, also + monkey-patch the underlying class, ``threading._Event``. Some code + may be type-checking for that. See :issue:`1136`. + +- Fix libuv io watchers polling for events that only stopped watchers + are interested in, reducing CPU usage. Reported in :issue:`1144` by + wwqgtxx. + +Enhancements +------------ + +- Add additional optimizations for spawning greenlets, making it + faster than 1.3a2. + +- Use strongly typed watcher callbacks in the libuv CFFI extensions. + This prevents dozens of compiler warnings. + +- When gevent prints a timestamp as part of an error message, it is + now in UTC format as specified by RFC3339. + +- Threadpool threads that exit now always destroy their hub (if one + was created). This prevents some forms of resource leaks (notably + visible as blocking functions reported by the new monitoring abilities). + +- Hub objects now include the value of their ``name`` attribute in + their repr. + +- Pools for greenlets and threads have lower overhead, especially for + ``map``. See :pr:`1153`. + +- The undocumented, internal implementation classes ``IMap`` and + ``IMapUnordered`` classes are now compiled with Cython, further + reducing the overhead of ``[Thread]Pool.imap``. + +- The classes `gevent.event.Event` and `gevent.event.AsyncResult` + are compiled with Cython for improved performance, as is the + ``gevent.queue`` module and ``gevent.hub.Waiter`` and certain + time-sensitive parts of the hub itself. Please report any + compatibility issues. + +- ``python -m gevent.monkey + + +You're also welcome to join `#gevent`_ IRC channel on freenode. + + +Russian group +============= + +Русскоязычная группа находится здесь: `Google Groups (gevent-ru)`_. Чтобы подписаться, отправьте сообщение на gevent-ru+subscribe@googlegroups.com + + +.. _Google Groups (gevent): http://groups.google.com/group/gevent +.. _#gevent: http://webchat.freenode.net/?channels=gevent +.. _Google Groups (gevent-ru): http://groups.google.com/group/gevent-ru diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..31d5c2f --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,247 @@ +# -*- coding: utf-8 -*- +# +# gevent documentation build configuration file, created by +# sphinx-quickstart on Thu Oct 1 09:30:02 2009. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +from __future__ import print_function +import sys +import os + +# Use the python versions instead of the cython compiled versions +# for better documentation extraction and ease of tweaking docs. +os.environ['PURE_PYTHON'] = '1' + +sys.path.append(os.path.dirname(__file__)) # for mysphinxext + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.coverage', + 'sphinx.ext.intersphinx', + 'mysphinxext', + 'sphinx.ext.extlinks', + 'sphinx.ext.viewcode', + 'repoze.sphinx.autointerface', +] + +intersphinx_mapping = { + 'http://docs.python.org/': None, + 'https://greenlet.readthedocs.io/en/latest/': None, + 'https://zopeevent.readthedocs.io/en/latest/': None, + 'https://zopecomponent.readthedocs.io/en/latest/': None, +} + +extlinks = {'issue': ('https://github.com/gevent/gevent/issues/%s', + 'issue #'), + 'pr': ('https://github.com/gevent/gevent/pull/%s', + 'pull request #')} + +autodoc_default_flags = ['members', 'show-inheritance'] +autodoc_member_order = 'groupwise' +autoclass_content = 'both' + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'contents' + +# General information about the project. +project = u'gevent' +copyright = u'2009-2018, gevent contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +from gevent import __version__ +version = __version__ +# The full version, including alpha/beta/rc tags. +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +default_role = 'obj' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = False + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'perldoc' + +# A list of ignored prefixes for module index sorting. +modindex_common_prefix = ['gevent.'] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'mytheme' +html_theme_path = ['.'] +html_theme_options = {'gevent_version': __version__} +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#if html_theme == 'default': +# html_theme_options = {'rightsidebar' : True} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +html_short_title = 'Documentation' + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# This is true by default in sphinx 1.6 +html_use_smartypants = True +smartquotes = True # 1.7 + +# Custom sidebar templates, maps document names to template names. +html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {'contentstable': 'contentstable.html'} + +# If false, no module index is generated. +html_use_modindex = True + +# If false, no index is generated. +html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +html_show_sourcelink = False + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'geventdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'gevent.tex', u'gevent Documentation', + u'gevent contributors', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + + +############################################################################### + + +# prevent some stuff from showing up in docs +import socket +import gevent.socket +for item in gevent.socket.__all__[:]: + if getattr(gevent.socket, item) is getattr(socket, item, None): + gevent.socket.__all__.remove(item) + +if not hasattr(gevent.socket, '_fileobject'): + # Python 3 building Python 2 docs. + gevent.socket._fileobject = object() diff --git a/doc/configuration.rst b/doc/configuration.rst new file mode 100644 index 0000000..4a3a8c6 --- /dev/null +++ b/doc/configuration.rst @@ -0,0 +1,10 @@ +.. _gevent-configuration: + +==================== + Configuring gevent +==================== + +.. seealso:: :func:`gevent.setswitchinterval` + For additional runtime configuration. + +.. autoclass:: gevent._config.Config diff --git a/doc/contents.rst b/doc/contents.rst new file mode 100644 index 0000000..c214a22 --- /dev/null +++ b/doc/contents.rst @@ -0,0 +1,44 @@ +=================== + Table Of Contents +=================== + +Introduction and Basics +======================= + +.. toctree:: + :maxdepth: 2 + + install + intro + whatsnew_1_3 + api/gevent + api/gevent.greenlet + servers + dns + monitoring + loop_impls + configuration + changelog + +API Details +=========== + +.. toctree:: + :maxdepth: 2 + + api/index + +Related Information +=================== + +.. toctree:: + :maxdepth: 1 + + success + community + older_releases + + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/dns.rst b/doc/dns.rst new file mode 100644 index 0000000..3f74be1 --- /dev/null +++ b/doc/dns.rst @@ -0,0 +1,38 @@ +======================= + Name Resolution (DNS) +======================= + +gevent includes support for a pluggable hostname resolution system. +Pluggable resolvers are (generally) intended to be cooperative. +This pluggable resolution system is used automatically when the system +is :mod:`monkey patched `, and may be used manually +through the :attr:`resolver attribute ` of the +:class:`gevent.hub.Hub` or the corresponding methods in the +:mod:`gevent.socket` module. + +A resolver implements the 5 standandard functions from the +:mod:`socket` module for resolving hostnames and addresses: + +* :func:`socket.gethostbyname` +* :func:`socket.gethostbyname_ex` +* :func:`socket.getaddrinfo` +* :func:`socket.gethostbyaddr` +* :func:`socket.getnameinfo` + +Configuration +============= + +gevent includes four implementations of resolvers, and applications +can provide their own implementation. By default, gevent uses +:class:`a threadpool `. This can +:attr:`be customized `. + +Please see the documentation for each resolver class to understand the +relative performance and correctness tradeoffs. + +.. toctree:: + + api/gevent.resolver.thread + api/gevent.resolver.ares + api/gevent.resolver.dnspython + api/gevent.resolver.blocking diff --git a/doc/examples/concurrent_download.rst b/doc/examples/concurrent_download.rst new file mode 100644 index 0000000..d44e309 --- /dev/null +++ b/doc/examples/concurrent_download.rst @@ -0,0 +1,8 @@ +================================ + Example concurrent_download.py +================================ +.. literalinclude:: ../../examples/concurrent_download.py + :language: python + :linenos: + +`Current source `_ diff --git a/doc/examples/dns_mass_resolve.rst b/doc/examples/dns_mass_resolve.rst new file mode 100644 index 0000000..e739f62 --- /dev/null +++ b/doc/examples/dns_mass_resolve.rst @@ -0,0 +1,9 @@ +============================= +Example dns_mass_resolve.py +============================= +.. literalinclude:: ../../examples/dns_mass_resolve.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/echoserver.rst b/doc/examples/echoserver.rst new file mode 100644 index 0000000..04b5146 --- /dev/null +++ b/doc/examples/echoserver.rst @@ -0,0 +1,9 @@ +============================= +Example echoserver.py +============================= +.. literalinclude:: ../../examples/echoserver.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/geventsendfile.rst b/doc/examples/geventsendfile.rst new file mode 100644 index 0000000..22df9a8 --- /dev/null +++ b/doc/examples/geventsendfile.rst @@ -0,0 +1,9 @@ +============================= +Example geventsendfile.py +============================= +.. literalinclude:: ../../examples/geventsendfile.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/index.rst b/doc/examples/index.rst new file mode 100644 index 0000000..1fbab0b --- /dev/null +++ b/doc/examples/index.rst @@ -0,0 +1,28 @@ +========== + Examples +========== + +.. + All files generated with shell oneliner: + for i in examples/*py; do bn=`basename $i`; bnp=`basename $i .py`; echo -e "=============================\nExample $bn\n=============================\n.. literalinclude:: ../../examples/$bn\n :language: python\n :linenos:\n\n\`Current source \`_\n" > doc/examples/$bnp.rst; done + +This is a snapshot of the examples contained in `the gevent source +`_. + +.. toctree:: + concurrent_download + dns_mass_resolve + echoserver + geventsendfile + portforwarder + processes + psycopg2_pool + threadpool + udp_client + udp_server + unixsocket_client + unixsocket_server + webproxy + webpy + wsgiserver + wsgiserver_ssl diff --git a/doc/examples/portforwarder.rst b/doc/examples/portforwarder.rst new file mode 100644 index 0000000..1ec1f94 --- /dev/null +++ b/doc/examples/portforwarder.rst @@ -0,0 +1,9 @@ +============================= +Example portforwarder.py +============================= +.. literalinclude:: ../../examples/portforwarder.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/processes.rst b/doc/examples/processes.rst new file mode 100644 index 0000000..10217d0 --- /dev/null +++ b/doc/examples/processes.rst @@ -0,0 +1,9 @@ +============================= +Example processes.py +============================= +.. literalinclude:: ../../examples/processes.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/psycopg2_pool.rst b/doc/examples/psycopg2_pool.rst new file mode 100644 index 0000000..6e94f8e --- /dev/null +++ b/doc/examples/psycopg2_pool.rst @@ -0,0 +1,9 @@ +============================= +Example psycopg2_pool.py +============================= +.. literalinclude:: ../../examples/psycopg2_pool.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/threadpool.rst b/doc/examples/threadpool.rst new file mode 100644 index 0000000..f0f4d50 --- /dev/null +++ b/doc/examples/threadpool.rst @@ -0,0 +1,9 @@ +============================= +Example threadpool.py +============================= +.. literalinclude:: ../../examples/threadpool.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/udp_client.rst b/doc/examples/udp_client.rst new file mode 100644 index 0000000..ec3ed5c --- /dev/null +++ b/doc/examples/udp_client.rst @@ -0,0 +1,9 @@ +============================= +Example udp_client.py +============================= +.. literalinclude:: ../../examples/udp_client.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/udp_server.rst b/doc/examples/udp_server.rst new file mode 100644 index 0000000..6cf243d --- /dev/null +++ b/doc/examples/udp_server.rst @@ -0,0 +1,9 @@ +============================= +Example udp_server.py +============================= +.. literalinclude:: ../../examples/udp_server.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/unixsocket_client.rst b/doc/examples/unixsocket_client.rst new file mode 100644 index 0000000..56a6068 --- /dev/null +++ b/doc/examples/unixsocket_client.rst @@ -0,0 +1,9 @@ +============================= +Example unixsocket_client.py +============================= +.. literalinclude:: ../../examples/unixsocket_client.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/unixsocket_server.rst b/doc/examples/unixsocket_server.rst new file mode 100644 index 0000000..2373b27 --- /dev/null +++ b/doc/examples/unixsocket_server.rst @@ -0,0 +1,9 @@ +============================= +Example unixsocket_server.py +============================= +.. literalinclude:: ../../examples/unixsocket_server.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/webproxy.rst b/doc/examples/webproxy.rst new file mode 100644 index 0000000..4471a40 --- /dev/null +++ b/doc/examples/webproxy.rst @@ -0,0 +1,9 @@ +============================= +Example webproxy.py +============================= +.. literalinclude:: ../../examples/webproxy.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/webpy.rst b/doc/examples/webpy.rst new file mode 100644 index 0000000..0084a7c --- /dev/null +++ b/doc/examples/webpy.rst @@ -0,0 +1,9 @@ +============================= +Example webpy.py +============================= +.. literalinclude:: ../../examples/webpy.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/wsgiserver.rst b/doc/examples/wsgiserver.rst new file mode 100644 index 0000000..42bd251 --- /dev/null +++ b/doc/examples/wsgiserver.rst @@ -0,0 +1,9 @@ +============================= +Example wsgiserver.py +============================= +.. literalinclude:: ../../examples/wsgiserver.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/examples/wsgiserver_ssl.rst b/doc/examples/wsgiserver_ssl.rst new file mode 100644 index 0000000..9ef4221 --- /dev/null +++ b/doc/examples/wsgiserver_ssl.rst @@ -0,0 +1,9 @@ +============================= +Example wsgiserver_ssl.py +============================= +.. literalinclude:: ../../examples/wsgiserver_ssl.py + :language: python + :linenos: + +`Current source `_ + diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..f756d6e --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,39 @@ +================= + What is gevent? +================= + +.. include:: _about.rst + + + +:ref:`Continue reading ` » + +If you like gevent, :doc:`donate ` to support the development. + +What are others saying? +======================= + + +Mailing List +------------ + +.. raw:: html + + + + + +.. _coroutine: https://en.wikipedia.org/wiki/Coroutine +.. _Python: http://python.org +.. _greenlet: https://greenlet.readthedocs.io +.. _libev: http://software.schmorp.de/pkg/libev.html +.. _libuv: http://libuv.org diff --git a/doc/install.rst b/doc/install.rst new file mode 100644 index 0000000..efafe18 --- /dev/null +++ b/doc/install.rst @@ -0,0 +1,230 @@ +=============================== + Installation and Requirements +=============================== + +.. _installation: + +.. + This file is included in README.rst so it is limited to plain + ReST markup, not Sphinx. + +Supported Platforms +=================== + +`gevent 1.3`_ runs on Python 2.7 and Python 3. Releases 3.4, 3.5 and +3.6 of Python 3 are supported. (Users of older versions of Python 2 +need to install gevent 1.0.x (2.5), 1.1.x (2.6) or 1.2.x (<=2.7.8); +gevent 1.2 can be installed on Python 3.3.) gevent requires the +`greenlet `_ library and will install +the `cffi`_ library by default on Windows. + +gevent 1.3 also runs on PyPy 5.5 and above, although 5.9 or above is +strongly recommended. On PyPy, there are no external dependencies. + +gevent is tested on Windows, OS X, and Linux, and should run on most +other Unix-like operating systems (e.g., FreeBSD, Solaris, etc.) + +.. note:: On Windows using the libev backend, gevent is + limited to a maximum of 1024 open sockets due to + `limitations in libev`_. This limitation should not exist + with the default libuv backend. + +Installation +============ + +.. note:: + + This section is about installing released versions of gevent + as distributed on the `Python Package Index`_ + +.. _Python Package Index: http://pypi.org/project/gevent + +gevent and greenlet can both be installed with `pip`_, e.g., ``pip +install gevent``. Installation using `buildout +`_ is also supported. + +On Windows, OS X, and Linux, both gevent and greenlet are +distributed as binary `wheels`_. + +.. tip:: + + You need Pip 8.0 or later, or buildout 2.10.0 to install the + binary wheels. + +.. tip:: + + On Linux, you'll need to install gevent from source if you wish to + use the libuv loop implementation. This is because the `manylinux1 + `_ specification for the + distributed wheels does not support libuv. The `cffi`_ library + *must* be installed at build time. + + +Installing From Source +---------------------- + +If you are unable to use the binary wheels (for platforms where no +pre-built wheels are available or if wheel installation is disabled, +e.g., for libuv support on Linux), here are some things you need to know. + +- You can install gevent from source with ``pip install --no-binary + gevent gevent``. + +- You'll need a working C compiler that can build Python extensions. + On some platforms, you may need to install Python development + packages. + +- Installing from source requires ``setuptools``. This is installed + automatically in virtual environments and by buildout. However, + gevent uses :pep:`496` environment markers in ``setup.py``. + Consequently, you'll need a version of setuptools newer than 25 + (mid 2016) to install gevent from source; a version that's too old + will produce a ``ValueError``. Older versions of pipenv may also + `have issues installing gevent for this reason + `_. + +- To build the libuv backend (which is required on Windows and + optional elsewhere), or the CFFI-based libev backend, you must + install `cffi`_ before attempting to install gevent on CPython (on + PyPy this step is not necessary). + + +Common Installation Issues +-------------------------- + +The following are some common installation problems and solutions for +those compiling gevent from source. + +- Some Linux distributions are now mounting their temporary + directories with the ``noexec`` option. This can cause a standard + ``pip install gevent`` to fail with an error like ``cannot run C + compiled programs``. One fix is to mount the temporary directory + without that option. Another may be to use the ``--build`` option to + ``pip install`` to specify another directory. See `issue #570 + `_ and `issue #612 + `_ for examples. + +- Also check for conflicts with environment variables like ``CFLAGS``. For + example, see `Library Updates `_. + +- Users of a recent SmartOS release may need to customize the + ``CPPFLAGS`` (the environment variable containing the default + options for the C preprocessor) if they are using the libev shipped + with gevent. See `Operating Systems + `_ + for more information. + +- If you see ``ValueError: ("Expected ',' or end-of-list in", "cffi >= + 1.11.5 ; sys_platform == 'win32' and platform_python_implementation + == 'CPython'", 'at', " ; sys_platform == 'win32' and + platform_python_implementation == 'CPython'")``, the version of + setuptools is too old. Install a more recent version of setuptools. + + +Extra Dependencies +================== + +gevent has no runtime dependencies outside the standard library, +greenlet and (on some platforms) `cffi`_. However, there are a +number of additional libraries that extend gevent's functionality and +will be used if they are available. + +The `psutil `_ library is needed to +monitor memory usage. + +`zope.event `_ is highly +recommended for configurable event support; it can be installed with +the ``events`` extra, e.g., ``pip install gevent[events]``. + +`dnspython `_ is required for the +new pure-Python resolver, and on Python 2, so is `idna +`_. They can be installed with the +``dnspython`` extra. + + +Development +=========== + +To install the latest development version:: + + pip install setuptools cffi 'cython>=0.28' git+git://github.com/gevent/gevent.git#egg=gevent + +.. note:: + + You will not be able to run gevent's test suite using that method. + +To hack on gevent (using a virtualenv):: + + $ git clone https://github.com/gevent/gevent.git + $ cd gevent + $ virtualenv env + $ source env/bin/activate + (env) $ pip install -r dev-requirements.txt + +.. note:: + + The notes above about installing from source apply here as well. + The ``dev-requirements.txt`` file takes care of the library + prerequisites (CFFI, Cython), but having a working C compiler that + can create Python extensions is up to you. + + +Running Tests +------------- + +There are a few different ways to run the tests. To simply run the +tests on one version of Python during development, begin with the +above instructions to install gevent in a virtual environment and then +run:: + + (env) $ python -mgevent.tests + +Before submitting a pull request, it's a good idea to run the tests +across all supported versions of Python, and to check the code quality +using prospector. This is what is done on Travis CI. Locally it +can be done using tox:: + + pip install tox + tox + +The testrunner accepts a ``--coverage`` argument to enable code +coverage metrics through the `coverage.py`_ package. That would go +something like this:: + + python -m gevent.tests --coverage + coverage combine + coverage html -i + + +Continuous integration +---------------------- + +A test suite is run for every push and pull request submitted. Travis +CI is used to test on Linux, and `AppVeyor`_ runs the builds on +Windows. + +.. image:: https://travis-ci.org/gevent/gevent.svg?branch=master + :target: https://travis-ci.org/gevent/gevent + +.. image:: https://ci.appveyor.com/api/projects/status/q4kl21ng2yo2ixur?svg=true + :target: https://ci.appveyor.com/project/denik/gevent + + +Builds on Travis CI automatically submit updates to `coveralls.io`_ to +monitor test coverage. + +.. image:: https://coveralls.io/repos/gevent/gevent/badge.svg?branch=master&service=github + :target: https://coveralls.io/github/gevent/gevent?branch=master + +.. note:: On Debian, you will probably need ``libpythonX.Y-testsuite`` + installed to run all the tests. + +.. _coverage.py: https://pypi.python.org/pypi/coverage/ +.. _coveralls.io: https://coveralls.io/github/gevent/gevent +.. _`pip`: https://pip.pypa.io/en/stable/installing/ +.. _`wheels`: http://pythonwheels.com +.. _`gevent 1.3`: whatsnew_1_3.html + +.. _`cffi`: https://cffi.readthedocs.io +.. _`limitations in libev`: http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod#WIN32_PLATFORM_LIMITATIONS_AND_WORKA +.. _AppVeyor: https://ci.appveyor.com/project/denik/gevent diff --git a/doc/intro.rst b/doc/intro.rst new file mode 100644 index 0000000..97c151d --- /dev/null +++ b/doc/intro.rst @@ -0,0 +1,300 @@ +============== + Introduction +============== + +.. include:: _about.rst + + +Example +======= + +The following example shows how to run tasks concurrently. + + >>> import gevent + >>> from gevent import socket + >>> urls = ['www.google.com', 'www.example.com', 'www.python.org'] + >>> jobs = [gevent.spawn(socket.gethostbyname, url) for url in urls] + >>> gevent.joinall(jobs, timeout=2) + >>> [job.value for job in jobs] + ['74.125.79.106', '208.77.188.166', '82.94.164.162'] + +After the jobs have been spawned, :func:`gevent.joinall` waits for +them to complete, allowing up to 2 seconds. The results are +then collected by checking the :attr:`~gevent.Greenlet.value` property. +The :func:`gevent.socket.gethostbyname` function has the same +interface as the standard :func:`socket.gethostbyname` but it does not +block the whole interpreter and thus lets the other greenlets proceed +with their requests unhindered. + +.. _monkey-patching: + + +Monkey patching +=============== + +The example above used :mod:`gevent.socket` for socket operations. If the standard :mod:`socket` +module was used the example would have taken 3 times longer to complete because the DNS requests would +be sequential (serialized). Using the standard socket module inside greenlets makes gevent rather +pointless, so what about existing modules and packages that are built +on top of :mod:`socket` (including the standard library modules like :mod:`urllib`)? + +That's where monkey patching comes in. The functions in :mod:`gevent.monkey` carefully +replace functions and classes in the standard :mod:`socket` module with their cooperative +counterparts. That way even the modules that are unaware of gevent can benefit from running +in a multi-greenlet environment. + + >>> from gevent import monkey; monkey.patch_socket() + >>> import urllib2 # it's usable from multiple greenlets now + +See :doc:`examples/concurrent_download`. + +Beyond sockets +-------------- + +Of course, there are several other parts of the standard library that can +block the whole interpreter and result in serialized behavior. gevent +provides cooperative versions of many of those as well. They can be +patched independently through individual functions, but most programs +using monkey patching will want to patch the entire recommended set of +modules using the :func:`gevent.monkey.patch_all` function:: + + >>> from gevent import monkey; monkey.patch_all() + >>> import subprocess # it's usable from multiple greenlets now + + +.. tip:: + + When monkey patching, it is recommended to do so as early as + possible in the lifetime of the process. If possible, + monkey patching should be the first lines executed. Monkey + patching later, especially if native threads have been + created, :mod:`atexit` or signal handlers have been installed, + or sockets have been created, may lead to unpredictable + results including unexpected :exc:`~gevent.hub.LoopExit` errors. + + +Event loop +========== + +Instead of blocking and waiting for socket operations to complete (a +technique known as polling), gevent arranges for the operating system +to deliver an event letting it know when, for example, data has +arrived to be read from the socket. Having done that, gevent can move +on to running another greenlet, perhaps one that itself now has an +event ready for it. This repeated process of registering for events +and reacting to them as they arrive is the event loop. + +Unlike other network libraries, though in a similar fashion as +eventlet, gevent starts the event loop implicitly in a dedicated +greenlet. There's no ``reactor`` that you must call a ``run()`` or +``dispatch()`` function on. When a function from gevent's API wants to +block, it obtains the :class:`gevent.hub.Hub` instance --- a special +greenlet that runs the event loop --- and switches to it (it is said +that the greenlet *yielded* control to the Hub). If there's no +:class:`~gevent.hub.Hub` instance yet, one is automatically created. + +.. tip:: Each operating system thread has its own + :class:`~gevent.hub.Hub`. This makes it possible to use the + gevent blocking API from multiple threads (with care). + +The event loop uses the best polling mechanism available on the system +by default. + +.. note:: + + A low-level event loop API is available under the + :mod:`gevent.core` module. This module is not documented, not meant + for general purpose usage, and its exact contents and semantics + change slightly depending on whether the libev or libuv event loop + is being used. The callbacks supplied to the event loop API are run + in the :class:`~gevent.hub.Hub` greenlet and thus cannot use the + synchronous gevent API. It is possible to use the asynchronous API + there, like :func:`gevent.spawn` and + :meth:`gevent.event.Event.set`. + + +Cooperative multitasking +======================== + +.. currentmodule:: gevent + +The greenlets all run in the same OS thread and are scheduled +cooperatively. This means that until a particular greenlet gives up +control, (by calling a blocking function that will switch to the +:class:`~gevent.hub.Hub`), other greenlets won't get a chance to run. +This is typically not an issue for an I/O bound app, but one should be +aware of this when doing something CPU intensive, or when calling +blocking I/O functions that bypass the event loop. + +.. tip:: Even some apparently cooperative functions, like + :func:`gevent.sleep`, can temporarily take priority over + waiting I/O operations in some circumstances. + +Synchronizing access to objects shared across the greenlets is +unnecessary in most cases (because yielding control is usually +explict), thus traditional synchronization devices like the +:class:`gevent.lock.BoundedSemaphore`, :class:`gevent.lock.RLock` and +:class:`gevent.lock.Semaphore` classes, although present, aren't used very +often. Other abstractions from threading and multiprocessing remain +useful in the cooperative world: + +- :class:`~event.Event` allows one to wake up a number of greenlets + that are calling :meth:`~event.Event.wait` method. +- :class:`~event.AsyncResult` is similar to :class:`~event.Event` but + allows passing a value or an exception to the waiters. +- :class:`~queue.Queue` and :class:`~queue.JoinableQueue`. + +.. _greenlet-basics: + +Lightweight pseudothreads +========================= + +.. currentmodule:: gevent + +New greenlets are spawned by creating a :class:`~Greenlet` instance +and calling its :meth:`start ` method. (The +:func:`gevent.spawn` function is a shortcut that does exactly that). +The :meth:`start ` method schedules a switch to +the greenlet that will happen as soon as the current greenlet gives up +control. If there is more than one active greenlet, they will be +executed one by one, in an undefined order as they each give up +control to the :class:`~gevent.hub.Hub`. + +If there is an error during execution it won't escape the greenlet's +boundaries. An unhandled error results in a stacktrace being printed, +annotated by the failed function's signature and arguments: + + >>> gevent.spawn(lambda : 1/0) + >>> gevent.sleep(1) + Traceback (most recent call last): + ... + ZeroDivisionError: integer division or modulo by zero + > failed with ZeroDivisionError + +The traceback is asynchronously printed to ``sys.stderr`` when the greenlet dies. + +:class:`Greenlet` instances have a number of useful methods: + +- :meth:`join ` -- waits until the greenlet exits; +- :meth:`kill ` -- interrupts greenlet's execution; +- :meth:`get ` -- returns the value returned by greenlet or re-raises the exception that killed it. + +Greenlets can be subclassed with care. One use for this is to +customize the string printed after the traceback by subclassing the +:class:`~gevent.Greenlet` class and redefining its ``__str__`` method. +For more information, see :ref:`subclassing-greenlet`. + + +Greenlets can be killed synchronously from another greenlet. Killing +will resume the sleeping greenlet, but instead of continuing +execution, a :exc:`GreenletExit` will be raised. + + >>> g = Greenlet(gevent.sleep, 4) + >>> g.start() + >>> g.kill() + >>> g.dead + True + +The :exc:`GreenletExit` exception and its subclasses are handled +differently than other exceptions. Raising :exc:`~GreenletExit` is not +considered an exceptional situation, so the traceback is not printed. +The :exc:`~GreenletExit` is returned by :meth:`get +` as if it were returned by the greenlet, not +raised. + +The :meth:`kill ` method can accept a custom exception to be raised: + + >>> g = Greenlet.spawn(gevent.sleep, 5) # spawn() creates a Greenlet and starts it + >>> g.kill(Exception("A time to kill")) + Traceback (most recent call last): + ... + Exception: A time to kill + Greenlet(5) failed with Exception + +The :meth:`kill ` can also accept a *timeout* +argument specifying the number of seconds to wait for the greenlet to +exit. Note that :meth:`kill ` cannot guarantee +that the target greenlet will not ignore the exception (i.e., it might +catch it), thus it's a good idea always to pass a timeout to +:meth:`kill ` (otherwise, the greenlet doing the +killing will remain blocked forever). + +.. tip:: The exact timing at which an exception is raised within a + target greenlet as the result of :meth:`kill + ` is not defined. See that function's + documentation for more details. + +.. caution:: + Use care when killing greenlets, especially arbitrary + greenlets spawned by a library or otherwise executing code you are + not familiar with. If the code being executed is not prepared to + deal with exceptions, object state may be corrupted. For example, + if it has acquired a ``Lock`` but *does not* use a ``finally`` + block to release it, killing the greenlet at the wrong time could + result in the lock being permanently locked:: + + def func(): + # DON'T DO THIS + lock.acquire() + socket.sendall(data) # This could raise many exceptions, including GreenletExit + lock.release() + + `This document + `_ + describes a similar situation for threads. + +Timeouts +======== + +Many functions in the gevent API are synchronous, blocking the current +greenlet until the operation is done. For example, :meth:`kill +` waits until the target greenlet is +:attr:`~gevent.Greenlet.dead` before returning [#f1]_. Many of +those functions can be made asynchronous by passing the keyword argument +``block=False``. + +Furthermore, many of the synchronous functions accept a *timeout* +argument, which specifies a limit on how long the function can block +(examples include :meth:`gevent.event.Event.wait`, +:meth:`gevent.Greenlet.join`, :meth:`gevent.Greenlet.kill`, +:meth:`gevent.event.AsyncResult.get`, and many more). + +The :class:`socket ` and :class:`SSLObject +` instances can also have a timeout, set by the +:meth:`settimeout ` method. + +When these are not enough, the :class:`gevent.Timeout` class and +:func:`gevent.with_timeout` can be used to add timeouts to arbitrary +sections of (cooperative, yielding) code. + + +Further Reading +=============== + +To limit concurrency, use the :class:`gevent.pool.Pool` class (see +:doc:`examples/dns_mass_resolve`). + +Gevent comes with TCP/SSL/HTTP/WSGI servers. See :doc:`servers`. + +There are a number of configuration options for gevent. See +:ref:`gevent-configuration` for details. This document also explains how +to enable gevent's builtin monitoring and debugging features. + +The objects in :mod:`gevent.util` may be helpful for monitoring and +debugging purposes. + +See :doc:`api/index` for a complete API reference. + + +External resources +================== + +`Gevent for working Python developer`__ is a comprehensive tutorial. + +__ http://sdiehl.github.io/gevent-tutorial/ + +.. rubric:: Footnotes + +.. [#f1] This was not the case before 0.13.0, :meth:`kill ` method in 0.12.2 and older was asynchronous by default. + +.. LocalWords: Greenlets diff --git a/doc/loop_impls.rst b/doc/loop_impls.rst new file mode 100644 index 0000000..518fe5d --- /dev/null +++ b/doc/loop_impls.rst @@ -0,0 +1,244 @@ +============================================= + Event Loop Implementations: libuv and libev +============================================= + +.. versionadded:: 1.3 + +gevent offers a choice of two event loop libraries (`libev`_ and +`libuv`_) and three event loop implementations. This document will +explore those implementations and compare them to each other. + +Using A Non-Default Loop +======================== + +First, we will describe how to choose an event loop other than the +default loop for a given platform. This is done by setting the +``GEVENT_LOOP`` environment variable before starting the program, or +by setting :attr:`gevent.config.loop ` in +code. + +.. important:: + + If you choose to configure the loop in Python code, it must be done + *immediately* after importing gevent and before any other gevent + imports or asynchronous operations are done, preferably at the top + of your program, right above monkey-patching (if done):: + + import gevent + gevent.config.loop = "libuv" + +Loop Implementations +==================== + +Here we will describe the available loop implementations. + ++----------+-------+------------+------------+-----+--------------+---------+--------+ +|Name |Library|Default |Interpreters|Age |Implementation|Build |Embedded| +| | | | | | |Status | | ++==========+=======+============+============+=====+==============+=========+========+ +|libev |libev |CPython on |CPython only|7 |Cython |Default |Default;| +| | |non-Windows | |years| | |optional| +| | |platforms | | | | | | ++----------+-------+------------+------------+-----+--------------+---------+--------+ +|libev-cffi|libev |PyPy on |CPython and |3 |CFFI |Optional;|Required| +| | |non-Windows |PyPy |years| |default | | +| | |platforms | | | | | | ++----------+-------+------------+------------+-----+--------------+---------+--------+ +|libuv |libuv |All |CPython and |1 |CFFI |Optional;|Required| +| | |interpreters|PyPy |years| |default | | +| | |on Windows | | | | | | ++----------+-------+------------+------------+-----+--------------+---------+--------+ + +.. _libev-impl: + +libev +----- + +`libev`_ is a venerable event loop library that has been the default +in gevent since 1.0a1 in 2011 when it replaced libevent. libev has +existed since 2007. + +.. note:: + + In the future, this Cython implementation may be deprecated to be + replaced with :ref:`libev-cffi`. + +.. _libev-dev: + +.. rubric:: Development and Source + +libev is a stable library and does not change quickly. Changes are +accepted in the form of patches emailed to a mailing list. Due to its +age and its portability requirements, it makes heavy use of +preprocessor macros, which some may find hinders readability of the +source code. + +.. _libev-plat: + +.. rubric:: Platform Support + +gevent tests libev on Linux, and macOS (previously it was also tested +on Windows). There is no known list of platforms officially supported +by libev, although FreeBSD, OpenBSD and Solaris/SmartOS have been +reported to work with gevent on libev at various points. + +On Windows, libev has `many limitations +`_. +gevent relies on the Microsoft C runtime functions to map from Windows +socket handles to integer file descriptors for libev using a somewhat +complex mapping; this prevents the CFFI implementation from being +used (which in turn prevents PyPy from using libev on Windows). + +There is no known public CI infrastructure for libev itself. + +.. _libev-cffi: + +libev-cffi +---------- + +This uses libev exactly as above, but instead of using Cython it uses +CFFI. That makes it suitable (and the default) for PyPy. It can also +make it easier to debug, since more details are preserved for +tracebacks. + + +.. note:: + + In the future, this CFFI implementation may become the default and replace + :ref:`libev-impl`. + +.. rubric:: When To Use + +On PyPy or when debugging. + + +libuv +----- + +libuv is an event loop library developed since 2011 for the use of +node 0.5. It was originally a wrapper around libev on non-Windows +platforms and directly used the native Windows IOCP support on Windows +(this code was contributed by Microsoft). Now it has its own loop +implementation on all supported platforms. + +libuv provides libev-like `"poll handles" +`_, and in gevent 1.3 that is +what gevent uses for IO. But libuv also provides higher-level +abstractions around read and write requests that may offer improved +performance. In the future, gevent might use those abstractions. + +.. note:: + + In the future, this implementation may become the default on all + platforms. + +.. rubric:: Development and Source + +libuv is developed by the libuv organization on `github +`_. It has a large, active community +and is used in many popular projects including node.js. + +The source code is written in a clean and consistent coding style, +potentially making it easier to read and debug. + +.. rubric:: Platform Support + +gevent tests libuv on Linux, Windows and macOS. libuv publishes an +extensive list of `supported platforms +`_ +that are likely to work with gevent. libuv `maintains a public CI +infrastructure `_. + +.. rubric:: When To Use libuv + + +- You want to use PyPy on Windows. +- You want to develop on Windows (Windows is not recommended for + production). +- You want to use an operating system not supported by libev such as + IBM i. + + .. note:: + + Platforms other than Linux, macOS and Windows are not + tested by gevent. + +.. _libuv-limits: + +Limitations and Differences +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because of its newness, and because of some design decisions inherent +in the library and the ecosystem, there are some limitations and +differences in the way gevent behaves using libuv compared to libev. + +- libuv support is not available in the manylinux wheels uploaded to + PyPI. The manylinux specification requires glibc 2.5, while libuv + requires glibc 2.12. Install from source to access libuv on Linux + (e.g., pip's ``--no-binary`` option). + +- Timers (such as ``gevent.sleep`` and ``gevent.Timeout``) only + support a resolution of 1ms (in practice, it's closer to 1.5ms). + Attempting to use something smaller will automatically increase it + to 1ms and issue a warning. Because libuv only supports millisecond + resolution by rounding a higher-precision clock to an integer number + of milliseconds, timers apparently suffer from more jitter. + +- Using negative timeouts may behave differently from libev. + +- libuv blocks delivery of all signals, so signals are handled using + an (arbitrary) 0.3 second timer. This means that signal handling + will be delayed by up to that amount, and that the longest the + event loop can sleep in the operating system's ``poll`` call is + that amount. Note that this is what gevent does for libev on + Windows too. + +- libuv only supports one io watcher per file descriptor, whereas + libev and gevent have always supported many watchers using + different settings. The libev behaviour is emulated at the Python + level. + +- Looping multiple times and expecting events for the same file + descriptor to be raised each time without any data being read or + written (as works with libev) does not appear to work correctly on + Linux when using ``gevent.select.poll`` or a monkey-patched + ``selectors.PollSelector``. + +- The build system does not support using a system libuv; the + embedded copy must be used. Using setuptools to compile libuv was + the most portable method found. + +- If anything unexpected happens, libuv likes to ``abort()`` the + entire process instead of reporting an error. For example, closing + a file descriptor it is using in a watcher may cause the entire + process to be exited. + +- The order in which timers and other callbacks are invoked may be + different than in libev. In particular, timers and IO callbacks + happen in a different order, and timers may easily be off by up to + half of the nominal 1ms resolution. See :issue:`1057`. + +- There is no support for priorities within classes of watchers. libev + has some support for priorities and this is exposed in the low-level + gevent API, but it was never documented. + +- Low-level ``fork`` and ``child`` watchers are not available. gevent + emulates these in Python on platforms that supply :func:`os.fork`. + Child watchers use ``SIGCHLD``, just as on libev, so the same + caveats apply. + +- Low-level ``prepare`` watchers are not available. gevent uses + prepare watchers for internal purposes. If necessary, this could be + emulated in Python. + +Performance +=========== + +In the various micro-benchmarks gevent has, performance among all three +loop implementations is roughly the same. There doesn't seem to be a +clear winner or loser. + +.. _libev: http://software.schmorp.de/pkg/libev.html +.. _libuv: http://libuv.org + +.. LocalWords: gevent libev cffi PyPy CFFI libuv FreeBSD CPython Cython diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 0000000..b89dc4e --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,113 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +set SPHINXBUILD=sphinx-build +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\gevent.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\gevent.ghc + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/doc/monitoring.rst b/doc/monitoring.rst new file mode 100644 index 0000000..a533854 --- /dev/null +++ b/doc/monitoring.rst @@ -0,0 +1,103 @@ +============================================== + Monitoring and Debugging gevent Applications +============================================== + +gevent applications are often long-running server processes. Beginning +with version 1.3, gevent has special support for monitoring such +applications and getting visibility into them. + +.. tip:: + + For some additional tools, see the comments on `issue 1021 + `_. + +The Monitor Thread +================== + +gevent can be :attr:`configured +` to start a native thread to +watch over each hub it creates. Out of the box, that thread has +support to watch two things, but you can :func:`add your own functions +` to be +called periodically in this thread. + +Blocking +-------- + +When the monitor thread is enabled, by default it will watch for +greenlets that block the event loop for longer than a +:attr:`configurable ` time +interval. When such a blocking greenlet is detected, it will print +:func:`a report ` to the hub's +:attr:`~gevent.hub.Hub.exception_stream`. It will also emit the +:class:`gevent.events.EventLoopBlocked` event. + +.. seealso:: :func:`gevent.util.assert_switches` + + For a scoped version of this. + +Memory Usage +------------ + +Optionally, you can set a :attr:`memory limit +`. The monitor thread will +check the process's memory usage every +:attr:`~gevent._config.Config.memory_monitor_period` seconds, and if +it is found to exceed this value, the +:class:`gevent.events.MemoryUsageThresholdExceeded` event will be +emitted. If in the future memory usage declines below the configured +value, the :class:`gevent.events.MemoryUsageUnderThreshold` event will +be emitted. + +.. important:: + + `psutil `_ must be + installed to monitor memory usage. + +Visibility +========== + +It is sometimes useful to get an overview of all existing greenlets +and their stack traces. The function +:func:`gevent.util.print_run_info` will collect this info and print it +(:func:`gevent.util.format_run_info` only collects and returns this +information). The greenlets are organized into a tree based on the +greenlet that spawned them. + +The ``print_run_info`` function is commonly hooked up to a signal +handler to get the application state at any given time. + +For each greenlet the following information is printed: + +- Its current execution stack +- If it is not running, its termination status and + :attr:`gevent.Greenlet.value` or + :attr:`gevent.Greenlet.exception` +- The :attr:`stack at which it was spawned + ` +- Its parent (usually the hub) +- Its :attr:`~gevent.Greenlet.minimal_ident` +- Its :attr:`~gevent.Greenlet.name` +- The :attr:`spawn tree locals ` + (only for the root of the spawn tree). +- The dicts of all :class:`gevent.local.local` objects that are used + in the greenlet. + +The greenlet tree itself is represented as an object that you can also +use for your own purposes: :class:`gevent.util.GreenletTree`. + +Profiling +========= + +The github repository `nylas/nylas-perftools +`_ has some +gevent-compatible profilers. + +- ``stacksampler`` is a sampling profiler meant to be run in a + greenlet in your server process and exposes data through an HTTP + server; it is designed to be suitable for production usage. +- ``py2devtools`` is a greenlet-aware tracing profiler that outputs data + that can be used by the Chrome dev tools; it is intended for + developer usage. + +.. LocalWords: greenlets gevent greenlet diff --git a/doc/mysphinxext.py b/doc/mysphinxext.py new file mode 100644 index 0000000..ed72106 --- /dev/null +++ b/doc/mysphinxext.py @@ -0,0 +1,74 @@ +from __future__ import print_function +from sphinx.ext.autodoc import cut_lines +from sphinx.ext import intersphinx +from docutils import nodes + +noisy = 0 +message_cache = set() + + +def missing_reference(app, env, node, contnode): + """Search the index for missing references. + For example, resolve :class:`Event` to :class:`Event `""" + # XXX methods and functions resolved by this function miss their () + + if intersphinx.missing_reference(app, env, node, contnode) is not None: + # is there a better way to give intersphinx a bigger priority? + return + + env = app.builder.env + + type = node['reftype'] + target = node['reftarget'] + modname = node.get('py:module') + classname = node.get('py:class') + + if modname and classname: + return + + def new_reference(refuri, reftitle): + newnode = nodes.reference('', '') + newnode['refuri'] = refuri + newnode['reftitle'] = reftitle + newnode['py:class'] = 'external-xref' + newnode['classname'] = 'external-xref' + newnode.append(contnode) + msg = 'Resolved missing-reference: :%5s:`%s` -> %s' % (type, target, refuri) + if noisy >= 1 or msg not in message_cache: + print(msg) + message_cache.add(msg) + return newnode + + if noisy >= 1: + print('Looking for %s' % [type, target, modname, classname]) + print(node) + + for docname, items in env.indexentries.items(): + if noisy >= 2: + print(docname) + for _x in items: + if noisy >= 4: + print(_x) + (i_type, i_string, i_target, i_aliasname) = _x[:4] + if noisy >= 3: + print('---', [i_type, i_string, i_target, i_aliasname]) + if i_aliasname.endswith(target): + stripped_aliasname = i_aliasname[len(docname):] + if stripped_aliasname: + assert stripped_aliasname[0] == '.', repr(stripped_aliasname) + stripped_aliasname = stripped_aliasname[1:] + if stripped_aliasname == target: + if noisy >= 1: + print('--- found %s %s in %s' % (type, target, i_aliasname)) + return new_reference(docname + '.html#' + i_aliasname, i_aliasname) + + if type == 'mod': + modules = [x for x in env.indexentries.keys() if x.startswith('gevent.')] + target = 'gevent.' + target + if target in modules: + return new_reference(target + '.html', target) + + +def setup(app): + app.connect('missing-reference', missing_reference) + app.connect('autodoc-process-docstring', cut_lines(2, what=['module'])) diff --git a/doc/mytheme/changes/frameset.html b/doc/mytheme/changes/frameset.html new file mode 100644 index 0000000..9d9af9e --- /dev/null +++ b/doc/mytheme/changes/frameset.html @@ -0,0 +1,11 @@ + + + + {% trans version=version|e, docstitle=docstitle|e %}Changes in Version {{ version }} — {{ docstitle }}{% endtrans %} + + + + + + diff --git a/doc/mytheme/changes/rstsource.html b/doc/mytheme/changes/rstsource.html new file mode 100644 index 0000000..abd12c1 --- /dev/null +++ b/doc/mytheme/changes/rstsource.html @@ -0,0 +1,15 @@ + + + + {% trans filename=filename, docstitle=docstitle|e %}{{ filename }} — {{ docstitle }}{% endtrans %} + + + + 
+      {{ text }}
+
+ + diff --git a/doc/mytheme/changes/versionchanges.html b/doc/mytheme/changes/versionchanges.html new file mode 100644 index 0000000..09651bf --- /dev/null +++ b/doc/mytheme/changes/versionchanges.html @@ -0,0 +1,33 @@ +{% macro entries(changes) %} +
    {% for entry, docname, lineno in changes %} +
  • {{ entry }}
    • +{% endfor %}
+{% endmacro -%} + + + + + + {% trans version=version|e, docstitle=docstitle|e %}Changes in Version {{ version }} — {{ docstitle }}{% endtrans %} + + +
+
+

{% trans version=version|e %}Automatically generated list of changes in version {{ version }}{% endtrans %}

+

{{ _('Library changes') }}

+ {% for modname, changes in libchanges %} +

{{ modname }}

+ {{ entries(changes) }} + {% endfor %} +

{{ _('C API changes') }}

+ {{ entries(apichanges) }} +

{{ _('Other changes') }}

+ {% for (fn, title), changes in otherchanges %} +

{{ title }} ({{ fn }})

+ {{ entries(changes) }} + {% endfor %} +
+
+ + diff --git a/doc/mytheme/defindex.html b/doc/mytheme/defindex.html new file mode 100644 index 0000000..40f4f4c --- /dev/null +++ b/doc/mytheme/defindex.html @@ -0,0 +1,26 @@ +{% extends "layout.html" %} +{% set title = _('Overview') %} +{% block body %} +

{{ docstitle|e }}

+

+ Welcome! This is + {% block description %}the documentation for {{ project|e }} + {{ release|e }}{% if last_updated %}, last updated {{ last_updated|e }}{% endif %}{% endblock %}. +

+ {% block tables %} +

{{ _('Indices and tables:') }}

+ + +
+

{{ _('Complete Table of Contents') }}
+ {{ _('lists all sections and subsections') }}

+

{{ _('Search Page') }}
+ {{ _('search this documentation') }}

+ 		+

{{ _('Global Module Index') }}
+ {{ _('quick access to all modules') }}

+

{{ _('General Index') }}
+ {{ _('all functions, classes, terms') }}

+
+ {% endblock %} +{% endblock %} diff --git a/doc/mytheme/domainindex.html b/doc/mytheme/domainindex.html new file mode 100644 index 0000000..0aca7e6 --- /dev/null +++ b/doc/mytheme/domainindex.html @@ -0,0 +1,57 @@ +{# + basic/domainindex.html + ~~~~~~~~~~~~~~~~~~~~~~ + + Template for domain indices (module index, ...). + + :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +#} +{% extends "layout.html" %} +{% set title = indextitle %} +{% block extrahead %} +{{ super() }} +{% if not embedded and collapse_index %} + +{% endif %} +{% endblock %} +{% block body %} + + {%- set curr_group = 0 %} + +

{{ indextitle }}

+ +
+ {%- for (letter, entries) in content %} + {{ letter }} + {%- if not loop.last %} | {% endif %} + {%- endfor %} +
+ + + {%- for letter, entries in content %} + + + {%- for (name, grouptype, page, anchor, extra, qualifier, description) + in entries %} + {%- if grouptype == 1 %}{% set curr_group = curr_group + 1 %}{% endif %} + + + + {%- endfor %} + {%- endfor %} +
 
+ {{ letter }}
{% if grouptype == 1 -%} + + {%- endif %}{% if grouptype == 2 %}   {% endif %} + {% if page %}{% endif -%} + {{ name|e }} + {%- if page %}{% endif %} + {%- if extra %} ({{ extra|e }}){% endif -%} + {% if qualifier %}{{ qualifier|e }}:{% endif %} + {{ description|e }}
+ +{% endblock %} diff --git a/doc/mytheme/genindex-single.html b/doc/mytheme/genindex-single.html new file mode 100644 index 0000000..9aaaeb0 --- /dev/null +++ b/doc/mytheme/genindex-single.html @@ -0,0 +1,46 @@ +{% extends "layout.html" %} +{% set title = _('Index') %} +{% block body %} + +

{% trans key=key %}Index – {{ key }}{% endtrans %}

+ +
+
+{%- set breakat = count // 2 %} +{%- set numcols = 1 %} +{%- set numitems = 0 %} +{% for entryname, (links, subitems) in entries %} +
{%- if links -%}{{ entryname|e }} + {%- for link in links[1:] %}, [{{ loop.index }}]{% endfor -%} + {%- else -%} +{{ entryname|e }} + {%- endif -%}
+ {%- if subitems %} +
+ {%- for subentryname, subentrylinks in subitems %} +
{{ subentryname|e }} + {%- for link in subentrylinks[1:] %}, [{{ loop.index }}]{% endfor -%} +
+ {%- endfor %} +
+ {%- endif -%} +{%- set numitems = numitems + 1 + (subitems|length) -%} +{%- if numcols < 2 and numitems > breakat -%} +{%- set numcols = numcols+1 -%} +
+{%- endif -%} +{%- endfor %} +
+ +{% endblock %} + +{% block sidebarrel %} +

Index

+

{% for key, dummy in genindexentries -%} + {{ key }} + {% if not loop.last %}| {% endif %} + {%- endfor %}

+ +

{{ _('Full index on one page') }}

+ {{ super() }} +{% endblock %} diff --git a/doc/mytheme/genindex-split.html b/doc/mytheme/genindex-split.html new file mode 100644 index 0000000..ab099e5 --- /dev/null +++ b/doc/mytheme/genindex-split.html @@ -0,0 +1,30 @@ +{% extends "layout.html" %} +{% set title = _('Index') %} +{% block body %} + +

{{ _('Index') }}

+ +

{{ _('Index pages by letter') }}:

+ +

{% for key, dummy in genindexentries -%} + {{ key }} + {% if not loop.last %}| {% endif %} + {%- endfor %}

+ +

{{ _('Full index on one page') }} + ({{ _('can be huge') }})

+ +{% endblock %} + +{% block sidebarrel %} +{% if split_index %} +

Index

+

{% for key, dummy in genindexentries -%} + {{ key }} + {% if not loop.last %}| {% endif %} + {%- endfor %}

+ +

{{ _('Full index on one page') }}

+{% endif %} + {{ super() }} +{% endblock %} diff --git a/doc/mytheme/genindex.html b/doc/mytheme/genindex.html new file mode 100644 index 0000000..d46b673 --- /dev/null +++ b/doc/mytheme/genindex.html @@ -0,0 +1,58 @@ +{% extends "layout.html" %} +{% set title = _('Index') %} +{% block body %} + +

{{ _('Index') }}

+ + {% for key, dummy in genindexentries -%} + {{ key }} {% if not loop.last %}| {% endif %} + {%- endfor %} + +
+ + {% for key, entries in genindexentries %} +

{{ key }}

+
+
+{%- set breakat = genindexcounts[loop.index0] // 2 %} +{%- set numcols = 1 %} +{%- set numitems = 0 %} +{% for entryname, links_subitems in entries %} +{%- set links, subitems = links_subitems[:2] %} +
{%- if links -%}{{ entryname|e }} + {%- for link in links[1:] %}, [{{ loop.index }}]{% endfor -%} + {%- else -%} +{{ entryname|e }} + {%- endif -%}
+ {%- if subitems %} +
+ {%- for subentryname, subentrylinks in subitems %} +
{{ subentryname|e }} + {%- for link in subentrylinks[1:] %}, [{{ loop.index }}]{% endfor -%} +
+ {%- endfor %} +
+ {%- endif -%} +{%- set numitems = numitems + 1 + (subitems|length) -%} +{%- if numcols < 2 and numitems > breakat -%} +{%- set numcols = numcols+1 -%} +
+{%- endif -%} +{%- endfor %} +
+{% endfor %} + +{% endblock %} + +{% block sidebarrel %} +{% if split_index %} +

{{ _('Index') }}

+

{% for key, dummy in genindexentries -%} + {{ key }} + {% if not loop.last %}| {% endif %} + {%- endfor %}

+ +

{{ _('Full index on one page') }}

+{% endif %} + {{ super() }} +{% endblock %} diff --git a/doc/mytheme/layout.html b/doc/mytheme/layout.html new file mode 100644 index 0000000..cad852d --- /dev/null +++ b/doc/mytheme/layout.html @@ -0,0 +1,269 @@ +{%- block doctype -%} + +{%- endblock %} +{%- set reldelim1 = reldelim1 is not defined and ' »' or reldelim1 %} +{%- set reldelim2 = reldelim2 is not defined and ' |' or reldelim2 %} +{%- set url_root = pathto('', 1) %} +{%- if url_root == '#' %}{% set url_root = '' %}{% endif %} + +{%- macro relbar() %} +
+

{{ _('Navigation') }}

+
    + {%- for rellink in rellinks %} +
  • + {{ rellink[3] }} + {%- if not loop.first %}{{ reldelim2 }}{% endif %}
    • + {%- endfor %} + {%- block rootrellink %} +
  • {{ shorttitle|e }}{{ reldelim1 }}
    • + {%- endblock %} + {%- for parent in parents %} +
  • {{ parent.title }}{{ reldelim1 }}
    • + {%- endfor %} + {%- block relbaritems %} {% endblock %} +
+
+{%- endmacro %} + +{%- macro sidebar() %} + {%- if not embedded %}{% if not theme_nosidebar|tobool %} +
+
+ {%- block sidebarlogo %} + {%- if logo %} + + {%- endif %} + {%- endblock %} + {%- block sidebartoc %} + {%- if display_toc %} +

{{ _('Table Of Contents') }}

+ {{ toc }} + + {%- else %} +

{{ _('Navigation') }}

+ + + {%- endif %} + {%- endblock %} + {%- block sidebarsourcelink %} + {%- if show_source and has_source and sourcename %} +

{{ _('This Page') }}

+ + {%- endif %} + {%- endblock %} + {%- if customsidebar %} + {% include customsidebar %} + {%- endif %} +{# + {%- block sidebarsearch %} + {%- if pagename != "search" %} + + + {%- endif %} + {%- endblock %} #} +
+
+ {%- endif %}{% endif %} +{%- endmacro %} + + + + + + {{ metatags }} + {%- if not embedded and docstitle %} + {%- set titlesuffix = " — "|safe + docstitle|e %} + {%- else %} + {%- set titlesuffix = "" %} + {%- endif %} + {{ title|striptags }}{{ titlesuffix }} + + + {%- if not embedded %} +{# + {%- for scriptfile in script_files %} + + {%- endfor %} + {%- if use_opensearch %} + + {%- endif %} #} + {%- if favicon %} + + {%- endif %} + {%- endif %} +{%- block linktags %} + {%- if hasdoc('about') %} + + {%- endif %} + {%- if hasdoc('genindex') %} + + {%- endif %} + {%- if hasdoc('search') %} + + {%- endif %} + {%- if hasdoc('copyright') %} + + {%- endif %} + + {%- if parents %} + + {%- endif %} + {%- if next %} + + {%- endif %} + {%- if prev %} + + {%- endif %} +{%- endblock %} +{%- block extrahead %} {% endblock %} + + + + +
+ + +
+ +
+ +
+{%- block document %} +
+{%- if not embedded %}{% if not theme_nosidebar|tobool %} +
+{%- endif %}{% endif %} +
+ + + {% block body %} {% endblock %} + + {%- if next %} +

Next page: {{ next.title }}

+ {%- endif %} + +
+{%- if not embedded %}{% if not theme_nosidebar|tobool %} +
+{%- endif %}{% endif %} +
+{%- endblock %} +
+
+ +
+
+ {%- block sidebar2 %}{{ sidebar() }}{% endblock %} +
+
+
 
+
+ + + +
+ + + diff --git a/doc/mytheme/modindex.html b/doc/mytheme/modindex.html new file mode 100644 index 0000000..0392edc --- /dev/null +++ b/doc/mytheme/modindex.html @@ -0,0 +1,42 @@ +{% extends "layout.html" %} +{% set title = _('Global Module Index') %} +{% block extrahead %} +{{ super() }} +{% if not embedded and collapse_modindex %} + +{% endif %} +{% endblock %} +{% block body %} + +

{{ _('Global Module Index') }}

+ + {%- for letter in letters %} + {{ letter }} {% if not loop.last %}| {% endif %} + {%- endfor %} +
+ + + {%- for modname, collapse, cgroup, indent, fname, synops, pform, dep, stripped in modindexentries %} + {%- if not modname -%} + + + {%- else -%} + + + + {%- endif -%} + {% endfor %} +
 
{{ fname }}
{% if collapse -%} + + {%- endif %}{% if indent %}   {% endif %} + {% if fname %}{% endif -%} + {{ stripped|e }}{{ modname|e }} + {%- if fname %}{% endif %} + {%- if pform and pform[0] %} ({{ pform|join(', ') }}){% endif -%} + {% if dep %}{{ _('Deprecated')}}:{% endif %} + {{ synops|e }}
+ +{% endblock %} diff --git a/doc/mytheme/page.html b/doc/mytheme/page.html new file mode 100644 index 0000000..17a9301 --- /dev/null +++ b/doc/mytheme/page.html @@ -0,0 +1,4 @@ +{% extends "layout.html" %} +{% block body %} + {{ body }} +{% endblock %} diff --git a/doc/mytheme/search.html b/doc/mytheme/search.html new file mode 100644 index 0000000..96c4065 --- /dev/null +++ b/doc/mytheme/search.html @@ -0,0 +1,45 @@ +{% extends "layout.html" %} +{% set title = _('Search') %} +{% set script_files = script_files + ['_static/searchtools.js'] %} +{% block body %} +

{{ _('Search') }}

+
+ +

+ {% trans %}Please activate JavaScript to enable the search + functionality.{% endtrans %} +

+
+

+ {% trans %}From here you can search these documents. Enter your search + words into the box below and click "search". Note that the search + function will automatically search for all of the words. Pages + containing fewer words won't appear in the result list.{% endtrans %} +

+
+ + + +
+ {% if search_performed %} +

{{ _('Search Results') }}

+ {% if not search_results %} +

{{ _('Your search did not match any results.') }}

+ {% endif %} + {% endif %} +
+ {% if search_results %} +
    + {% for href, caption, context in search_results %} +
  • {{ caption }} +
    {{ context|e }}
    +
    • + {% endfor %} +
+ {% endif %} +
+{% endblock %} +{% block footer %} + {{ super() }} + +{% endblock %} diff --git a/doc/mytheme/static/basic.css_t b/doc/mytheme/static/basic.css_t new file mode 100644 index 0000000..df55497 --- /dev/null +++ b/doc/mytheme/static/basic.css_t @@ -0,0 +1,1724 @@ +/* +Template name: Simple Organization +Template URI: http://templates.arcsin.se/simple-organization-website-template/ +Release date: 2009-09-20 +Last updated: 2009-09-24 +Description: A simple and elegant template suitable for organizations. +Author: Viktor Persson +Author URI: http://arcsin.se/ + +This template is licensed under a Creative Commons Attribution 2.5 License: +http://templates.arcsin.se/license/ +*/ + + +/* + Reset +------------------------------------------------------------------- */ + +html, body, div, span, object, iframe, h1, h2, h3, h4, h5, h6, p, blockquote, /*pre,*/ a, abbr, acronym, address, code, del, dfn, em, img, q, dl, dt, dd, ol, ul, li, fieldset, form, label, legend, table, caption, tbody, tfoot, thead, tr, th, td, textarea, input, select {margin: 0; padding: 0; border: 0; font-weight: inherit; font-style: inherit; font-size: 100%; font-family: inherit; vertical-align: baseline;} +table {border-collapse: collapse; border-spacing: 0;} +caption, th, td {text-align: left; font-weight: normal;} +table, td, th {vertical-align: middle;} +blockquote:before, blockquote:after, q:before, q:after {content: "";} +blockquote, q {quotes: "" "";} +a img {border: none;} +:focus {outline: 0;} + + +/* + General +------------------------------------------------------------------- */ + +html { + height: 100%; + padding-bottom: 1px; /* force scrollbars */ +} + +body { + background: #FFF; + color: #444; + font: normal 75% sans-serif; + line-height: 1.5; +} + + +/* + Typography +------------------------------------------------------------------- */ + +/* Headings */ + +h1,h2,h3,h4,h5,h6 { + color: #444; + font-weight: normal; + line-height: 1; + margin-bottom: 0.3em; +} +/*h4,h5,h6 {font-weight: bold;}*/ + +h1 {font-size: 2em;} +h2 {font-size: 2em;} +h3 {font-size: 1.5em;} +h4 {font-size: 1.25em;} +h5 {font-size: 1.1em;} +h6 {font-size: 1em;} + +h1 img, h2 img, h3 img, h4 img, h5 img, h6 img {margin: 0;} + +.document h1, .document h2, .document h3 { + /* label */ + border-left-style: solid; + border-left-width: 4px; + margin-bottom: 0.2em; + padding-left: 10px; + /* label-green */ + border-left-color: #B7D897; +} + +.document h1 {font-size: 2em; margin-bottom: 1em; } +.document h2 {font-size: 1.5em; margin-bottom: 1em; margin-top: 1em; } +.document h3 {font-size: 1.25em; margin-bottom: 1em; margin-top: 1em; } +.document h4 {font-size: 1.1em; margin-bottom: 1em; margin-top: 1em; } +.document h5 {font-size: 1em; margin-bottom: 1em; margin-top: 1em; } + +.title {color: #7c9a5e;} + +/* Links */ + +a:focus,a:hover {color: {{ theme_linkcolor }}; /*#039;*/} +a { + color: #456; + text-decoration: none; +} +a:hover {text-decoration: underline;} + +a.feed { + background: url('img/icon-feed.gif') no-repeat left center; + padding-left: 18px; +} +a.more { + color: #579; + font-weight: bold; +} +a.more:hover {color: #234;} + + +h2 a {color: #444; text-decoration: none;} +h3 a {color: #444; text-decoration: none;} + +h2 a:hover {color: #000; text-decoration: none;} +h3 a:hover {color: #000; text-decoration: none;} + + +a .regular {color: #444; } + +a.nobr { white-space: nowrap; } + + +/* Text elements */ + +p {margin-bottom: 1em; + margin-top: 1em; +} + +abbr, acronym {border-bottom: 1px dotted #666;} +address {margin-bottom: 1.5em;} +blockquote {margin: 1.5em;} +del, blockquote { color:#666; } +em, dfn, blockquote, address {font-style: italic;} +strong, dfn {font-weight: bold;} +sup, sub {line-height: 0;} + +/*pre { + margin: 1.5em 0; + white-space: pre; +} +pre,code,tt { + font: 1em monospace; + line-height: 1.5; +}*/ + + +/* Lists */ + +li ul, li ol {margin-left: 1.5em;} +ul, ol {margin: 1.5em 0 1.5em 1.5em;} + +/*ul {list-style-type: disc;}*/ +ol { + /*list-style-type: decimal;*/ + margin-left: 1.9em; +} + +dl {margin: 0 0 1.5em 0;} +dl dt {font-weight: bold;} +dd {margin-left: 1.5em;} + + +/* Special lists */ + +ul.plain-list li, ul.nice-list li, ul.tabbed li { + list-style: none; + margin-top: 0; +} + +ul.tabbed { + display: inline; + margin: 0; +} +ul.tabbed li {float: left;} + +ul.plain-list {margin: 0;} + +ul.nice-list {margin-left: 0;} +ul.nice-list li { + border-top: 1px solid #EEE; + list-style: none; + padding: 4px 0; +} +ul.nice-list li:first-child {border-top: none;} +ul.nice-list li .right {color: #999;} + + +/* Tables */ + +table {margin-bottom: 1.4em; width: 100%;} +th {font-weight: bold;} +thead th {background: #C3D9FF;} +th,td,caption {padding: 4px 10px 4px 5px;} +tr.even td {background: #F2F6FA;} +tfoot {font-style: italic;} +caption {background: #EEE;} + +table.data-table { + border: 1px solid #CCB; + margin-bottom: 2em; + width: 100%; +} +table.data-table th { + background: #F0F0F0; + border: 1px solid #DDD; + color: #555; + text-align: left; +} +table.data-table tr {border-bottom: 1px solid #DDD;} +table.data-table td, table th {padding: 10px;} +table.data-table td { + background: #F6F6F6; + border: 1px solid #DDD; +} +table.data-table tr.even td {background: #FCFCFC;} + + +/* Misc classes */ + +.small {font-size: 0.9em;} +.smaller {font-size: 0.8em;} +.smallest {font-size: 0.7em;} + +.large {font-size: 1.15em;} +.larger {font-size: 1.25em;} +.largest {font-size: 1.35em;} + +.hidden {display: none;} + +.quiet, .quiet a {color: #999;} +.loud, .loud a {color: #000;} +.highlight, .highlight a {background:#ff0;} + +.text-left {text-align: left;} +.text-right {text-align: right;} +.text-center {text-align: center;} +.text-separator {padding: 0 5px;} + +.error, .notice, .success { + border: 1px solid #DDD; + margin-bottom: 1em; + padding: 0.6em 0.8em; +} + +.error {background: #FBE3E4; color: #8A1F11; border-color: #FBC2C4;} +.error a {color: #8A1F11;} + +.notice {background: #FFF6BF; color: #514721; border-color: #FFD324;} +.notice a {color: #514721;} + +.success {background: #E6EFC2; color: #264409; border-color: #C6D880;} +.success a {color: #264409;} + + +/* Labels */ +h1.label { + border-left-style: solid; + border-left-width: 4px; + margin-bottom: 0.2em; + padding-left: 10px; +} + +h1.label-blue {border-left-color: #55AADA;} +h1.label-green {border-left-color: #B7D897;} +h1.label-orange {border-left-color: #FA8F6F;} + + +h2.label { + border-left-style: solid; + border-left-width: 4px; + margin-bottom: 0.2em; + padding-left: 10px; +} + +h2.label-blue {border-left-color: #55AADA;} +h2.label-green {border-left-color: #B7D897;} +h2.label-orange {border-left-color: #FA8F6F;} + +/* + Forms +------------------------------------------------------------------- */ + +label { + cursor: pointer; + font-weight: bold; +} +label.checkbox, label.radio {font-weight: normal;} +legend { + font-weight: bold; + font-size: 1.2em; +} +textarea {overflow: auto;} +input.text, textarea, select { + background: #FCFCFC; + border: 1px inset #AAA; + margin: 0.5em 0; + padding: 4px 5px; +} +input.text:focus, textarea:focus, select:focus {background: #FFFFF5;} + +input.button { + background: #DDD; + border: 1px outset #AAA; + padding: 4px 5px; +} +input.button:active {border-style: inset;} + + +/* Specific */ + +form .required {font-weight: bold;} + +.form-error {border-color: #F00;} +.form-row {padding: 5px 0;} +.form-row-submit { + border-top: 1px solid #DDD; + padding: 8px 0 10px 76px; + margin-top: 10px; +} +.legend { + background: #F0FAF0; + border: 1px solid #D6DFD6; + font-size: 1.5em; + margin: 0; + padding: 8px 14px; +} +.form-property, .form-value {float: left;} +.form-property { + padding-top: 8px; + text-align: right; + width: 60px; +} +.form-value {padding-left: 16px;} +.form-error {border-color: #F00;} + + + +/* + Alignment +------------------------------------------------------------------- */ + +/* General */ + +.center,.aligncenter { + display: block; + margin-left: auto; + margin-right: auto; +} + + +/* Images */ + +img.bordered,img.alignleft,img.alignright,img.aligncenter { + background-color: #FFF; + border: 1px solid #DDD; + padding: 3px; +} +img.alignleft, img.left {margin: 0 1.5em 1em 0;} +img.alignright, img.right {margin: 0 0 1em 1.5em;} + + +/* Floats */ + +.left,.alignleft {float: left;} +.right,.alignright {float: right;} + +.clear,.clearer {clear: both;} +.clearer { + display: block; + font-size: 0; + line-height: 0; + height: 0; +} + + +/* + Separators +------------------------------------------------------------------- */ + +.content-separator, .archive-separator { + background: #E5E5E5; + clear: both; + color: #FFE; + display: block; + font-size: 0; + line-height: 0; + height: 1px; +} +.content-separator {margin: 32px 0;} +.archive-separator {margin-bottom: 20px;} + + +/* + Posts +------------------------------------------------------------------- */ + +.post {margin-bottom: 20px;} + +.post img.left, .post img.right {margin-bottom: 0;} + +.post-date { + color: #777; + margin: 2px 0 10px; +} +.post-date a {color: #444;} + +.post-meta a {color: #345; } +.post-meta a:hover {color: #001;} + +/*.body {font-size: 133.33333%;}*/ +.body {font-size: 1.1em;} +.body a {color: {{ theme_linkcolor }}; /*#039;*/} +.body a:hover {color: {{ theme_linkcolor }}; /*#039;*/} + +.body img.left, .body img.right {margin-bottom: 1em;} + + +/* Archives */ +.archive-pagination { + color: #777; + padding: 10px 0; +} +.archive-pagination-top { + border-bottom: 2px solid #DDD; + margin-bottom: 24px; +} +.archive-pagination-bottom { + border-top: 2px solid #DDD; + margin-top: 24px; +} + +.archive-post-date { + background: #F5F5F5; + border-bottom: 1px solid #C5C5C5; + border-right: 1px solid #CFCFCF; + float: left; + margin-right: 12px; + padding: 2px 0 5px; + text-align: center; + width: 46px; +} +.archive-post-title .post-date {margin: 0;} +.archive-post-title {padding-top: 4px;} +.archive-post-day {font: normal 1.6em Georgia,serif;} + + +/* + Comments +------------------------------------------------------------------- */ + +/* +.comment-input-text textarea {width: 80%;} + +// Comment list + +.comment-list-wrapper { + background: #F6F6F6; + margin: 10px 0 0; + padding: 5px 12px 10px 7px; +} +.comment-list { + margin: 0; + padding: 0; +} +.comment-list li {list-style: none;} +.comment-list ul {margin-bottom: 0;} + +.comment-profile-wrapper { + text-align: center; + width: 105px; +} + +.comment-gravatar {margin-bottom: 3px;} + +.comment-content-wrapper { + float: right; + width: 481px; +} + +.comment-parent, .comment-single {margin-top: 15px;} + +.comment-list ul.children, #comments #respond ul { + border-left: 1px solid #CCC; + margin: 0 0 0 130px; +} +.comment-list ul.children ul.children {margin-left: 15px;} + +.comment-list ul.children li { + background: url('img/comment-reply.gif') no-repeat left top; + margin: 0; + padding: 10px 0 0 15px; +} + +.comment-body { + background: #FFF; + border: 1px solid #DDD; + padding: 10px 12px 0; +} +.comment-list ul.children .comment-body {background: #FCFCFC;} + +.comment-author {padding-top: 2px;} + +.comment-text p {margin-bottom: 0.8em;} + +.comment .post-date, .comment-author {font-size: 0.9em;} +.comment .post-date .right a {color: #BBB;} +.comment .post-date .right a:hover {color: #234;} + +.comment-arrow { + background: url('img/comment-arrow.gif') no-repeat left top; + display: block; + float: left; + height: 45px; + margin: 3px 0 -45px -41px; + position: absolute; + width: 29px; +} + +// Respond + +#respond li {list-style: none;} +#respond { + background: #F6F6F6; + padding: 10px 12px; +} +#respond ul {margin: 0;} +#respond .legend {margin-bottom: 10px;} + +#comments #respond {padding: 0;} +#comments #respond .legend { + border-bottom: 0; + margin-bottom: 0; +} +#comments #respond ul { + background: url('img/comment-reply.gif') no-repeat left top; + padding: 10px 0 0 15px; +} +#comments ul.children #respond ul { + margin-left: 30px; + padding: 0; +} + +#comments #respond .comment-profile-wrapper, #comments #respond .comment-arrow {display: none;} +#comments #respond .comment-body {background: #FFF;} +#comments #respond .comment-content-wrapper { + float: none; + width: 100%; +} + +*/ + +/* + Layout +------------------------------------------------------------------- */ + +/* Common */ +#top, #sub-nav {border-bottom: 1px solid #DDD;} + + +/* Wrapper */ +#site-wrapper { + margin: 0 auto; + width: 920px; +} + + +/* Header */ +#header {padding-top: 24px;} + +/* Top */ +#top {padding-bottom: 32px;} + + +/* Logo */ +#logo { border-right: 1px solid #DDD; + padding: 10px 40px 10px 0; + margin-right: 40px; +} +#logo img {} + +/* Splash */ +#splash {padding-top: 32px;} + + +/* Navigation */ +.navigation a { + color: #888; + text-decoration: none; +} +.navigation a:hover {color: #002;} +.navigation li.current-tab a {color: #222;} + +#main-nav li:first-child, #sub-nav li:first-child {margin-left: 0;} + +/* Main navigation */ +#main-nav {padding-top: 0px;} +#main-nav li {margin: 0 1.5em;} +#main-nav a { + font-size: 1.45em; + line-height: 2em; + padding-bottom: 2px; +} +#main-nav li.current-tab a {color: #333;} +#main-nav a:hover {color: #002;} +#main-nav li.current-tab a {border-bottom: 2px solid #94CC5F;} +#main-nav li.current-tab a:hover {color: #002;} + +#title {color: #7c9a5e; text-decoration: none} +#title:hover {text-decoration: none} + + +/* Subnav */ +#sub-nav { + border-bottom: 1px solid #DDD; + padding: 12px 0; +} +#sub-nav a { + font-size: 1.2em; + text-decoration: none; +} +#sub-nav li {margin: 0 1em;} +#sub-nav li.current-tab a {font-weight: bold;} + + +/* Main */ +.main {margin: 24px 0;} + +.main#main-two-columns {background: url('img/main-two-columns.gif') repeat-y right top;} +.main#main-two-columns-left {background: url('img/main-two-columns-left.gif') repeat-y left top;} +.main#main-two-columns #main-content, .main#main-two-columns-left #main-content {width: 620px;} + +/* Sidebar */ +#sidebar {width: 255px;} + + +/* Columns */ +.col3, .col3-mid {width: 31%;} +.col3-mid {margin-left: 3%;} + +.col3big { width: 65% } + +/* Sections */ +.section {margin-bottom: 24px;} +.section-title { + background-color: #F9F9F9; + border-top: 2px solid #DDD; + color: #7A7A7A; + font: bold 1.2em sans-serif; + margin-bottom: 16px; + padding: 7px 10px 6px; +} +.section-title a {color: #7A7A7A;} +.section-title a:hover {color: #444; text-decoration: none;} + +#sidebar .section-title {margin-bottom: 8px;} + + +/* Footer */ + +#footer { + border-top: 1px solid #DDD; + color: #777; + padding: 16px 0 4px; +} +#footer-left {width: 259px;} +#footer-right { + width: 659px; + text-align: right; +} +#footer p {margin-bottom: 0.4em;} +#footer .text-separator { + padding: 0 3px; + color: #BBB; +} +#footer a:hover {color: #000;} + +#footer a.quiet-link {text-decoration: none; color: #777} +#footer a.quiet-link:hover {text-decoration: underline; color: #000} + + + +/* + Misc overriding classes +------------------------------------------------------------------- */ + +/* Border */ + +.noborder {border: 0;} +.notborder {border-top: 0;} +.norborder {border-right: 0;} +.nobborder {border-bottom: 0;} +.nolborder {border-left: 0;} + +/* Margin */ + +.nomargin {margin: 0;} +.notmargin {margin-top: 0;} +.normargin {margin-right: 0;} +.nobmargin {margin-bottom: 0;} +.nolmargin {margin-left: 0;} + +/* Padding */ + +.nopadding {padding: 0;} +.notpadding {padding-top: 0;} +.norpadding {padding-right: 0;} +.nobpadding {padding-bottom: 0;} +.nolpadding {padding-left: 0;} + + +/* + IE Fixes (zzz) +------------------------------------------------------------------- */ + +* html .navigation, * html #footer, * html #splash, * html .comment ul {height: 0.01%;} +* html #footer-left {width: 500px;} +.navigation, #splash, .comment ul {min-height: 0.01%;} + + + + + + + +/* Sphinx stylesheet */ + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +div.admonition dl { + margin-bottom: 0; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + + + +tt { + background-color: #ecf0f3; + padding: 0 1px 0 1px; + font-size: 110%; +} + +.warning tt { + background: #efc2c2 !important; +} + +.note tt { + background: #d6d6d6; +} + +dt:target, .highlight { + background-color: #fbe54e; +} + + + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: {{ theme_linkcolor }}; + text-decoration: none; +} + +/* +a:hover { + text-decoration: underline; +} +*/ + +a.headerlink { + color: {{ theme_headlinkcolor }}; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: {{ theme_headlinkcolor }}; + color: white; +} + + +/* -- general body styles --------------------------------------------------- */ + +a.headerlink { + visibility: hidden; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink { + visibility: visible; +} + + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; +} + +td.linenos pre { + padding: 5px 0px; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +tt.descname { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; +} + +tt.descclassname { + background-color: transparent; +} + +tt.xref, a tt { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; +} + +h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { + background-color: transparent; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.admonition p { + margin-bottom: 5px; +} + +div.admonition pre { + margin-bottom: 5px; +} + +div.admonition ul, div.admonition ol { + margin-bottom: 5px; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.topic { + background-color: #eee; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; +} + + +div.caution { + background-color: #fff6f1; + border: 1px solid #ffaaa3; +} + +div.hint, div.tip { + border-left-style: solid; + border-left-width: 2px; + border-left-color: #B7D897; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + + + +pre { + padding: 5px; + background-color: {{ theme_codebgcolor }}; + color: {{ theme_codetextcolor }}; + font-size: 120%; + line-height: 150%; + border: 1px solid #ac9; + border-left: none; + border-right: none; + /*font-family: {{ theme_bodyfont }};*/ +} + +/* -- other body styles ----------------------------------------------------- */ + +ol.arabic { + list-style: decimal; +} + +ol.loweralpha { + list-style: lower-alpha; +} + +ol.upperalpha { + list-style: upper-alpha; +} + +ol.lowerroman { + list-style: lower-roman; +} + +ol.upperroman { + list-style: upper-roman; +} + +dl { + margin-bottom: 15px; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +dt:target, .highlight { + background-color: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +th.field-name { + vertical-align: top; +} + +.refcount { + color: #060; +} + +.optional { + font-size: 1.3em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +.classifier { + font-style: oblique; +} + + + +ul ul { margin-top: 0em; margin-bottom: 0em; } + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + + + +/* + + +// +// Sphinx stylesheet -- basic theme +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// + +// -- main layout ----------------------------------------------------------- + +div.clearer { + clear: both; +} + +// -- relbar ---------------------------------------------------------------- + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +// -- sidebar --------------------------------------------------------------- + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +img { + border: 0; +} + +// -- search page ----------------------------------------------------------- + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li div.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +// -- index page ------------------------------------------------------------ + +table.contentstable { + width: 90%; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +// -- general index --------------------------------------------------------- + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable dl, table.indextable dd { + margin-top: 0; + margin-bottom: 0; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +// -- general body styles --------------------------------------------------- + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.field-list ul { + padding-left: 1em; +} + +.first { + margin-top: 0 !important; +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +.align-left { + text-align: left; +} + +.align-center { + clear: both; + text-align: center; +} + +.align-right { + text-align: right; +} + +// -- sidebars -------------------------------------------------------------- + +div.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px 7px 0 7px; + background-color: #ffe; + width: 40%; + float: right; +} + +p.sidebar-title { + font-weight: bold; +} + +// -- topics ---------------------------------------------------------------- + +div.topic { + border: 1px solid #ccc; + padding: 7px 7px 0 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +// -- tables ---------------------------------------------------------------- + +table.docutils { + border: 0; + border-collapse: collapse; +} + +table.docutils td, table.docutils th { + padding: 1px 8px 1px 0; + border-top: 0; + border-left: 0; + border-right: 0; + border-bottom: 1px solid #aaa; +} + +table.field-list td, table.field-list th { + border: 0 !important; +} + +table.footnote td, table.footnote th { + border: 0 !important; +} + +th { + text-align: left; + padding-right: 5px; +} + +table.citation { + border-left: solid 1px gray; + margin-left: 1px; +} + +table.citation td { + border-bottom: none; +} + +// -- other body styles ----------------------------------------------------- + +ol.arabic { + list-style: decimal; +} + +ol.loweralpha { + list-style: lower-alpha; +} + +ol.upperalpha { + list-style: upper-alpha; +} + +ol.lowerroman { + list-style: lower-roman; +} + +ol.upperroman { + list-style: upper-roman; +} + +dl { + margin-bottom: 15px; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +dt:target, .highlight { + background-color: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.refcount { + color: #060; +} + +.optional { + font-size: 1.3em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +.classifier { + font-style: oblique; +} + +// -- code displays --------------------------------------------------------- + +pre { + overflow: auto; +} + +td.linenos pre { + padding: 5px 0px; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +tt.descname { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; +} + +tt.descclassname { + background-color: transparent; +} + +tt.xref, a tt { + background-color: transparent; + font-weight: bold; +} + +h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt { + background-color: transparent; +} + +// -- math display ---------------------------------------------------------- + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +// -- printout stylesheet --------------------------------------------------- + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } + +} + + +// default theme + + +body { + font-family: {{ theme_bodyfont }}; + font-size: 100%; + background-color: {{ theme_footerbgcolor }}; + color: #000; + margin: 0; + padding: 0; +} + +div.document { + background-color: {{ theme_sidebarbgcolor }}; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 230px; +} + +div.body { + background-color: {{ theme_bgcolor }}; + color: {{ theme_textcolor }}; + padding: 0 20px 30px 20px; +} + +{%- if theme_rightsidebar|tobool %} +div.bodywrapper { + margin: 0 230px 0 0; +} +{%- endif %} + +div.footer { + color: {{ theme_footertextcolor }}; + width: 100%; + padding: 9px 0 9px 0; + text-align: center; + font-size: 75%; +} + +div.footer a { + color: {{ theme_footertextcolor }}; + text-decoration: underline; +} + +div.related { + background-color: {{ theme_relbarbgcolor }}; + line-height: 30px; + color: {{ theme_relbartextcolor }}; +} + +div.related a { + color: {{ theme_relbarlinkcolor }}; +} + +div.sphinxsidebar { + {%- if theme_stickysidebar|tobool %} + top: 30px; + bottom: 0; + margin: 0; + position: fixed; + overflow: auto; + height: auto; + {%- endif %} + {%- if theme_rightsidebar|tobool %} + float: right; + {%- if theme_stickysidebar|tobool %} + right: 0; + {%- endif %} + {%- endif %} +} + +{%- if theme_stickysidebar|tobool %} +// this is nice, but it it leads to hidden headings when jumping +// to an anchor +// +//div.related { +// position: fixed; +//} +// +//div.documentwrapper { +// margin-top: 30px; +//} + +{%- endif %} + +div.sphinxsidebar h3 { + font-family: {{ theme_headfont }}; + color: {{ theme_sidebartextcolor }}; + font-size: 1.4em; + font-weight: normal; + margin: 0; + padding: 0; +} + +div.sphinxsidebar h3 a { + color: {{ theme_sidebartextcolor }}; +} + +div.sphinxsidebar h4 { + font-family: {{ theme_headfont }}; + color: {{ theme_sidebartextcolor }}; + font-size: 1.3em; + font-weight: normal; + margin: 5px 0 0 0; + padding: 0; +} + +div.sphinxsidebar p { + color: {{ theme_sidebartextcolor }}; +} + +div.sphinxsidebar p.topless { + margin: 5px 10px 10px 10px; +} + +div.sphinxsidebar ul { + margin: 10px; + padding: 0; + color: {{ theme_sidebartextcolor }}; +} + +div.sphinxsidebar a { + color: {{ theme_sidebarlinkcolor }}; +} + +div.sphinxsidebar input { + border: 1px solid {{ theme_sidebarlinkcolor }}; + font-family: sans-serif; + font-size: 1em; +} + +// -- body styles ----------------------------------------------------------- + +a { + color: {{ theme_linkcolor }}; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +div.body p, div.body dd, div.body li { + text-align: justify; + line-height: 130%; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: {{ theme_headfont }}; + background-color: {{ theme_headbgcolor }}; + font-weight: normal; + color: {{ theme_headtextcolor }}; + border-bottom: 1px solid #ccc; + margin: 20px -20px 10px -20px; + padding: 3px 0 3px 10px; +} + +div.body h1 { margin-top: 0; font-size: 200%; } +div.body h2 { font-size: 160%; } +div.body h3 { font-size: 140%; } +div.body h4 { font-size: 120%; } +div.body h5 { font-size: 110%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: {{ theme_headlinkcolor }}; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: {{ theme_headlinkcolor }}; + color: white; +} + +div.body p, div.body dd, div.body li { + text-align: justify; + line-height: 130%; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.admonition p { + margin-bottom: 5px; +} + +div.admonition pre { + margin-bottom: 5px; +} + +div.admonition ul, div.admonition ol { + margin-bottom: 5px; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +div.topic { + background-color: #eee; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre { + padding: 5px; + background-color: {{ theme_codebgcolor }}; + color: {{ theme_codetextcolor }}; + border: 1px solid #ac9; + border-left: none; + border-right: none; +} + +tt { + background-color: #ecf0f3; + padding: 0 1px 0 1px; + font-size: 0.95em; +} + +.warning tt { + background: #efc2c2; +} + +.note tt { + background: #d6d6d6; +} + + + +*/ diff --git a/doc/mytheme/static/file.png b/doc/mytheme/static/file.png new file mode 100644 index 0000000..d18082e Binary files /dev/null and b/doc/mytheme/static/file.png differ diff --git a/doc/mytheme/static/img/main-two-columns.gif b/doc/mytheme/static/img/main-two-columns.gif new file mode 100644 index 0000000..d9d350b Binary files /dev/null and b/doc/mytheme/static/img/main-two-columns.gif differ diff --git a/doc/mytheme/static/minus.png b/doc/mytheme/static/minus.png new file mode 100644 index 0000000..da1c562 Binary files /dev/null and b/doc/mytheme/static/minus.png differ diff --git a/doc/mytheme/static/omegle_48.png b/doc/mytheme/static/omegle_48.png new file mode 100644 index 0000000..d200338 Binary files /dev/null and b/doc/mytheme/static/omegle_48.png differ diff --git a/doc/mytheme/static/plus.png b/doc/mytheme/static/plus.png new file mode 100644 index 0000000..b3cb374 Binary files /dev/null and b/doc/mytheme/static/plus.png differ diff --git a/doc/mytheme/static/spotify_logo.png b/doc/mytheme/static/spotify_logo.png new file mode 100644 index 0000000..0486b6d Binary files /dev/null and b/doc/mytheme/static/spotify_logo.png differ diff --git a/doc/mytheme/static/transparent.gif b/doc/mytheme/static/transparent.gif new file mode 100644 index 0000000..f32722a Binary files /dev/null and b/doc/mytheme/static/transparent.gif differ diff --git a/doc/mytheme/theme.conf b/doc/mytheme/theme.conf new file mode 100644 index 0000000..006a8cf --- /dev/null +++ b/doc/mytheme/theme.conf @@ -0,0 +1,30 @@ +[theme] +inherit = none +stylesheet = basic.css +pygments_style = sphinx + +[options] +gevent_version = 0.0.0 +nosidebar = false +rightsidebar = false +stickysidebar = false + +footerbgcolor = #11303d +footertextcolor = #ffffff +sidebarbgcolor = #1c4e63 +sidebartextcolor = #ffffff +sidebarlinkcolor = #98dbcc +relbarbgcolor = #133f52 +relbartextcolor = #ffffff +relbarlinkcolor = #ffffff +bgcolor = #ffffff +textcolor = #000000 +headbgcolor = #f2f2f2 +headtextcolor = #20435c +headlinkcolor = #c60f0f +linkcolor = #355f7c +codebgcolor = #eeffcc +codetextcolor = #333333 + +bodyfont = sans-serif +headfont = 'Trebuchet MS', sans-serif diff --git a/doc/older_releases.rst b/doc/older_releases.rst new file mode 100644 index 0000000..8afa33e --- /dev/null +++ b/doc/older_releases.rst @@ -0,0 +1,11 @@ +================================== + Information About Older Releases +================================== + +.. toctree:: + + + whatsnew_1_2 + whatsnew_1_1 + whatsnew_1_0 + changelog_pre diff --git a/doc/servers.rst b/doc/servers.rst new file mode 100644 index 0000000..846ffbd --- /dev/null +++ b/doc/servers.rst @@ -0,0 +1,65 @@ +.. _implementing-servers: + +====================== + Implementing servers +====================== + +.. currentmodule:: gevent.baseserver + +There are a few classes to simplify server implementation with gevent. +They all share a similar interface, inherited from :class:`BaseServer`:: + + def handle(socket, address): + print('new connection!') + + server = StreamServer(('127.0.0.1', 1234), handle) # creates a new server + server.start() # start accepting new connections + +At this point, any new connection accepted on ``127.0.0.1:1234`` will result in a new +:class:`gevent.Greenlet` spawned running the *handle* function. To stop a server use :meth:`BaseServer.stop` method. + +In case of a :class:`gevent.pywsgi.WSGIServer`, *handle* must be a WSGI application callable. + +It is possible to limit the maximum number of concurrent connections, +by passing a :class:`gevent.pool.Pool` instance. In addition, passing +a pool allows the :meth:`BaseServer.stop` method to kill requests that +are in progress:: + + pool = Pool(10000) # do not accept more than 10000 connections + server = StreamServer(('127.0.0.1', 1234), handle, spawn=pool) + server.serve_forever() + + +.. tip:: If you don't want to limit concurrency, but you *do* want to + be able to kill outstanding requests, use a pool created with + a size of ``None``. + + +The :meth:`BaseServer.serve_forever` method calls +:meth:`BaseServer.start` and then waits until interrupted or until the +server is stopped. + +The :mod:`gevent.pywsgi` module contains an implementation of a :pep:`3333` +:class:`WSGI server `. In addition, +gunicorn_ is a stand-alone server that supports gevent. + +API Reference +============= + +- :doc:`api/gevent.baseserver` +- :doc:`api/gevent.server` +- :doc:`api/gevent.pywsgi` + + +Examples +======== + +More :doc:`examples ` are available: + +- :doc:`examples/echoserver` - demonstrates :class:`gevent.server.StreamServer` +- :doc:`examples/wsgiserver` - demonstrates :class:`gevent.pywsgi.WSGIServer ` +- :doc:`examples/wsgiserver_ssl` - demonstrates :class:`WSGIServer with ssl ` + + + +.. _gunicorn: http://gunicorn.org diff --git a/doc/sfc.rst b/doc/sfc.rst new file mode 100644 index 0000000..8ed7c0d --- /dev/null +++ b/doc/sfc.rst @@ -0,0 +1,49 @@ +.. raw:: html + + + + + + + + + + diff --git a/doc/success.rst b/doc/success.rst new file mode 100644 index 0000000..63209be --- /dev/null +++ b/doc/success.rst @@ -0,0 +1,129 @@ +================= + Success stories +================= + +If you have a success story for gevent, contact post to the `google group`_. + +.. _google group: http://groups.google.com/group/gevent/ + + +Omegle_ +======= + +I've been using gevent to power Omegle, my high-volume chat site, +since 2010. Omegle is used by nearly half a million people every day, +and it has as many as 20,000 users chatting at any given time. It +needs to needs to perform well and be extremely reliable, and gevent +makes that easy to do: gevent gives you power to do more creative +things, and it's fast enough that you can more easily write apps that +stand up to a lot of load. + +gevent is well-engineered, and its development has been maintaining an +active, dedicated pace for as long as I've been following it. Any time +I've had an issue with gevent that I couldn't solve on my own, the +friendly community has been extremely helpful and knowledgeable. I +really think gevent is the best library of its type for Python right +now, and I would recommend it to anyone who needs a good networking +library. + +-- Leif K-Brooks, Founder, Omegle.com_ + +.. _Omegle: http://omegle.com +.. _Omegle.com: http://omegle.com + + +Pediapress_ +=========== + +Pediapress_ powers Wikipedia_'s PDF rendering cluster. I've started using +gevent in 2009 after our NFS based job queue showed serious performance +problems on Wikipedia's PDF rendering cluster. I've replaced that with +a gevent based job queue server in a short time. gevent is managing the +generation of around 100000 PDF files daily and is serving them to wikipedia users. + +Recently I've refactored the component that fetches articles and +images from wikipedia to use gevent instead of twisted. The code is +much cleaner and much more manageable then before. + +-- Ralf Schmitt, Developer, Pediapress_ + +.. _Pediapress: http://pediapress.com/ +.. _Wikipedia: http://www.wikipedia.org/ + + +`ESN Social Software`_ +====================== + +Wanting to avoid the ravages of asynchronous programming we choose to base +our real-time web development framework Planet on gevent and Python. We’ve +found gevent to be stable, efficient, highly functional and still simplistic +enough for our needs and our customer’s requirements. + +-- Jonas Tärnström, Product Manager, `ESN Social Software`_ + +.. _ESN Social Software: http://esn.me + + +`Blue Shell Games`_ +=================== + +At Blue Shell Games we use gevent to power the application servers that +connect more than a million daily players of our social casino games. +Recognizing that our game code is largely I/O bound — whether waiting on +a database, social networking data providers, or the clients themselves — we chose +gevent as our asynchronous networking framework. Not only does gevent offer +the best performance of any of the Python async networking packages, its +threading model makes multithreaded application servers far easier to write +than traditional kernel threading-based approaches. As our applications add +more real-time multiplayer features, gevent is ready to handle these kinds +of problems with ease. + +-- David Young, CTO, Co-Founder, `Blue Shell Games`_ + +.. _Blue Shell Games: http://www.blueshellgames.com/ + + +TellApart_ +========== + +At TellApart, we have been using gevent since 2010 as the underpinnings of +our frontend servers. It enables us to serve millions of requests every hour +through only a handful of servers, while achieving the strict latency +constraints of Real-Time Bidding ad exchanges. Since then, we've expanded +our use of gevent throughout our stack. Combined with tools such as closures +and generators, gevent makes complicated queuing, distribution, and +streaming workloads dramatically easier to implement. Our open-source event +aggregation service, Taba, couldn't have been built without it. + +See also: `Gevent at TellApart`_ + +-- Kevin Ballard, Software Engineer, TellApart_ + +.. _TellApart: http://tellapart.com +.. _Gevent at TellApart: http://tellapart.com/gevent-at-tellapart + + +Disqus +====== + +See: `Making Disqus Realtime`_ + +.. _`Making Disqus Realtime`: https://ep2012.europython.eu/conference/talks/making-disqus-realtime + + +Pinterest +========= + +Pinterest is one of the biggest players of gevents. We started using gevent in +2011 to query our mysql shards concurrently. It served us well so far. We run +all our WSGI containers using gevent. We are in the process of making all our +service calls gevented. We use a gevented based thrift server which proved to +be way more efficient than the normal python version. I think there is a cost +upfront to make your code greenlet safe but we saw pretty huge win later. +If you are looking to scale out on python gevent is your best friend. + +-- Yash Nelapati, Engineer, Pinterest_ + +.. _Pinterest: http://pinterest.com/ + +TBA: Spotify, Twilio diff --git a/doc/whatsnew_1_0.rst b/doc/whatsnew_1_0.rst new file mode 100644 index 0000000..729acea --- /dev/null +++ b/doc/whatsnew_1_0.rst @@ -0,0 +1,137 @@ +========================== + What's new in gevent 1.0 +========================== + +.. toctree:: + :maxdepth: 2 + + changelog_1_0 + + +The detailed information is available in :doc:`changelog_1_0`. Below is the +summary of all changes since 0.13.8. + +Gevent 1.0 supports Python 2.5 - 2.7. The version of greenlet required is 0.3.2. The source distribution +now includes the dependencies (libev and c-ares) and has no dependencies other than greenlet. + + +New core +======== + +Now the event loop is using libev instead of libevent (see http://blog.gevent.org/2011/04/28/libev-and-libevent/ for motivation). + +The new :mod:`gevent.core` has been rewritten to wrap libev's API. (On Windows, the :mod:`gevent.core` accepts Windows handles +rather than stdio file descriptors.). + +The signal handlers set with the standard signal module are no longer blocked by the event loop. + +The event loops are now pluggable. The GEVENT_LOOP environment variable can specify the alternative class to use (the default is ``gevent.core.loop``). + +The error handling is now done by Hub.handle_error(). + +The system errors that usually kill the process (SystemError, SystemExit, KeyboardInterrupt) are now re-raised in the main greenlet. +Thus ``sys.exit()`` when run inside a greenlet is no longer trapped and kills the process as expected. + + +New dns resolver +================ + +Two new DNS resolvers: threadpool-based one (enabled by default) and c-ares based one. That threadpool-based resolver was added mostly for Windows and Mac OS X platforms where c-ares might behave differently w.r.t system configuration. On Linux, however, the c-ares based resolver is probably a better choice. To enable c-ares resolver set GEVENT_RESOLVER=ares environment variable. + +This fixes some major issues with DNS on 0.13.x, namely: + +- Issue #2: DNS resolver no longer breaks after ``fork()``. You still need to call :func:`gevent.fork` (``os.fork`` is monkey patched with it if ``monkey.patch_all()`` was called). +- DNS resolver no longer ignores ``/etc/resolv.conf`` and ``/etc/hosts``. + +The following functions were added to socket module: + +- gethostbyname_ex +- getnameinfo +- gethostbyaddr +- getfqdn + +It is possible to implement your own DNS resolver and make gevent use it. The GEVENT_RESOLVER variable can point to alternative implementation using the format: ``package.module.class``. The default is ``gevent.resolver_thread.Resolver``. The alternative "ares" resolver is an alias for ``gevent.resolver_ares.Resolver``. + + +New API +======= + +- :func:`gevent.wait` and :func:`gevent.iwait` +- UDP server: gevent.server.DatagramServer +- Subprocess support + + New :mod:`gevent.subprocess` implements the interface of the standard subprocess module in a cooperative way. + It is possible to monkey patch the standard subprocess module with ``patch_all(subprocess=True)`` (not done by default). + +- Thread pool + + **Warning:** this feature is experimental and should be used with care. + + The :mod:`gevent.threadpool` module provides the usual pool methods (apply, map, imap, etc) but runs passed functions + in a real OS thread. + + There's a default threadpool, available as ``gevent.get_hub().threadpool``. + + +Breaking changes +================ + +Removed features +---------------- + +- gevent.dns module (wrapper around libevent-dns) +- gevent.http module (wrapper around libevent-http) +- ``util.lazy_property`` property. +- deprecated gevent.sslold module +- deprecated gevent.rawgreenlet module +- deprecated name ``GreenletSet`` which used to be alias for :class:`Group`. +- link to greenlet feature of Greenlet +- undocumented bind_and_listen and tcp_listener + +Renamed gevent.coros to gevent.lock. The gevent.coros is still available but deprecated. + + +API changes +----------- + +In all servers, method "kill" was renamed to "close". The old name is available as deprecated alias. + +- ``Queue(0)`` is now equivalent to an unbound queue and raises :exc:`DeprecationError`. Use :class:`gevent.queue.Channel` if you need a channel. + +The :class:`gevent.Greenlet` objects: + +- Added ``__nonzero__`` implementation that returns `True` after greenlet was started until it's dead. This overrides + greenlet's __nonzero__ which returned `False` after `start()` until it was first switched to. + + +Bugfixes +======== + +- Issue #302: "python -m gevent.monkey" now sets __file__ properly. +- Issue #143: greenlet links are now executed in the order they were added +- Fixed monkey.patch_thread() to patch threading._DummyThread to avoid leak in threading._active. +- gevent.thread: allocate_lock is now an alias for LockType/Semaphore. That way it does not fail when being used as class member. +- It is now possible to add raw greenlets to the pool. +- The :meth:`map` and :meth:`imap` methods now start yielding the results as soon as possible. +- The :meth:`imap_unordered` no longer swallows an exception raised while iterating its argument. +- `gevent.sleep()` no longer raises an exception, instead it does `sleep(0)`. +- The :class:`WSGIServer` now sets `max_accept` to 1 if `wsgi.multiprocessing` is set to `True`. +- Added :func:`monkey.patch_module` function that monkey patches module using `__implements__` list provided by gevent module. + All of gevent modules that replace stdlib module now have `__implements__` attribute. + + +pywsgi: + +- Fix logging when bound on unix socket (#295). +- readout request data to prevent ECONNRESET +- Fix #79: Properly handle HTTP versions. +- Fix #86: bytearray is now supported. +- Fix #92: raise IOError on truncated POST requests. +- Fix #93: do not sent multiple "100 continue" responses +- Fix #116: Multiline HTTP headers are now handled properly. +- Fix #216: propagate errors raised by Pool.map/imap +- Fix #303: 'requestline' AttributeError in pywsgi. +- Raise an AssertionError if non-zero content-length is passed to start_response(204/304) or if non-empty body is attempted to be written for 304/204 response +- Made sure format_request() does not fail if 'status' attribute is not set yet +- Added REMOTE_PORT variable to the environment. +- Removed unused deprecated 'wfile' property from WSGIHandler diff --git a/doc/whatsnew_1_1.rst b/doc/whatsnew_1_1.rst new file mode 100644 index 0000000..db76d18 --- /dev/null +++ b/doc/whatsnew_1_1.rst @@ -0,0 +1,326 @@ +========================== + What's new in gevent 1.1 +========================== + +.. toctree:: + :maxdepth: 2 + + changelog_1_1 + + +Detailed information an what has changed is available in +:doc:`changelog_1_1`. This document summarizes the most important changes +since :doc:`gevent 1.0.2 `. + +Broader Platform Support +======================== + +gevent 1.1 supports Python 2.6, 2.7, 3.3, and 3.4 on the CPython +(`python.org`_) interpreter. It also supports `PyPy`_ 2.6.1 and above +(PyPy 4.0.1 or higher is recommended); PyPy3 is not supported. + +Support for Python 2.5 was removed when support for Python 3 was +added. Any further releases in the 1.0.x line will maintain support +for Python 2.5. + +.. note:: Version 1.1.x will be the last series of gevent releases + to support Python 2.6. The next major release will only + support Python 2.7 and above. + +Python 3.5 has preliminary support, which means that gevent is +expected to generally run and function with the same level of support +as on Python 3.4, but new features and APIs introduced in 3.5 may not +be properly supported (e.g., `DevpollSelector`_) and due to the recent +arrival of Python 3.5, the level of testing it has received is lower. + +For ease of installation on Windows and OS X, gevent 1.1 is +distributed as pre-compiled binary wheels, in addition to source code. + +.. _python.org: http://www.python.org/downloads/ +.. _PyPy: http://pypy.org +.. _DevpollSelector: https://docs.python.org/3.5/whatsnew/3.5.html#selectors + +PyPy Notes +---------- + +PyPy has been tested on OS X and 64-bit Linux from version 2.6.1 +through 4.0.0 and 4.0.1, and on 32-bit ARM on Raspbian with version 4.0.1. + +.. note:: PyPy is not supported on Windows. (gevent's CFFI backend is not + available on Windows.) + +- Version 4.0.1 or above is **highly recommended** due to its extensive + bug fixes relative to earlier versions. +- Version 2.6.1 or above is **required** for proper signal handling. Prior + to 2.6.1 and its inclusion of `cffi 1.3.0`_, signals could be + delivered incorrectly or fail to be delivered during a blocking + operation. (PyPy 2.5.0 includes CFFI 0.8.6 while 2.6.0 has 1.1.0; + the necessary feature was added in `1.2.0`_ which is not itself + directly present in any PyPy release.) CFFI 1.3.0 also allows using + the CFFI backend on CPython. +- Overall performance seems to be quite acceptable with newer versions + of PyPy. The benchmarks distributed with gevent typically perform as + well or better on PyPy than on CPython at least on some platforms. + Things that are known or expected to be (relatively) slower under + PyPy include the :mod:`c-ares resolver ` and + :class:`~gevent.lock.Semaphore`. Whether or not these matter will + depend on the workload of each application (:pr:`708` mentions + some specific benchmarks for ``Semaphore``). + +.. caution:: The ``c-ares`` resolver is considered highly experimental + under PyPy and is not recommended for production use. + Released versions of PyPy through at least 4.0.1 have `a + bug`_ that can cause a memory leak when subclassing + objects that are implemented in Cython, as is the c-ares + resolver. In addition, thanks to reports like + :issue:`704`, we know that the PyPy garbage collector can + interact badly with Cython-compiled code, leading to + crashes. While the intended use of the ares resolver has + been loosely audited for these issues, no guarantees are made. + +.. note:: PyPy 4.0.x on Linux is known to *rarely* (once per 24 hours) + encounter crashes when running heavily loaded, heavily + networked gevent programs (even without ``c-ares``). The + exact cause is unknown and is being tracked in :issue:`677`. + +.. _cffi 1.3.0: https://bitbucket.org/cffi/cffi/src/ad3140a30a7b0ca912185ef500546a9fb5525ece/doc/source/whatsnew.rst?at=default +.. _1.2.0: https://cffi.readthedocs.io/en/latest/whatsnew.html#v1-2-0 +.. _a bug: https://bitbucket.org/pypy/pypy/issues/2149/memory-leak-for-python-subclass-of-cpyext + +.. _operating_systems_label: + +Operating Systems +----------------- + +gevent is regularly built and tested on Mac OS X, Ubuntu Linux, and +Windows, in both 32- and 64-bit configurations. All three platforms +are primarily tested on the x86/amd64 architecture, while Linux is +also occasionally tested on Raspian on ARM. + +In general, gevent should work on any platform that both Python and +`libev support`_. However, some less commonly used platforms may +require tweaks to the gevent source code or user environment to +compile (e.g., `SmartOS`_). Also, due to differences in +things such as timing, some platforms may not be able to fully pass gevent's +extensive test suite (e.g., `OpenBSD`_). + +.. _libev support: http://pod.tst.eu/http://cvs.schmorp.de/libev/ev.pod#PORTABILITY_NOTES +.. _SmartOS: https://github.com/gevent/gevent/pull/711 +.. _OpenBSD: https://github.com/gevent/gevent/issues/737 + +Bug Fixes +========= + +Since 1.0.2, gevent 1.1 contains over 600 commits from nearly two +dozen contributors. Over 200 issues were closed, and over 50 pull +requests were merged. + +Improved subprocess support +=========================== + +In gevent 1.0, support and monkey patching for the :mod:`subprocess` +module was added. Monkey patching this module was off by default. + +In 1.1, monkey patching ``subprocess`` is on by default due to +improvements in handling child processes and requirements by +downstream libraries, notably `gunicorn`_. + +- :func:`gevent.os.fork`, which is monkey patched by default (and + should be used to fork a gevent-aware process that expects to use + gevent in the child process) has been improved and cooperates with + :func:`gevent.os.waitpid` (again monkey patched by default) and + :func:`gevent.signal.signal` (which is monkey patched only for the + :data:`signal.SIGCHLD` case). The latter two patches are new in 1.1. +- In gevent 1.0, use of libev child watchers (which are used + internally by ``gevent.subprocess``) had race conditions with + user-provided ``SIGCHLD`` handlers, causing many types of + unpredictable breakage. The two new APIs described above are + intended to rectify this. +- Fork-watchers will be called, even in multi-threaded programs + (except on Windows). +- The default threadpool and threaded resolver work in child + processes. +- File descriptors are no longer leaked if + :class:`gevent.subprocess.Popen` fails to start the child. + +In addition, simple use of :class:`multiprocessing.Process` is now +possible in a monkey patched system, at least on POSIX platforms. + +.. caution:: Use of :class:`multiprocessing.Queue` when :mod:`thread` + has been monkey-patched will lead to a hang due to + ``Queue``'s internal use of a blocking pipe and threads. For the same + reason, :class:`concurrent.futures.ProcessPoolExecutor`, + which internally uses a ``Queue``, will hang. + +.. caution:: It is not possible to use :mod:`gevent.subprocess` from + native threads. See :mod:`gevent.subprocess` for details. + +.. note:: If the ``SIGCHLD`` signal is to be handled, it is important + to monkey patch (or directly use) both :mod:`os` and + :mod:`signal`; this is the default for + :func:`~gevent.monkey.patch_all`. Failure to do so can + result in the ``SIGCHLD`` signal being lost. + +.. tip:: All of the above entail forking a child process. Forking + a child process that uses gevent, greenlets, and libev + can have some unexpected consequences if the child + doesn't immediately ``exec`` a new binary. Be sure you + understand these consequences before using this + functionality, especially late in a program's lifecycle. + For a more robust solution to certain uses of child + process, consider `gipc`_. + +.. _gunicorn: http://gunicorn.org +.. _gipc: https://gehrcke.de/gipc/ + +Monkey patching +=============== + +Monkey patching is more robust, especially if the standard library +:mod:`threading` or :mod:`logging` modules had been imported before +applying the patch. In addition, there are now supported ways to +determine if something has been monkey patched. + +API Additions +============= + +Numerous APIs offer slightly expanded functionality in this version. +Look for "changed in version 1.1" or "added in version 1.1" throughout +the documentation for specifics. Highlights include: + +- A gevent-friendly version of :obj:`select.poll` (on platforms that + implement it). +- :class:`~gevent.fileobject.FileObjectPosix` uses the :mod:`io` + package on both Python 2 and Python 3, increasing its functionality, + correctness, and performance. (Previously, the Python 2 implementation used the + undocumented class :class:`socket._fileobject`.) +- Locks raise the same error as standard library locks if they are + over-released. Likewise, SSL sockets raise the same errors as their + bundled counterparts if they are read or written after being closed. +- :meth:`ThreadPool.apply ` can + now be used recursively. +- The various pool objects (:class:`~gevent.pool.Group`, + :class:`~gevent.pool.Pool`, :class:`~gevent.threadpool.ThreadPool`) + support the same improved APIs: :meth:`imap ` + and :meth:`imap_unordered ` accept + multiple iterables, :meth:`apply ` raises any exception raised by the + target callable, etc. +- Killing a greenlet (with :func:`gevent.kill` or + :meth:`Greenlet.kill `) before it is actually started and + switched to now prevents the greenlet from ever running, instead of + raising an exception when it is later switched to. Attempting to + spawn a greenlet with an invalid target now immediately produces + a useful :exc:`TypeError`, instead of spawning a greenlet that would + (usually) immediately die the first time it was switched to. +- Almost anywhere that gevent raises an exception from one greenlet to + another (e.g., :meth:`Greenlet.get `), + the original traceback is preserved and raised. +- Various logging/debugging outputs have been cleaned up. +- The WSGI server found in :mod:`gevent.pywsgi` is more robust against + errors in either the client or the WSGI application, fixing several + hangs or HTTP protocol violations. It also supports new + functionality such as configurable error handling and logging. +- Documentation has been expanded and clarified. + +.. _library_updates_label: + +Library Updates +=============== + +The two C libraries that are bundled with gevent have been updated. +libev has been updated from 4.19 to 4.20 (`libev release notes`_) and +c-ares has been updated from 1.9.1 to 1.10.0 (`c-ares release notes`_). + +.. caution:: The c-ares ``configure`` script is now *much* stricter + about the contents of compilation environment variables + such as ``$CFLAGS`` and ``$LDFLAGS``. For example, + ``$CFLAGS`` is no longer allowed to contain ``-I`` + directives; instead, these must be placed in + ``$CPPFLAGS``. That's one common cause of an error + like the following when compiling from scratch on a POSIX + platform:: + + Running '(cd "/tmp/easy_install-NT921u/gevent-1.1b2/c-ares" && if [ -e ares_build.h ]; then cp ares_build.h ares_build.h.orig; fi && /bin/sh ./configure CONFIG_COMMANDS= CONFIG_FILES= && cp ares_config.h ares_build.h "$OLDPWD" && mv ares_build.h.orig ares_build.h) > configure-output.txt' in /tmp/easy_install-NT921u/gevent-1.1b2/build/temp.linux-x86_64-2.7/c-ares + configure: error: Can not continue. Fix errors mentioned immediately above this line. + +.. _libev release notes: https://github.com/gevent/gevent/blob/master/libev/Changes#L17 +.. _c-ares release notes: https://raw.githubusercontent.com/bagder/c-ares/cares-1_10_0/RELEASE-NOTES + +Compatibility +============= + +This release is intended to be compatible with 1.0.x with minimal or +no changes to client source code. However, there are a few changes to +be aware of that might affect some applications. Most of these changes +are due to the increased platform support of Python 3 and PyPy and +reduce the cases of undocumented or non-standard behaviour. + +- :class:`gevent.baseserver.BaseServer` deterministically + `closes its sockets `_. + + As soon as a request completes (the request handler returns), + the ``BaseServer`` and its subclasses including + :class:`gevent.server.StreamServer` and + :class:`gevent.pywsgi.WSGIServer` close the client socket. + + In gevent 1.0, the client socket was left to the mercies of the + garbage collector (this was undocumented). In the typical case, the + socket would still be closed as soon as the request handler returned + due to CPython's reference-counting garbage collector. But this + meant that a reference cycle could leave a socket dangling open for + an indeterminate amount of time, and a reference leak would result + in it never being closed. It also meant that Python 3 would produce + ResourceWarnings, and PyPy (which, unlike CPython, `does not use a + reference-counted GC`_) would only close (and flush!) the socket at + an arbitrary time in the future. + + If your application relied on the socket not being closed when the + request handler returned (e.g., you spawned a greenlet that + continued to use the socket) you will need to keep the request + handler from returning (e.g., ``join`` the greenlet). If for some + reason that isn't possible, you may subclass the server to prevent + it from closing the socket, at which point the responsibility for + closing and flushing the socket is now yours; *but* the former + approach is strongly preferred, and subclassing the server for this + reason may not be supported in the future. + +.. _does not use a reference-counted GC: http://doc.pypy.org/en/latest/cpython_differences.html#differences-related-to-garbage-collection-strategies + +- :class:`gevent.pywsgi.WSGIServer` ensures that headers (names and values) and the + status line set by the application can be encoded in the ISO-8859-1 + (Latin-1) charset and are of the *native string type*. + + Under gevent 1.0, non-``bytes`` headers (that is, ``unicode``, since + gevent 1.0 only ran on Python 2, although objects like ``int`` were + also allowed) were encoded according to the current default Python + encoding. In some cases, this could allow non-Latin-1 characters to + be sent in the headers, but this violated the HTTP specification, + and their interpretation by the recipient is unknown. In other + cases, gevent could send malformed partial HTTP responses. Now, a + :exc:`UnicodeError` will be raised proactively. + + Most applications that adhered to the WSGI PEP, :pep:`3333`, will not + need to make any changes. See :issue:`614` for more discussion. + + +- Under Python 2, the previously undocumented ``timeout`` parameter to + :meth:`Popen.wait ` (a gevent extension + ) now throws an exception, just like the documented parameter to the + same stdlib method in Python 3. + +- Under Python 3, several standard library methods added ``timeout`` + parameters. These often default to -1 to mean "no timeout", whereas + gevent uses a default of ``None`` to mean the same thing, + potentially leading to great confusion and bugs in portable code. In + gevent, using a negative value has always been ill-defined and hard + to reason about. Because of those two things, as of this release, + negative ``timeout`` values should be considered deprecated (unless + otherwise documented). The current ill-defined behaviour is + maintained, but future releases may choose to treat it the same as + ``None`` or raise an error. No runtime warnings are issued for this + change for performance reasons. + +- The previously undocumented class + ``gevent.fileobject.SocketAdapter`` has been removed, as have the + internal ``gevent._util`` module and some internal implementation modules + found in early pre-releases of 1.1. diff --git a/doc/whatsnew_1_2.rst b/doc/whatsnew_1_2.rst new file mode 100644 index 0000000..a98528f --- /dev/null +++ b/doc/whatsnew_1_2.rst @@ -0,0 +1,112 @@ +========================== + What's new in gevent 1.2 +========================== + +.. toctree:: + :maxdepth: 2 + + changelog_1_2 + +Detailed information on what has changed is available in +:doc:`changelog_1_2`. This document summarizes the most important changes +since :doc:`gevent 1.1 `. + +In general, gevent 1.2 is a smaller update than gevent 1.1, focusing +on platform support, standard library compatibility, security, bug +fixes and consistency. + +Platform Support +======================== + +gevent 1.2 supports Python 2.7, 3.4, 3.5 and 3.6 on the CPython +(`python.org`_) interpreter. It also supports `PyPy2`_ 4.0.1 and above +(PyPy2 5.4 or higher is recommended) and PyPy3 5.5.0. + + +.. caution:: Support for Python 2.6 was removed. Support for Python 3.3 is only + tested on PyPy3. + +.. note:: PyPy is not supported on Windows. (gevent's CFFI backend is not + available on Windows.) + +Python 3.6 was released recently and is supported at the same level as 3.5. + +For ease of installation on Windows and OS X, gevent 1.2 is +distributed as pre-compiled binary wheels, in addition to source code. + +.. _python.org: http://www.python.org/downloads/ +.. _PyPy2: http://pypy.org + +Bug Fixes +========= + +Since 1.1.2, gevent 1.2 contains over 240 commits from nine different +dozen contributors. About two dozen pull requests were merged. + +Improved subprocess support +=========================== + +In gevent 1.1, subprocess monkey-patching was on by default for the +first time. Over time this led to discovery of a few issues and corner +cases that have been fixed in 1.2. + +- Setting SIGCHLD to SIG_IGN or SIG_DFL after :mod:`gevent.subprocess` + had been used previously could not be reversed, causing + ``Popen.wait`` and other calls to hang. Now, if SIGCHLD has been + ignored, the next time :mod:`gevent.subprocess` is used this will be + detected and corrected automatically. (This potentially leads to + issues with :func:`os.popen` on Python 2, but the signal can always + be reset again. Mixing the low-level process handling calls, + low-level signal management and high-level use of + :mod:`gevent.subprocess` is tricky.) Reported in :issue:`857` by + Chris Utz. +- ``Popen.kill`` and ``send_signal`` no longer attempt to send signals + to processes that are known to be exited. +- The :func:`gevent.os.waitpid` function is cooperative in more + circumstances. Reported in :issue:`878` by Heungsub Lee. + +API Additions +============= + +Numerous APIs offer slightly expanded functionality in this version. +Look for "changed in version 1.2" or "added in version 1.2" throughout +the documentation for specifics. + +Of particular note, several backwards compatible updates to the +subprocess module have been backported from Python 3 to Python 2, +making :mod:`gevent.subprocess` smaller, easier to maintain and in +some cases safer, while letting gevent clients use the updated APIs +even on older versions of Python. + +If ``concurrent.futures`` is available (Python 3, or if the Python 2 +backport has been installed), then the class +:class:`gevent.threadpool.ThreadPoolExecutor` is defined to create an +executor that always uses native threads, even when the system is +monkey-patched. + +Library Updates +=============== + +The two C libraries that are bundled with gevent have been updated. +libev has been updated from 4.20 to 4.23 (`libev release notes`_) and +c-ares has been updated from 1.10.0 to 1.12.0 (`c-ares release notes`_). + + +.. _libev release notes: https://github.com/gevent/gevent/blob/master/deps/libev/Changes +.. _c-ares release notes: https://c-ares.haxx.se/changelog.html + +Compatibility +============= + +This release is intended to be compatible with 1.1.x with no changes +to client source code, so long as only non-deprecated and supported +interfaces were used (as always, internal, non-documented +implementation details may have changed). + +In particular the deprecated ``gevent.coros`` module has been removed +and ``gevent.corecext`` and ``gevent.corecffi`` have also been removed. + +For security, ``gevent.pywsgi`` no longer accepts incoming headers +containing an underscore, and header values passed to +``start_response`` cannot contain a carriage return or newline. See +:issue:`819` and :issue:`775`, respectively. diff --git a/doc/whatsnew_1_3.rst b/doc/whatsnew_1_3.rst new file mode 100644 index 0000000..172db25 --- /dev/null +++ b/doc/whatsnew_1_3.rst @@ -0,0 +1,188 @@ +========================== + What's new in gevent 1.3 +========================== + +.. currentmodule:: gevent + +.. toctree:: + :maxdepth: 2 + + changelog + +Detailed information on what has changed is available in the +:doc:`changelog`. This document summarizes the most important changes +since :doc:`gevent 1.2 `. + +gevent 1.3 is an important update for performance, debugging and +monitoring, and platform support. It introduces an (optional) `libuv +`_ loop implementation and supports PyPy on Windows. +See :doc:`loop_impls` for more. + +Since gevent 1.2.2 there have been about 450 commits from a half-dozen +contributors. Almost 100 pull requests and more than 100 issues have +been closed. + +Platform Support +================ + +gevent 1.3 supports Python 2.7, 3.4, 3.5, 3.6 and 3.7 on the CPython +(`python.org`_) interpreter. It also supports `PyPy2`_ 5.8.0 and above +(PyPy2 5.10 or higher is recommended) and PyPy3 5.10.0. + +.. caution:: + + Python 2.7.8 and below (Python 2.7 without a modern + ``ssl`` module), is no longer tested or supported. The + support code remains in this release and gevent can be + installed on such implementations, but such usage is not + supported. Support for Python 2.7.8 will be removed in the next + major version of gevent. + +.. note:: + + PyPy is now supported on Windows with the libuv loop + implementation. + +Python 3.7 is in the process of release right now and gevent is tested +with 3.7b4, the last scheduled beta for Python 3.7. + +For ease of installation on Windows, OS X and Linux, gevent 1.3 is +distributed as pre-compiled binary wheels, in addition to source code. + +.. note:: + + On Linux, you'll need to install gevent from source if you wish to + use the libuv loop implementation. This is because the `manylinux1 + `_ specification for the + distributed wheels does not support libuv. The CFFI library *must* + be installed at build time. + +.. _python.org: http://www.python.org/downloads/ +.. _PyPy2: http://pypy.org + +Greenlet Attributes +=================== + +:class:`Greenlet` objects have gained some useful new +attributes: + +- :attr:`Greenlet.spawning_greenlet` is the greenlet that created this + greenlet. Since the ``parent`` of a greenlet is almost always gevent's + :class:`hub `, this can be more + useful to understand greenlet relationships. +- :attr:`Greenlet.spawn_tree_locals` is a dictionary of values + maintained through the spawn tree (i.e., all descendents of a + particular greenlet based on ``spawning_greenlet``). This is + convenient to share values between a set of greenlets, for example, + all those involved in processing a request. +- :attr:`Greenlet.spawning_stack` is a :obj:`frame ` -like object that + captures where the greenlet was created and can be passed to :func:`traceback.print_stack`. +- :attr:`Greenlet.minimal_ident` is a small integer unique across all + greenlets. +- :attr:`Greenlet.name` is a string printed in the greenlet's repr by default. + +"Raw" greenlets created with `spawn_raw` default to having the +``spawning_greenlet`` and ``spawn_tree_locals``. + +This extra data is printed by the new +:func:`gevent.util.print_run_info` function. + +Performance +=========== + +gevent 1.3 uses Cython on CPython to compile several performance +critical modules. As a result, overall performance is improved. +Specifically, queues are up to 5 times faster, pools are 10-20% +faster, and the :class:`gevent.local.local` is up to 40 times faster. +See :pr:`1156`, :pr:`1155`, :pr:`1117` and :pr:`1154`. + + +Better Behaved Callbacks +======================== + +In gevent 1.2.2, event loop callbacks (including things like +``sleep(0)``) would be run in sequence until we ran them all, or until +we ran 10,000. Simply counting the number of callbacks could lead to +no IO being serviced for an arbitrary, unbound, amount of time. To +correct this, gevent 1.3 introduces `gevent.getswitchinterval` and +will run callbacks for only (approximately) that amount of time before +checking for IO. (This is similar to the way that Python 2 counted +bytecode instructions between thread switches but Python 3 uses the +more deterministic timer approach.) The hope is that this will result +in "smoother" application behaviour and fewer pitfalls. See +:issue:`1072` for more details. + +Monitoring and Debugging +======================== + +Many of the new greenlet attributes are useful for monitoring and +debugging gevent applications. gevent also now has the (optional) +ability to monitor for greenlets that call blocking functions and +stall the event loop and to periodically check if the application has +exceeded a configured memory limit. See :doc:`monitoring` for more +information. + + +New Pure-Python DNS Resolver +============================ + +The `dnspython `_ library is a +new, pure-Python option for :doc:`/dns`. Benchmarks show it to be +faster than the existing c-ares resolver and it is also more stable on +PyPy. The c-ares resolver may be deprecated and removed in the future. + +API Additions +============= + +Numerous APIs offer slightly expanded functionality in this version. +Look for "changed in version 1.3" or "added in version 1.3" throughout +the documentation for specifics. + +A few changes of note: + +- The low-level watcher objects now have a + :func:`~gevent._interfaces.IWatcher.close` method that *must* be + called to promptly dispose of native (libev or libuv) resources. +- `gevent.monkey.patch_all` defaults to patching ``Event``. +- `gevent.subprocess.Popen` accepts the same keyword arguments in + Python 2 as it does in Python 3. +- `gevent.monkey.patch_all` and the various individual patch + functions, emit events as patching is being done. This can be used + to extend the patching process for new modules. ``patch_all`` also + passes all unknown keyword arguments to these events. See + :pr:`1169`. +- The module :mod:`gevent.events` contains the events that parts of + gevent can emit. It will use :mod:`zope.event` if that is installed. + +Library Updates +=============== + +One of the C libraries that are bundled with gevent have been updated. +c-ares has been updated from 1.13.0 to 1.14.0 (`c-ares release notes`_). + +.. _c-ares release notes: https://c-ares.haxx.se/changelog.html + +Compatibility +============= + +This release is intended to be compatible with 1.2.x with no changes +to client source code, so long as only non-deprecated and supported +interfaces were used (as always, internal, non-documented +implementation details may have changed). Here are some specific +compatibility notes. + +- The :doc:`resolvers ` have been refactored. As a result, + ``gevent.ares``, ``gevent.resolver_ares`` and + ``gevent.resolver_thread`` have been deprecated. Choosing a resolver + by alias (e.g., 'thread') in the ``GEVENT_RESOLVER`` environment + variable continues to work as before. + +- The internal module ``gevent._threading`` was significantly + refactored. As the name indicates this is an internal module not + intended as part of the public API, but such uses have been observed. + +- The module ``gevent.wsgi`` was removed. Use :mod:`gevent.pywsgi` + instead. ``gevent.wsgi`` was nothing but an alias for + :mod:`gevent.pywsgi` since gevent 1.0a1 (2011). + +.. LocalWords: Greenlet diff --git a/examples/concurrent_download.py b/examples/concurrent_download.py new file mode 100755 index 0000000..ecf7373 --- /dev/null +++ b/examples/concurrent_download.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +# Copyright (c) 2009 Denis Bilenko. See LICENSE for details. + +"""Spawn multiple workers and wait for them to complete""" +from __future__ import print_function +import gevent +from gevent import monkey + +# patches stdlib (including socket and ssl modules) to cooperate with other greenlets +monkey.patch_all() + +import requests + +# Note that we're using HTTPS, so +# this demonstrates that SSL works. +urls = [ + 'https://www.google.com/', + 'https://www.apple.com/', + 'https://www.python.org/' +] + + + +def print_head(url): + print('Starting %s' % url) + data = requests.get(url).text + print('%s: %s bytes: %r' % (url, len(data), data[:50])) + +jobs = [gevent.spawn(print_head, _url) for _url in urls] + +gevent.wait(jobs) diff --git a/examples/dns_mass_resolve.py b/examples/dns_mass_resolve.py new file mode 100755 index 0000000..3e8d80b --- /dev/null +++ b/examples/dns_mass_resolve.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +"""Resolve hostnames concurrently, exit after 2 seconds. + +Under the hood, this might use an asynchronous resolver based on +c-ares (the default) or thread-pool-based resolver. + +You can choose between resolvers using GEVENT_RESOLVER environment +variable. To enable threading resolver: + + GEVENT_RESOLVER=thread python dns_mass_resolve.py +""" +from __future__ import print_function +import gevent +from gevent import socket +from gevent.pool import Pool + +N = 1000 +# limit ourselves to max 10 simultaneous outstanding requests +pool = Pool(10) +finished = 0 + + +def job(url): + global finished + try: + try: + ip = socket.gethostbyname(url) + print('%s = %s' % (url, ip)) + except socket.gaierror as ex: + print('%s failed with %s' % (url, ex)) + finally: + finished += 1 + +with gevent.Timeout(2, False): + for x in range(10, 10 + N): + pool.spawn(job, '%s.com' % x) + pool.join() + +print('finished within 2 seconds: %s/%s' % (finished, N)) diff --git a/examples/echoserver.py b/examples/echoserver.py new file mode 100755 index 0000000..b9f2820 --- /dev/null +++ b/examples/echoserver.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +"""Simple server that listens on port 16000 and echos back every input to the client. + +Connect to it with: + telnet 127.0.0.1 16000 + +Terminate the connection by terminating telnet (typically Ctrl-] and then 'quit'). +""" +from __future__ import print_function +from gevent.server import StreamServer + + +# this handler will be run for each incoming connection in a dedicated greenlet +def echo(socket, address): + print('New connection from %s:%s' % address) + socket.sendall(b'Welcome to the echo server! Type quit to exit.\r\n') + # using a makefile because we want to use readline() + rfileobj = socket.makefile(mode='rb') + while True: + line = rfileobj.readline() + if not line: + print("client disconnected") + break + if line.strip().lower() == b'quit': + print("client quit") + break + socket.sendall(line) + print("echoed %r" % line) + rfileobj.close() + +if __name__ == '__main__': + # to make the server use SSL, pass certfile and keyfile arguments to the constructor + server = StreamServer(('127.0.0.1', 16000), echo) + # to start the server asynchronously, use its start() method; + # we use blocking serve_forever() here because we have no other jobs + print('Starting echo server on port 16000') + server.serve_forever() diff --git a/examples/geventsendfile.py b/examples/geventsendfile.py new file mode 100644 index 0000000..4bb7e92 --- /dev/null +++ b/examples/geventsendfile.py @@ -0,0 +1,28 @@ +"""An example how to use sendfile[1] with gevent. + +[1] http://pypi.python.org/pypi/py-sendfile/ +""" +# pylint:disable=import-error +from errno import EAGAIN +from sendfile import sendfile as original_sendfile +from gevent.socket import wait_write + + +def gevent_sendfile(out_fd, in_fd, offset, count): + total_sent = 0 + while total_sent < count: + try: + _offset, sent = original_sendfile(out_fd, in_fd, offset + total_sent, count - total_sent) + #print('%s: sent %s [%d%%]' % (out_fd, sent, 100*total_sent/count)) + total_sent += sent + except OSError as ex: + if ex.args[0] == EAGAIN: + wait_write(out_fd) + else: + raise + return offset + total_sent, total_sent + + +def patch_sendfile(): + import sendfile + sendfile.sendfile = gevent_sendfile diff --git a/examples/portforwarder.py b/examples/portforwarder.py new file mode 100644 index 0000000..0e3b54f --- /dev/null +++ b/examples/portforwarder.py @@ -0,0 +1,112 @@ +"""Port forwarder with graceful exit. + +Run the example as + + python portforwarder.py :8080 gevent.org:80 + +Then direct your browser to http://localhost:8080 or do "telnet localhost 8080". + +When the portforwarder receives TERM or INT signal (type Ctrl-C), +it closes the listening socket and waits for all existing +connections to finish. The existing connections will remain unaffected. +The program will exit once the last connection has been closed. +""" +import socket +import sys +import signal +import gevent +from gevent.server import StreamServer +from gevent.socket import create_connection, gethostbyname + + +class PortForwarder(StreamServer): + + def __init__(self, listener, dest, **kwargs): + StreamServer.__init__(self, listener, **kwargs) + self.dest = dest + + def handle(self, source, address): # pylint:disable=method-hidden + log('%s:%s accepted', *address[:2]) + try: + dest = create_connection(self.dest) + except IOError as ex: + log('%s:%s failed to connect to %s:%s: %s', address[0], address[1], self.dest[0], self.dest[1], ex) + return + forwarders = (gevent.spawn(forward, source, dest, self), + gevent.spawn(forward, dest, source, self)) + # if we return from this method, the stream will be closed out + # from under us, so wait for our children + gevent.joinall(forwarders) + + def close(self): + if self.closed: + sys.exit('Multiple exit signals received - aborting.') + else: + log('Closing listener socket') + StreamServer.close(self) + + +def forward(source, dest, server): + try: + source_address = '%s:%s' % source.getpeername()[:2] + dest_address = '%s:%s' % dest.getpeername()[:2] + except socket.error as e: + # We could be racing signals that close the server + # and hence a socket. + log("Failed to get all peer names: %s", e) + return + + try: + while True: + try: + data = source.recv(1024) + log('%s->%s: %r', source_address, dest_address, data) + if not data: + break + dest.sendall(data) + except KeyboardInterrupt: + # On Windows, a Ctrl-C signal (sent by a program) usually winds + # up here, not in the installed signal handler. + if not server.closed: + server.close() + break + except socket.error: + if not server.closed: + server.close() + break + finally: + source.close() + dest.close() + server = None + + +def parse_address(address): + try: + hostname, port = address.rsplit(':', 1) + port = int(port) + except ValueError: + sys.exit('Expected HOST:PORT: %r' % address) + return gethostbyname(hostname), port + + +def main(): + args = sys.argv[1:] + if len(args) != 2: + sys.exit('Usage: %s source-address destination-address' % __file__) + source = args[0] + dest = parse_address(args[1]) + server = PortForwarder(source, dest) + log('Starting port forwarder %s:%s -> %s:%s', *(server.address[:2] + dest)) + gevent.signal(signal.SIGTERM, server.close) + gevent.signal(signal.SIGINT, server.close) + server.start() + gevent.wait() + + +def log(message, *args): + message = message % args + sys.stderr.write(message + '\n') + + +if __name__ == '__main__': + main() diff --git a/examples/processes.py b/examples/processes.py new file mode 100755 index 0000000..4fc28a0 --- /dev/null +++ b/examples/processes.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +from __future__ import print_function +import gevent +from gevent import subprocess + +import sys + +if sys.platform.startswith("win"): + print("Unable to run on windows") +else: + # run 2 jobs in parallel + p1 = subprocess.Popen(['uname'], stdout=subprocess.PIPE) + p2 = subprocess.Popen(['ls'], stdout=subprocess.PIPE) + + gevent.wait([p1, p2], timeout=2) + + # print the results (if available) + if p1.poll() is not None: + print('uname: %r' % p1.stdout.read()) + else: + print('uname: job is still running') + if p2.poll() is not None: + print('ls: %r' % p2.stdout.read()) + else: + print('ls: job is still running') + + p1.stdout.close() + p2.stdout.close() diff --git a/examples/psycopg2_pool.py b/examples/psycopg2_pool.py new file mode 100644 index 0000000..4929409 --- /dev/null +++ b/examples/psycopg2_pool.py @@ -0,0 +1,164 @@ +from __future__ import print_function +# pylint:disable=import-error,broad-except,bare-except +import sys +import contextlib + +import gevent +from gevent.queue import Queue +from gevent.socket import wait_read, wait_write +from psycopg2 import extensions, OperationalError, connect + + +if sys.version_info[0] >= 3: + integer_types = (int,) +else: + import __builtin__ + integer_types = (int, __builtin__.long) + + +def gevent_wait_callback(conn, timeout=None): + """A wait callback useful to allow gevent to work with Psycopg.""" + while 1: + state = conn.poll() + if state == extensions.POLL_OK: + break + elif state == extensions.POLL_READ: + wait_read(conn.fileno(), timeout=timeout) + elif state == extensions.POLL_WRITE: + wait_write(conn.fileno(), timeout=timeout) + else: + raise OperationalError( + "Bad result from poll: %r" % state) + + +extensions.set_wait_callback(gevent_wait_callback) + + +class AbstractDatabaseConnectionPool(object): + + def __init__(self, maxsize=100): + if not isinstance(maxsize, integer_types): + raise TypeError('Expected integer, got %r' % (maxsize, )) + self.maxsize = maxsize + self.pool = Queue() + self.size = 0 + + def create_connection(self): + raise NotImplementedError() + + def get(self): + pool = self.pool + if self.size >= self.maxsize or pool.qsize(): + return pool.get() + + self.size += 1 + try: + new_item = self.create_connection() + except: + self.size -= 1 + raise + return new_item + + def put(self, item): + self.pool.put(item) + + def closeall(self): + while not self.pool.empty(): + conn = self.pool.get_nowait() + try: + conn.close() + except Exception: + pass + + @contextlib.contextmanager + def connection(self, isolation_level=None): + conn = self.get() + try: + if isolation_level is not None: + if conn.isolation_level == isolation_level: + isolation_level = None + else: + conn.set_isolation_level(isolation_level) + yield conn + except: + if conn.closed: + conn = None + self.closeall() + else: + conn = self._rollback(conn) + raise + else: + if conn.closed: + raise OperationalError("Cannot commit because connection was closed: %r" % (conn, )) + conn.commit() + finally: + if conn is not None and not conn.closed: + if isolation_level is not None: + conn.set_isolation_level(isolation_level) + self.put(conn) + + @contextlib.contextmanager + def cursor(self, *args, **kwargs): + isolation_level = kwargs.pop('isolation_level', None) + with self.connection(isolation_level) as conn: + yield conn.cursor(*args, **kwargs) + + def _rollback(self, conn): + try: + conn.rollback() + except: + gevent.get_hub().handle_error(conn, *sys.exc_info()) + return + return conn + + def execute(self, *args, **kwargs): + with self.cursor(**kwargs) as cursor: + cursor.execute(*args) + return cursor.rowcount + + def fetchone(self, *args, **kwargs): + with self.cursor(**kwargs) as cursor: + cursor.execute(*args) + return cursor.fetchone() + + def fetchall(self, *args, **kwargs): + with self.cursor(**kwargs) as cursor: + cursor.execute(*args) + return cursor.fetchall() + + def fetchiter(self, *args, **kwargs): + with self.cursor(**kwargs) as cursor: + cursor.execute(*args) + while True: + items = cursor.fetchmany() + if not items: + break + for item in items: + yield item + + +class PostgresConnectionPool(AbstractDatabaseConnectionPool): + + def __init__(self, *args, **kwargs): + self.connect = kwargs.pop('connect', connect) + maxsize = kwargs.pop('maxsize', None) + self.args = args + self.kwargs = kwargs + AbstractDatabaseConnectionPool.__init__(self, maxsize) + + def create_connection(self): + return self.connect(*self.args, **self.kwargs) + + +def main(): + import time + pool = PostgresConnectionPool("dbname=postgres", maxsize=3) + start = time.time() + for _ in range(4): + gevent.spawn(pool.execute, 'select pg_sleep(1);') + gevent.wait() + delay = time.time() - start + print('Running "select pg_sleep(1);" 4 times with 3 connections. Should take about 2 seconds: %.2fs' % delay) + +if __name__ == '__main__': + main() diff --git a/examples/server.crt b/examples/server.crt new file mode 100644 index 0000000..1379e1d --- /dev/null +++ b/examples/server.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICYzCCAcwCCQD5jx1Aa0dytjANBgkqhkiG9w0BAQQFADB2MQswCQYDVQQGEwJU +UzENMAsGA1UECBMEVGVzdDENMAsGA1UEBxMEVGVzdDEWMBQGA1UEChMNVGVzdCBF +dmVudGxldDENMAsGA1UECxMEVGVzdDENMAsGA1UEAxMEVGVzdDETMBEGCSqGSIb3 +DQEJARYEVGVzdDAeFw0wODA3MDgyMTExNDJaFw0xMDAyMDgwODE1MTBaMHYxCzAJ +BgNVBAYTAlRTMQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQHEwRUZXN0MRYwFAYDVQQK +Ew1UZXN0IEV2ZW50bGV0MQ0wCwYDVQQLEwRUZXN0MQ0wCwYDVQQDEwRUZXN0MRMw +EQYJKoZIhvcNAQkBFgRUZXN0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDM +WcyeIiHQuEGQxgTIvu0aOW4iRFAyUEi8pLWNCxMEHglF8k6OxFVq7XWZMDnDFVnb +ZjmQh5Tc21Ae6cXzxXln578fROXHEzXo3Is8HUlq3ug1yYOGHjxw++Opjf1uoHwP +EBUKsz/flS7knuscgFM9FO05KSPn2wHnZeIDta4yTwIDAQABMA0GCSqGSIb3DQEB +BAUAA4GBAKM71aP0r26gEEEBzovfXm1IwKav6R9/xiWsJ4pFsUXVotcaIjcVBDG1 +Z7tz688hokb+GNxsTI2gNfqanqUnfP9wZxnKRmfTSOvb5aWHIiaiMXSgjiPlqBcm +6mnSeEbSMM9cw479wWhh1YqY8tf3gYJa+sxznVWLSfVLpsjRMphe +-----END CERTIFICATE----- diff --git a/examples/server.key b/examples/server.key new file mode 100644 index 0000000..24cd8e5 --- /dev/null +++ b/examples/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDMWcyeIiHQuEGQxgTIvu0aOW4iRFAyUEi8pLWNCxMEHglF8k6O +xFVq7XWZMDnDFVnbZjmQh5Tc21Ae6cXzxXln578fROXHEzXo3Is8HUlq3ug1yYOG +Hjxw++Opjf1uoHwPEBUKsz/flS7knuscgFM9FO05KSPn2wHnZeIDta4yTwIDAQAB +AoGBAKWfvq0IIvok7Ncm92ew/0D6/R1+2rT8xwdGQ/Nt31q98WwkqLEjxctlbKPd +J2PLIUomf0955BhhFH4JoSwjiHJQ6uishY7srjQQDX/Dxdi5wZAyxYCIVW/kAA9N +/u2s75hSD3s/rqAwOZ182DwAPIqJc4KQoYzvlKERSMDT1PJhAkEA5SUFsiSzBEMX +FyZ++ZMMs1vHrTu5oTK7WHznh9lk7dvsnp9BoUPqhiu8iJ7Q23zj0u5asz2czu11 +nnczXgU6XwJBAORM5Ib4I7nAsoUWn9wDiTwVQeE+D9P1ac9p7EHm7XXuf8o2irRZ +wYYfpXXsjk496YfyQFcQRMk0tU0gegCP7hECQFWRWqwoajUoPIInnPjjwbVki48U +I4CfqjgkBG3Fb5wnKRgezmpDK1vJD1FRRRsBay4EVhhi5KCdKfPv/V2ZxC8CQQCu +U5SxBytofJ8UhxkcTErvaR/8GYLGi//21GAGVop+YdaMlydE3cCrZODYcgCb+CSp +nS7KDG8p4KiMMz9VzJGxAkEAv85K6Sa3H8g9h7LwopBZ5tFNZUaFWo7lEP7DDMH0 +eckZTb1JVpyT/8zrDtsis4WlV9zVkVHxkIaad503BjqvEQ== +-----END RSA PRIVATE KEY----- diff --git a/examples/threadpool.py b/examples/threadpool.py new file mode 100644 index 0000000..0bdd3de --- /dev/null +++ b/examples/threadpool.py @@ -0,0 +1,13 @@ +from __future__ import print_function +import time +import gevent +from gevent.threadpool import ThreadPool + + +pool = ThreadPool(3) +start = time.time() +for _ in range(4): + pool.spawn(time.sleep, 1) +gevent.wait() +delay = time.time() - start +print('Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs' % delay) diff --git a/examples/udp_client.py b/examples/udp_client.py new file mode 100644 index 0000000..74e5cf3 --- /dev/null +++ b/examples/udp_client.py @@ -0,0 +1,22 @@ +# Copyright (c) 2012 Denis Bilenko. See LICENSE for details. +"""Send a datagram to localhost:9000 and receive a datagram back. + +Usage: python udp_client.py MESSAGE + +Make sure you're running a UDP server on port 9001 (see udp_server.py). + +There's nothing gevent-specific here. +""" +from __future__ import print_function +import sys +from gevent import socket + +address = ('localhost', 9001) +message = ' '.join(sys.argv[1:]) +sock = socket.socket(type=socket.SOCK_DGRAM) +sock.connect(address) +print('Sending %s bytes to %s:%s' % ((len(message), ) + address)) +sock.send(message.encode()) +data, address = sock.recvfrom(8192) +print('%s:%s: got %r' % (address + (data, ))) +sock.close() diff --git a/examples/udp_server.py b/examples/udp_server.py new file mode 100644 index 0000000..1517177 --- /dev/null +++ b/examples/udp_server.py @@ -0,0 +1,21 @@ +# Copyright (c) 2012 Denis Bilenko. See LICENSE for details. +"""A simple UDP server. + +For every message received, it sends a reply back. + +You can use udp_client.py to send a message. +""" +from __future__ import print_function +from gevent.server import DatagramServer + + +class EchoServer(DatagramServer): + + def handle(self, data, address): # pylint:disable=method-hidden + print('%s: got %r' % (address[0], data)) + self.socket.sendto(('Received %s bytes' % len(data)).encode('utf-8'), address) + + +if __name__ == '__main__': + print('Receiving datagrams on :9000') + EchoServer(':9000').serve_forever() diff --git a/examples/unixsocket_client.py b/examples/unixsocket_client.py new file mode 100644 index 0000000..5a581e6 --- /dev/null +++ b/examples/unixsocket_client.py @@ -0,0 +1,10 @@ +from __future__ import print_function +import socket + +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +s.connect("./unixsocket_server.py.sock") +s.send('GET / HTTP/1.0\r\n\r\n') +data = s.recv(1024) +print('received %s bytes' % len(data)) +print(data) +s.close() diff --git a/examples/unixsocket_server.py b/examples/unixsocket_server.py new file mode 100644 index 0000000..549c4d1 --- /dev/null +++ b/examples/unixsocket_server.py @@ -0,0 +1,19 @@ +import os +from gevent.pywsgi import WSGIServer +from gevent import socket + + +def application(environ, start_response): + assert environ + start_response('200 OK', []) + return [] + + +if __name__ == '__main__': + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sockname = './' + os.path.basename(__file__) + '.sock' + if os.path.exists(sockname): + os.remove(sockname) + listener.bind(sockname) + listener.listen(1) + WSGIServer(listener, application).serve_forever() diff --git a/examples/webchat/README b/examples/webchat/README new file mode 100644 index 0000000..3a71571 --- /dev/null +++ b/examples/webchat/README @@ -0,0 +1,4 @@ +An example of AJAX chat taken from Tornado demos and converted to use django and gevent. + +To start the server, run +$ python run.py diff --git a/examples/webchat/__init__.py b/examples/webchat/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/webchat/application.py b/examples/webchat/application.py new file mode 100755 index 0000000..328e4cd --- /dev/null +++ b/examples/webchat/application.py @@ -0,0 +1,20 @@ +#!/usr/bin/python +from gevent import monkey; monkey.patch_all() +import os +import traceback +from django.core.handlers.wsgi import WSGIHandler +from django.core.signals import got_request_exception +from django.core.management import call_command + +os.environ['DJANGO_SETTINGS_MODULE'] = 'webchat.settings' + + +def exception_printer(sender, **kwargs): + traceback.print_exc() + + +got_request_exception.connect(exception_printer) + +call_command('syncdb') + +application = WSGIHandler() diff --git a/examples/webchat/chat/__init__.py b/examples/webchat/chat/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/webchat/chat/views.py b/examples/webchat/chat/views.py new file mode 100644 index 0000000..7819865 --- /dev/null +++ b/examples/webchat/chat/views.py @@ -0,0 +1,65 @@ +import uuid +import simplejson +from django.shortcuts import render_to_response +from django.template.loader import render_to_string +from django.http import HttpResponse +from gevent.event import Event +from webchat import settings + + +class ChatRoom(object): + cache_size = 200 + + def __init__(self): + self.cache = [] + self.new_message_event = Event() + + def main(self, request): + if self.cache: + request.session['cursor'] = self.cache[-1]['id'] + return render_to_response('index.html', {'MEDIA_URL': settings.MEDIA_URL, 'messages': self.cache}) + + def message_new(self, request): + name = request.META.get('REMOTE_ADDR') or 'Anonymous' + forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + if forwarded_for and name == '127.0.0.1': + name = forwarded_for + msg = create_message(name, request.POST['body']) + self.cache.append(msg) + if len(self.cache) > self.cache_size: + self.cache = self.cache[-self.cache_size:] + self.new_message_event.set() + self.new_message_event.clear() + return json_response(msg) + + def message_updates(self, request): + cursor = request.session.get('cursor') + if not self.cache or cursor == self.cache[-1]['id']: + self.new_message_event.wait() + assert cursor != self.cache[-1]['id'], cursor + try: + for index, m in enumerate(self.cache): + if m['id'] == cursor: + return json_response({'messages': self.cache[index + 1:]}) + return json_response({'messages': self.cache}) + finally: + if self.cache: + request.session['cursor'] = self.cache[-1]['id'] + else: + request.session.pop('cursor', None) + +room = ChatRoom() +main = room.main +message_new = room.message_new +message_updates = room.message_updates + + +def create_message(from_, body): + data = {'id': str(uuid.uuid4()), 'from': from_, 'body': body} + data['html'] = render_to_string('message.html', dictionary={'message': data}) + return data + + +def json_response(value, **kwargs): + kwargs.setdefault('content_type', 'text/javascript; charset=UTF-8') + return HttpResponse(simplejson.dumps(value), **kwargs) diff --git a/examples/webchat/manage.py b/examples/webchat/manage.py new file mode 100755 index 0000000..bcf991f --- /dev/null +++ b/examples/webchat/manage.py @@ -0,0 +1,14 @@ +#!/usr/bin/python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("""Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things. +You'll have to run django-admin.py, passing it your settings module. +(If the file settings.py does indeed exist, it's causing an ImportError somehow.) +""" % __file__) + raise + +if __name__ == "__main__": + execute_manager(settings) diff --git a/examples/webchat/run_standalone.py b/examples/webchat/run_standalone.py new file mode 100755 index 0000000..ea73fc9 --- /dev/null +++ b/examples/webchat/run_standalone.py @@ -0,0 +1,6 @@ +#!/usr/bin/python +from __future__ import print_function +from gevent.wsgi import WSGIServer +from application import application +print('Serving on 8000...') +WSGIServer(('', 8000), application).serve_forever() diff --git a/examples/webchat/run_uwsgi b/examples/webchat/run_uwsgi new file mode 100755 index 0000000..39a3e1e --- /dev/null +++ b/examples/webchat/run_uwsgi @@ -0,0 +1,3 @@ +#!/bin/sh +# see http://projects.unbit.it/uwsgi and http://projects.unbit.it/uwsgi/wiki/Gevent +exec uwsgi --loop gevent --http-socket :8000 --module application --async 1000 diff --git a/examples/webchat/settings.py b/examples/webchat/settings.py new file mode 100644 index 0000000..478e54b --- /dev/null +++ b/examples/webchat/settings.py @@ -0,0 +1,38 @@ +from os.path import dirname, join, abspath +__dir__ = dirname(abspath(__file__)) + +DEBUG = True +TEMPLATE_DEBUG = DEBUG +ADMINS = () +MANAGERS = ADMINS +DATABASE_ENGINE = 'sqlite3' +DATABASE_NAME = '/tmp/gevent-webchat.sqlite' +DATABASE_USER = '' +DATABASE_PASSWORD = '' +DATABASE_HOST = '' +DATABASE_PORT = '' +TIME_ZONE = 'America/Chicago' +LANGUAGE_CODE = 'en-us' +SITE_ID = 1 +USE_I18N = True +MEDIA_ROOT = join(__dir__, 'static') +MEDIA_URL = '/media/' +SECRET_KEY = 'nv8(yg*&1-lon-8i-3jcs0y!01+rem*54051^5xt#^tzujdj!c' +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +) +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', +) +ROOT_URLCONF = 'webchat.urls' +TEMPLATE_DIRS = ( + join(__dir__, 'templates') +) +INSTALLED_APPS = ( + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'webchat.chat', +) diff --git a/examples/webchat/static/chat.css b/examples/webchat/static/chat.css new file mode 100644 index 0000000..3f1be77 --- /dev/null +++ b/examples/webchat/static/chat.css @@ -0,0 +1,58 @@ +/* + * Copyright 2009 FriendFeed + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +body { + background: white; + margin: 10px; +} + +body, +input { + font-family: sans-serif; + font-size: 10pt; + color: black; +} + +table { + border-collapse: collapse; + border: 0; +} + +td { + border: 0; + padding: 0; +} + +#body { + position: absolute; + bottom: 10px; + left: 10px; + right: 100px; +} + +#input { + margin-top: 0.5em; +} + +#inbox .message { + padding-top: 0.25em; +} + +#nav { + text-align: right; + float: right; + z-index: 99; +} diff --git a/examples/webchat/static/chat.js b/examples/webchat/static/chat.js new file mode 100644 index 0000000..f23a9dd --- /dev/null +++ b/examples/webchat/static/chat.js @@ -0,0 +1,135 @@ +// Copyright 2009 FriendFeed +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +$(document).ready(function() { + if (!window.console) window.console = {}; + if (!window.console.log) window.console.log = function() {}; + + $("#messageform").live("submit", function() { + newMessage($(this)); + return false; + }); + $("#messageform").live("keypress", function(e) { + if (e.keyCode == 13) { + newMessage($(this)); + return false; + } + }); + $("#message").select(); + updater.poll(); +}); + +function newMessage(form) { + var message = form.formToDict(); + var disabled = form.find("input[type=submit]"); + disabled.disable(); + $.postJSON("/a/message/new", message, function(response) { + updater.showMessage(response); + if (message.id) { + form.parent().remove(); + } else { + form.find("input[type=text]").val("").select(); + disabled.enable(); + } + }); +} + +function getCookie(name) { + var r = document.cookie.match("\\b" + name + "=([^;]*)\\b"); + return r ? r[1] : undefined; +} + +jQuery.postJSON = function(url, args, callback) { + args._xsrf = getCookie("_xsrf"); + $.ajax({url: url, data: $.param(args), dataType: "text", type: "POST", + success: function(response) { + if (callback) callback(eval("(" + response + ")")); + }, error: function(response) { + console.log("ERROR:", response) + }}); +}; + +jQuery.fn.formToDict = function() { + var fields = this.serializeArray(); + var json = {} + for (var i = 0; i < fields.length; i++) { + json[fields[i].name] = fields[i].value; + } + if (json.next) delete json.next; + return json; +}; + +jQuery.fn.disable = function() { + this.enable(false); + return this; +}; + +jQuery.fn.enable = function(opt_enable) { + if (arguments.length && !opt_enable) { + this.attr("disabled", "disabled"); + } else { + this.removeAttr("disabled"); + } + return this; +}; + +var updater = { + errorSleepTime: 500, + cursor: null, + + poll: function() { + var args = {"_xsrf": getCookie("_xsrf")}; + if (updater.cursor) args.cursor = updater.cursor; + $.ajax({url: "/a/message/updates", type: "POST", dataType: "text", + data: $.param(args), success: updater.onSuccess, + error: updater.onError}); + }, + + onSuccess: function(response) { + try { + updater.newMessages(eval("(" + response + ")")); + } catch (e) { + updater.onError(); + return; + } + updater.errorSleepTime = 500; + window.setTimeout(updater.poll, 0); + }, + + onError: function(response) { + updater.errorSleepTime *= 2; + console.log("Poll error; sleeping for", updater.errorSleepTime, "ms"); + window.setTimeout(updater.poll, updater.errorSleepTime); + }, + + newMessages: function(response) { + if (!response.messages) return; + updater.cursor = response.cursor; + var messages = response.messages; + updater.cursor = messages[messages.length - 1].id; + console.log(messages.length, "new messages, cursor:", updater.cursor); + for (var i = 0; i < messages.length; i++) { + updater.showMessage(messages[i]); + } + }, + + showMessage: function(message) { + var existing = $("#m" + message.id); + if (existing.length > 0) return; + var node = $(message.html); + node.hide(); + $("#inbox").append(node); + node.slideDown(); + } +}; diff --git a/examples/webchat/templates/404.html b/examples/webchat/templates/404.html new file mode 100644 index 0000000..03332b4 --- /dev/null +++ b/examples/webchat/templates/404.html @@ -0,0 +1 @@ +

Not Found

diff --git a/examples/webchat/templates/500.html b/examples/webchat/templates/500.html new file mode 100644 index 0000000..27a5459 --- /dev/null +++ b/examples/webchat/templates/500.html @@ -0,0 +1 @@ +

Internal Server Error

diff --git a/examples/webchat/templates/index.html b/examples/webchat/templates/index.html new file mode 100644 index 0000000..8e32e22 --- /dev/null +++ b/examples/webchat/templates/index.html @@ -0,0 +1,36 @@ + + + + + Chat Demo + + + + +
+
+ {% for message in messages %} + {% include "message.html" %} + {% endfor %} +
+
+
+ + + + + +
+ + +
+
+
+
+ + + + diff --git a/examples/webchat/templates/message.html b/examples/webchat/templates/message.html new file mode 100644 index 0000000..64770f0 --- /dev/null +++ b/examples/webchat/templates/message.html @@ -0,0 +1 @@ +
{{ message.from }}: {{ message.body }}
diff --git a/examples/webchat/urls.py b/examples/webchat/urls.py new file mode 100644 index 0000000..00f0fe2 --- /dev/null +++ b/examples/webchat/urls.py @@ -0,0 +1,12 @@ +from django.conf.urls.defaults import * +from webchat import settings + +urlpatterns = patterns('webchat.chat.views', + ('^$', 'main'), + ('^a/message/new$', 'message_new'), + ('^a/message/updates$', 'message_updates')) + +urlpatterns += patterns('django.views.static', + (r'^%s(?P.*)$' % settings.MEDIA_URL.lstrip('/'), 'serve', + {'document_root': settings.MEDIA_ROOT, + 'show_indexes': True})) diff --git a/examples/webproxy.py b/examples/webproxy.py new file mode 100755 index 0000000..e381d6b --- /dev/null +++ b/examples/webproxy.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +"""A web application that retrieves other websites for you. + +To start serving the application on port 8088, type + + python webproxy.py + +To start the server on some other interface/port, use + + python -m gevent.wsgi -p 8000 -i 0.0.0.0 webproxy.py + +""" +from __future__ import print_function +from gevent import monkey; monkey.patch_all() +import sys +import re +import traceback +from cgi import escape + +try: + import urllib2 + from urlparse import urlparse + from urllib import unquote +except ImportError: + # pylint:disable=import-error,no-name-in-module + from urllib import request as urllib2 + from urllib.parse import urlparse + from urllib.parse import unquote + +LISTEN = ('127.0.0.1', 8088) + + +def _as_bytes(s): + if not isinstance(s, bytes): # Py3 + s = s.encode('utf-8') + return s + + +def _as_str(s): + if not isinstance(s, str): # Py3 + s = s.decode('latin-1') + return s + + +def application(env, start_response): + proxy_url = 'http://%s/' % env['HTTP_HOST'] + method = env['REQUEST_METHOD'] + path = env['PATH_INFO'] + if env['QUERY_STRING']: + path += '?' + env['QUERY_STRING'] + path = path.lstrip('/') + if (method, path) == ('GET', ''): + start_response('200 OK', [('Content-Type', 'text/html')]) + return [FORM] + elif method == 'GET': + return proxy(path, start_response, proxy_url) + elif (method, path) == ('POST', ''): + key, value = env['wsgi.input'].read().strip().split(b'=') + assert key == b'url', repr(key) + value = _as_str(value) + start_response('302 Found', [('Location', _as_str(join(proxy_url, unquote(value))))]) + elif method == 'POST': + start_response('404 Not Found', []) + else: + start_response('501 Not Implemented', []) + return [] + + +def proxy(path, start_response, proxy_url): + # pylint:disable=too-many-locals + if '://' not in path: + path = 'http://' + path + try: + try: + response = urllib2.urlopen(path) + except urllib2.HTTPError as ex: + response = ex + print('%s: %s %s' % (path, response.code, response.msg)) + headers = [(k, v) for (k, v) in response.headers.items() if k not in drop_headers] + scheme, netloc, path, _params, _query, _fragment = urlparse(path) + host = (scheme or 'http') + '://' + netloc + except Exception as ex: # pylint:disable=broad-except + sys.stderr.write('error while reading %s:\n' % path) + traceback.print_exc() + tb = traceback.format_exc() + start_response('502 Bad Gateway', [('Content-Type', 'text/html')]) + # pylint:disable=deprecated-method + error_str = escape(str(ex) or ex.__class__.__name__ or 'Error') + error_str = '

%s

%s

%s
' % (error_str, escape(path), escape(tb)) + return [_as_bytes(error_str)] + else: + start_response('%s %s' % (response.code, response.msg), headers) + data = response.read() + data = fix_links(data, proxy_url, host) + return [data] + + +def join(url1, *rest): + if not rest: + return url1 + url2, rest = rest[0], rest[1:] + url1 = _as_bytes(url1) + url2 = _as_bytes(url2) + if url1.endswith(b'/'): + if url2.startswith(b'/'): + return join(url1 + url2[1:], *rest) + return join(url1 + url2, *rest) + elif url2.startswith(b'/'): + return join(url1 + url2, *rest) + + return join(url1 + b'/' + url2, *rest) + + +def fix_links(data, proxy_url, host_url): + """ + >>> fix_links("> %r' % (m.group(0), result)) + return result + data = _link_re_1.sub(fix_link_cb, data) + data = _link_re_2.sub(fix_link_cb, data) + return data + +_link_re_1 = re.compile(br'''(?P(href|src|action)\s*=\s*)(?P['"])(?P[^#].*?)(?P=quote)''') +_link_re_2 = re.compile(br'''(?P(href|src|action)\s*=\s*)(?P[^'"#>][^ >]*)''') + +drop_headers = ['transfer-encoding', 'set-cookie'] + +FORM = b""" +Web Proxy - gevent example + + + +
Type in URL you want to visit and press Enter
+
+ +
+
+""" + +if __name__ == '__main__': + from gevent.pywsgi import WSGIServer + print('Serving on %s...' % (LISTEN,)) + WSGIServer(LISTEN, application).serve_forever() diff --git a/examples/webpy.py b/examples/webpy.py new file mode 100755 index 0000000..bf1564f --- /dev/null +++ b/examples/webpy.py @@ -0,0 +1,33 @@ +#!/usr/bin/python +"""A web.py application powered by gevent""" + +from __future__ import print_function +from gevent import monkey; monkey.patch_all() +from gevent.pywsgi import WSGIServer +import time +import web # pylint:disable=import-error + +urls = ("/", "index", + '/long', 'long_polling') + + +class index(object): + def GET(self): + return 'Hello, world!
/long' + + +class long_polling(object): + # Since gevent's WSGIServer executes each incoming connection in a separate greenlet + # long running requests such as this one don't block one another; + # and thanks to "monkey.patch_all()" statement at the top, thread-local storage used by web.ctx + # becomes greenlet-local storage thus making requests isolated as they should be. + def GET(self): + print('GET /long') + time.sleep(10) # possible to block the request indefinitely, without harming others + return 'Hello, 10 seconds later' + + +if __name__ == "__main__": + application = web.application(urls, globals()).wsgifunc() + print('Serving on 8088...') + WSGIServer(('', 8088), application).serve_forever() diff --git a/examples/wsgiserver.py b/examples/wsgiserver.py new file mode 100755 index 0000000..b8001a3 --- /dev/null +++ b/examples/wsgiserver.py @@ -0,0 +1,18 @@ +#!/usr/bin/python +"""WSGI server example""" +from __future__ import print_function +from gevent.pywsgi import WSGIServer + + +def application(env, start_response): + if env['PATH_INFO'] == '/': + start_response('200 OK', [('Content-Type', 'text/html')]) + return [b"hello world"] + + start_response('404 Not Found', [('Content-Type', 'text/html')]) + return [b'

Not Found

'] + + +if __name__ == '__main__': + print('Serving on 8088...') + WSGIServer(('127.0.0.1', 8088), application).serve_forever() diff --git a/examples/wsgiserver_ssl.py b/examples/wsgiserver_ssl.py new file mode 100755 index 0000000..42f752c --- /dev/null +++ b/examples/wsgiserver_ssl.py @@ -0,0 +1,20 @@ +#!/usr/bin/python +"""Secure WSGI server example based on gevent.pywsgi""" + +from __future__ import print_function +from gevent import pywsgi + + +def hello_world(env, start_response): + if env['PATH_INFO'] == '/': + start_response('200 OK', [('Content-Type', 'text/html')]) + return [b"hello world"] + + start_response('404 Not Found', [('Content-Type', 'text/html')]) + return [b'

Not Found

'] + +print('Serving on https://:8443') +server = pywsgi.WSGIServer(('127.0.0.1', 8443), hello_world, keyfile='server.key', certfile='server.crt') +# to start the server asynchronously, call server.start() +# we use blocking serve_forever() here because we have no other jobs +server.serve_forever() diff --git a/rtd-requirements.txt b/rtd-requirements.txt new file mode 100644 index 0000000..14679e5 --- /dev/null +++ b/rtd-requirements.txt @@ -0,0 +1,2 @@ +cython >= 0.28.1 +repoze.sphinx.autointerface diff --git a/scripts/gprospector.py b/scripts/gprospector.py new file mode 100644 index 0000000..b7b2165 --- /dev/null +++ b/scripts/gprospector.py @@ -0,0 +1,22 @@ +from __future__ import print_function +import re +import sys + +from prospector.run import main + +def _excepthook(e, t, tb): + while tb is not None: + frame = tb.tb_frame + print(frame.f_code, frame.f_code.co_name) + for n in ('self', 'node', 'elt'): + if n in frame.f_locals: + print(n, frame.f_locals[n]) + print('---') + tb = tb.tb_next + + +sys.excepthook = _excepthook + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..7cf67f1 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# GEVENT: Taken from https://raw.githubusercontent.com/DRMacIver/hypothesis/master/scripts/install.sh + +# Special license: Take literally anything you want out of this file. I don't +# care. Consider it WTFPL licensed if you like. +# Basically there's a lot of suffering encoded here that I don't want you to +# have to go through and you should feel free to use this to avoid some of +# that suffering in advance. + +set -e +set -x + +# This is to guard against multiple builds in parallel. The various installers will tend +# to stomp all over eachother if you do this and they haven't previously successfully +# succeeded. We use a lock file to block progress so only one install runs at a time. +# This script should be pretty fast once files are cached, so the lost of concurrency +# is not a major problem. +# This should be using the lockfile command, but that's not available on the +# containerized travis and we can't install it without sudo. +# Is is unclear if this is actually useful. I was seeing behaviour that suggested +# concurrent runs of the installer, but I can't seem to find any evidence of this lock +# ever not being acquired. + +BASE=${BUILD_RUNTIMES-$PWD/.runtimes} +echo $BASE +mkdir -p $BASE + +LOCKFILE="$BASE/.install-lockfile" +while true; do + if mkdir $LOCKFILE 2>/dev/null; then + echo "Successfully acquired installer." + break + else + echo "Failed to acquire lock. Is another installer running? Waiting a bit." + fi + + sleep $[ ( $RANDOM % 10) + 1 ].$[ ( $RANDOM % 100) ]s + + if (( $(date '+%s') > 300 + $(stat --format=%X $LOCKFILE) )); then + echo "We've waited long enough" + rm -rf $LOCKFILE + fi +done +trap "rm -rf $LOCKFILE" EXIT + + +PYENV=$BASE/pyenv + + +# The file for 3.7b1 shipped with pyenv on Feb 6 2018 +# won't compile on Travis. So we use a forked version that +# compiles openssl for us. We also beat them to the punch for 3.7b2, b3, b4, .0, .1 +# https://github.com/travis-ci/travis-ci/issues/9069 + +if [ ! -d "$PYENV/.git" ]; then + rm -rf $PYENV + git clone https://github.com/gevent/pyenv.git $BASE/pyenv +else + back=$PWD + cd $PYENV + git fetch || echo "Fetch failed to complete. Ignoring" + git reset --hard origin/master + cd $back +fi + + +SNAKEPIT=$BASE/snakepit + +install () { + + VERSION="$1" + ALIAS="$2" + mkdir -p $BASE/versions + SOURCE=$BASE/versions/$ALIAS + OPENSSL_PATH=$SOURCE/openssl/lib + + if [ ! -e "$SOURCE" ]; then + mkdir -p $SNAKEPIT + mkdir -p $BASE/versions + LD_LIBRARY_PATH="$OPENSSL_PATH" $BASE/pyenv/plugins/python-build/bin/python-build $VERSION $SOURCE + fi + rm -f $SNAKEPIT/$ALIAS + mkdir -p $SNAKEPIT + ls -l $SNAKEPIT + ls -l $BASE/versions + ls -l $SOURCE/ + ls -l $SOURCE/bin + ln -s $SOURCE/bin/python $SNAKEPIT/$ALIAS + LD_LIBRARY_PATH="$OPENSSL_PATH" $SOURCE/bin/python -m pip.__main__ install --upgrade pip wheel virtualenv +} + + +for var in "$@"; do + case "${var}" in + 2.7.8) + install 2.7.8 python2.7.8 + ;; + 2.7) + install 2.7.15 python2.7.15 + ;; + 3.4) + install 3.4.8 python3.4.8 + ;; + 3.5) + install 3.5.5 python3.5.5 + ;; + 3.6) + install 3.6.7 python3.6.7 + ;; + 3.7) + install 3.7.1 python3.7.1 + ;; + pypy) + install pypy2.7-6.0.0 pypy600 + ;; + pypy3) + install pypy3.5-6.0.0 pypy3.5_600 + ;; + esac +done diff --git a/scripts/releases/appveyor-download.py b/scripts/releases/appveyor-download.py new file mode 100644 index 0000000..fa5e80d --- /dev/null +++ b/scripts/releases/appveyor-download.py @@ -0,0 +1,108 @@ +""" +Use the AppVeyor API to download Windows artifacts. + +Taken from: https://bitbucket.org/ned/coveragepy/src/tip/ci/download_appveyor.py +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt +""" +import argparse +import os +import zipfile + +import requests + + +def make_auth_headers(): + """Make the authentication headers needed to use the Appveyor API.""" + if not os.path.exists(".appveyor.token"): + raise RuntimeError( + "Please create a file named `.appveyor.token` in the current directory. " + "You can get the token from https://ci.appveyor.com/api-token" + ) + with open(".appveyor.token") as f: + token = f.read().strip() + + headers = { + 'Authorization': 'Bearer {}'.format(token), + } + return headers + + +def make_url(url, **kwargs): + """Build an Appveyor API url.""" + return "https://ci.appveyor.com/api" + url.format(**kwargs) + + +def get_project_build(account_project): + """Get the details of the latest Appveyor build.""" + url = make_url("/projects/{account_project}", account_project=account_project) + response = requests.get(url, headers=make_auth_headers()) + return response.json() + + +def download_latest_artifacts(account_project): + """Download all the artifacts from the latest build.""" + build = get_project_build(account_project) + jobs = build['build']['jobs'] + print("Build {0[build][version]}, {1} jobs: {0[build][message]}".format(build, len(jobs))) + for job in jobs: + name = job['name'].partition(':')[2].split(',')[0].strip() + print(" {0}: {1[status]}, {1[artifactsCount]} artifacts".format(name, job)) + + url = make_url("/buildjobs/{jobid}/artifacts", jobid=job['jobId']) + response = requests.get(url, headers=make_auth_headers()) + artifacts = response.json() + + for artifact in artifacts: + is_zip = artifact['type'] == "Zip" + filename = artifact['fileName'] + print(" {0}, {1} bytes".format(filename, artifact['size'])) + + url = make_url( + "/buildjobs/{jobid}/artifacts/{filename}", + jobid=job['jobId'], + filename=filename + ) + download_url(url, filename, make_auth_headers()) + + if is_zip: + unpack_zipfile(filename) + os.remove(filename) + + +def ensure_dirs(filename): + """Make sure the directories exist for `filename`.""" + dirname, _ = os.path.split(filename) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + + +def download_url(url, filename, headers): + """Download a file from `url` to `filename`.""" + ensure_dirs(filename) + response = requests.get(url, headers=headers, stream=True) + if response.status_code == 200: + with open(filename, 'wb') as f: + for chunk in response.iter_content(16 * 1024): + f.write(chunk) + + +def unpack_zipfile(filename): + """Unpack a zipfile, using the names in the zip.""" + with open(filename, 'rb') as fzip: + z = zipfile.ZipFile(fzip) + for name in z.namelist(): + print(" extracting {}".format(name)) + ensure_dirs(name) + z.extract(name) + +parser = argparse.ArgumentParser(description='Download artifacts from AppVeyor.') +parser.add_argument('name', + metavar='ID', + help='Project ID in AppVeyor. Example: ionelmc/python-nameless') + +if __name__ == "__main__": + # import logging + # logging.basicConfig(level="DEBUG") + args = parser.parse_args() + download_latest_artifacts(args.name) diff --git a/scripts/releases/geventrel.sh b/scripts/releases/geventrel.sh new file mode 100755 index 0000000..39ebd6d --- /dev/null +++ b/scripts/releases/geventrel.sh @@ -0,0 +1,44 @@ +#!/opt/local/bin/bash +# +# Quick hack script to build a single gevent release in a virtual env. Takes one +# argument, the path to python to use. +# Has hardcoded paths, probably only works on my (JAM) machine. + +set -e +export WORKON_HOME=$HOME/Projects/VirtualEnvs +export VIRTUALENVWRAPPER_LOG_DIR=~/.virtualenvs +source `which virtualenvwrapper.sh` + +# Make sure there are no -march flags set +# https://github.com/gevent/gevent/issues/791 +unset CFLAGS +unset CXXFLAGS +unset CPPFLAGS + +# If we're building on 10.12, we have to exclude clock_gettime +# because it's not available on earlier releases and leads to +# segfaults because the symbol clock_gettime is NULL. +# See https://github.com/gevent/gevent/issues/916 +export CPPFLAGS="-D_DARWIN_FEATURE_CLOCK_GETTIME=0" + +BASE=`pwd`/../../ +BASE=`greadlink -f $BASE` + + +cd /tmp/gevent +virtualenv -p $1 `basename $1` +cd `basename $1` +echo "Made tmpenv" +echo `pwd` +source bin/activate +echo cloning $BASE +git clone $BASE gevent +cd ./gevent +pip install -U pip +pip install -U setuptools greenlet cffi +pip install -U wheel +# We may need different versions of deps depending on the +# version of python; that's captured in this file. +pip install -U -r dev-requirements.txt +python ./setup.py sdist bdist_wheel +cp dist/*whl /tmp/gevent/ diff --git a/scripts/releases/geventreleases.sh b/scripts/releases/geventreleases.sh new file mode 100755 index 0000000..b62f645 --- /dev/null +++ b/scripts/releases/geventreleases.sh @@ -0,0 +1,27 @@ +#!/opt/local/bin/bash + +# Quick hack script to create many gevent releases. +# Contains hardcoded paths. Probably only works on my (JAM) machine +# (OS X 10.11) + +mkdir /tmp/gevent/ + + +# 2.7 is a python.org build, builds a 10_6_intel wheel +./geventrel.sh /usr/local/bin/python2.7 + +# 3.4 is a python.org build, builds a 10_6_intel wheel +./geventrel.sh /usr/local/bin/python3.4 + +# 3.5 is a python.org build, builds a 10_6_intel wheel +./geventrel.sh /usr/local/bin/python3.5 + +# 3.6 is a python.org build, builds a 10_6_intel wheel +./geventrel.sh /usr/local/bin/python3.6 + +# 3.7 is a python.org build, builds a 10_6_intel wheel +./geventrel.sh /usr/local/bin/python3.7 + + +# PyPy 4.0 +./geventrel.sh `which pypy` diff --git a/scripts/releases/make-manylinux b/scripts/releases/make-manylinux new file mode 100755 index 0000000..2d26317 --- /dev/null +++ b/scripts/releases/make-manylinux @@ -0,0 +1,33 @@ +#!/bin/bash +# Initially based on a snippet from the greenlet project. +# This needs to be run from the root of the project. +# To update: docker pull quay.io/pypa/manylinux1_x86_64 +set -e +export PYTHONUNBUFFERED=1 +export PYTHONDONTWRITEBYTECODE=1 + +if [ -d /gevent -a -d /opt/python ]; then + # Running inside docker + yum -y install libffi-devel + cd /gevent + rm -rf wheelhouse + mkdir wheelhouse + for variant in `ls -d /opt/python/cp{37,27,34,35,36}*`; do + echo "Building $variant" + mkdir /tmp/build + cd /tmp/build + git clone /gevent gevent + cd gevent + $variant/bin/pip install -q -U -r ci-requirements.txt + # manylinux 1 is too old to build libuv, don't even try. + GEVENT_NO_LIBUV_BUILD=1 PATH=$variant/bin:$PATH $variant/bin/python setup.py bdist_wheel -q + auditwheel repair dist/*.whl + cp wheelhouse/*.whl /gevent/wheelhouse + cd /gevent + rm -rf /tmp/build + done + rm -rf dist build *.egg-info + exit 0 +fi + +docker run --rm -ti -v "$(pwd):/gevent" quay.io/pypa/manylinux1_x86_64 /gevent/scripts/releases/$(basename $0) diff --git a/scripts/travis.py b/scripts/travis.py new file mode 100644 index 0000000..cbbce58 --- /dev/null +++ b/scripts/travis.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# Support functions for travis +# See https://github.com/travis-ci/travis-rubies/blob/9f7962a881c55d32da7c76baefc58b89e3941d91/build.sh + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys + +commands = {} + +def command(func): + commands[func.__name__] = func + +@command +def fold_start(): + name = sys.argv[2] + msg = sys.argv[3] + sys.stdout.write('travis_fold:start:') + sys.stdout.write(name) + sys.stdout.write(chr(0o33)) + sys.stdout.write('[33;1m') + sys.stdout.write(msg) + sys.stdout.write(chr(0o33)) + sys.stdout.write('[33;0m\n') + +@command +def fold_end(): + name = sys.argv[2] + sys.stdout.write("\ntravis_fold:end:") + sys.stdout.write(name) + sys.stdout.write("\r\n") + + +def main(): + cmd = sys.argv[1] + commands[cmd]() + + +if __name__ == '__main__': + main() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..778107f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,16 @@ +[bdist_wheel] +universal = 0 + +[zest.releaser] +python-file-with-version = src/gevent/__init__.py +create-wheel = no + +[metadata] +long_description_content_type = text/x-rst + +[check-manifest] +ignore = + src/gevent/*.c + src/gevent/*.html + src/gevent/libev/corecext.h + src/gevent/libev/corecext.html diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..86d6c5a --- /dev/null +++ b/setup.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python +"""gevent build & installation script""" +from __future__ import print_function +import sys +import os +import os.path +import sysconfig + +# setuptools is *required* on Windows +# (https://bugs.python.org/issue23246) and for PyPy. No reason not to +# use it everywhere. v24.2.0 is needed for python_requires +from setuptools import Extension, setup +from setuptools import find_packages + + +from _setuputils import read +from _setuputils import read_version +from _setuputils import system +from _setuputils import PYPY, WIN +from _setuputils import IGNORE_CFFI +from _setuputils import SKIP_LIBUV +from _setuputils import ConfiguringBuildExt +from _setuputils import BuildFailed +from _setuputils import cythonize1 + + + +if WIN: + # Make sure the env vars that make.cmd needs are set + if not os.environ.get('PYTHON_EXE'): + os.environ['PYTHON_EXE'] = 'pypy' if PYPY else 'python' + if not os.environ.get('PYEXE'): + os.environ['PYEXE'] = os.environ['PYTHON_EXE'] + + +if PYPY and sys.pypy_version_info[:3] < (2, 6, 1): # pylint:disable=no-member + # We have to have CFFI >= 1.3.0, and this platform cannot upgrade + # it. + raise Exception("PyPy >= 2.6.1 is required") + + + +__version__ = read_version() + + +from _setuplibev import libev_configure_command +from _setuplibev import LIBEV_EMBED +from _setuplibev import CORE + +from _setupares import ARES + +# Get access to the greenlet header file. +# The sysconfig dir is not enough if we're in a virtualenv +# See https://github.com/pypa/pip/issues/4610 +include_dirs = [sysconfig.get_path("include")] +venv_include_dir = os.path.join(sys.prefix, 'include', 'site', + 'python' + sysconfig.get_python_version()) +venv_include_dir = os.path.abspath(venv_include_dir) +if os.path.exists(venv_include_dir): + include_dirs.append(venv_include_dir) + +# If we're installed via buildout, and buildout also installs +# greenlet, we have *NO* access to greenlet.h at all. So include +# our own copy as a fallback. +include_dirs.append('deps') + +SEMAPHORE = Extension(name="gevent.__semaphore", + sources=["src/gevent/_semaphore.py"], + depends=['src/gevent/__semaphore.pxd'], + include_dirs=include_dirs) + + +LOCAL = Extension(name="gevent._local", + sources=["src/gevent/local.py"], + depends=['src/gevent/_local.pxd'], + include_dirs=include_dirs) + + +GREENLET = Extension(name="gevent._greenlet", + sources=[ + "src/gevent/greenlet.py", + ], + depends=[ + 'src/gevent/_greenlet.pxd', + 'src/gevent/__ident.pxd', + 'src/gevent/_ident.py' + ], + include_dirs=include_dirs) + +ABSTRACT_LINKABLE = Extension(name="gevent.__abstract_linkable", + sources=["src/gevent/_abstract_linkable.py"], + depends=['src/gevent/__abstract_linkable.pxd'], + include_dirs=include_dirs) + + +IDENT = Extension(name="gevent.__ident", + sources=["src/gevent/_ident.py"], + depends=['src/gevent/__ident.pxd'], + include_dirs=include_dirs) + + +IMAP = Extension(name="gevent.__imap", + sources=["src/gevent/_imap.py"], + depends=['src/gevent/__imap.pxd'], + include_dirs=include_dirs) + +EVENT = Extension(name="gevent._event", + sources=["src/gevent/event.py"], + depends=['src/gevent/_event.pxd'], + include_dirs=include_dirs) + +QUEUE = Extension(name="gevent._queue", + sources=["src/gevent/queue.py"], + depends=['src/gevent/_queue.pxd'], + include_dirs=include_dirs) + +HUB_LOCAL = Extension(name="gevent.__hub_local", + sources=["src/gevent/_hub_local.py"], + depends=['src/gevent/__hub_local.pxd'], + include_dirs=include_dirs) + +WAITER = Extension(name="gevent.__waiter", + sources=["src/gevent/_waiter.py"], + depends=['src/gevent/__waiter.pxd'], + include_dirs=include_dirs) + +HUB_PRIMITIVES = Extension(name="gevent.__hub_primitives", + sources=["src/gevent/_hub_primitives.py"], + depends=['src/gevent/__hub_primitives.pxd'], + include_dirs=include_dirs) + +GLT_PRIMITIVES = Extension(name="gevent.__greenlet_primitives", + sources=["src/gevent/_greenlet_primitives.py"], + depends=['src/gevent/__greenlet_primitives.pxd'], + include_dirs=include_dirs) + +TRACER = Extension(name="gevent.__tracer", + sources=["src/gevent/_tracer.py"], + depends=['src/gevent/__tracer.pxd'], + include_dirs=include_dirs) + + +_to_cythonize = [ + GLT_PRIMITIVES, + HUB_PRIMITIVES, + HUB_LOCAL, + WAITER, + GREENLET, + TRACER, + + ABSTRACT_LINKABLE, + SEMAPHORE, + LOCAL, + + IDENT, + IMAP, + EVENT, + QUEUE, +] + +EXT_MODULES = [ + CORE, + ARES, + ABSTRACT_LINKABLE, + SEMAPHORE, + LOCAL, + GREENLET, + IDENT, + IMAP, + EVENT, + QUEUE, + HUB_LOCAL, + WAITER, + HUB_PRIMITIVES, + GLT_PRIMITIVES, + TRACER, +] + +LIBEV_CFFI_MODULE = 'src/gevent/libev/_corecffi_build.py:ffi' +LIBUV_CFFI_MODULE = 'src/gevent/libuv/_corecffi_build.py:ffi' +cffi_modules = [] + +if not WIN: + # We can't properly handle (hah!) file-descriptors and + # handle mapping on Windows/CFFI with libev, because the file needed, + # libev_vfd.h, can't be included, linked, and used: it uses + # Python API functions, and you're not supposed to do that from + # CFFI code. Plus I could never get the libraries= line to ffi.compile() + # correct to make linking work. + cffi_modules.append( + LIBEV_CFFI_MODULE + ) + +if not SKIP_LIBUV: + # libuv can't be built on manylinux1 because it needs glibc >= 2.12 + # but manylinux1 has only 2.5, so we set SKIP_LIBUV in the script make-manylinux + cffi_modules.append(LIBUV_CFFI_MODULE) + +greenlet_requires = [ + # We need to watch our greenlet version fairly carefully, + # since we compile cython code that extends the greenlet object. + # Binary compatibility would break if the greenlet struct changes. + # (Which it did in 0.4.14 for Python 3.7) + 'greenlet >= 0.4.14; platform_python_implementation=="CPython"', +] + +# Note that we don't add cffi to install_requires, it's +# optional. We tend to build and distribute wheels with the CFFI +# modules built and they can be imported if CFFI is installed. +# We need cffi 1.4.0 for new style callbacks; +# we need cffi 1.11.3 (on CPython 3) to avoid test errors. + +# The exception is on Windows, where we want the libuv backend we distribute +# to be the default, and that requires cffi; but don't try to install it +# on PyPy or it messes up the build +cffi_requires = [ + "cffi >= 1.11.5 ; sys_platform == 'win32' and platform_python_implementation == 'CPython'", +] + + +install_requires = greenlet_requires + cffi_requires + +# We use headers from greenlet, so it needs to be installed before we +# can compile. If it isn't already installed before we start +# installing, and we say 'pip install gevent', a 'setup_requires' +# doesn't save us: pip happily downloads greenlet and drops it in a +# .eggs/ directory in the build directory, but that directory doesn't +# have includes! So we fail to build a wheel, pip goes ahead and +# installs greenlet, and builds gevent again, which works. + +# Since we ship the greenlet header for buildout support (which fails +# to install the headers at all, AFAICS, we don't need to bother with +# the buggy setup_requires.) + +setup_requires = cffi_requires + [] + +if PYPY: + # These use greenlet/greenlet.h, which doesn't exist on PyPy + EXT_MODULES.remove(LOCAL) + EXT_MODULES.remove(GREENLET) + EXT_MODULES.remove(SEMAPHORE) + EXT_MODULES.remove(ABSTRACT_LINKABLE) + + # As of PyPy 5.10, this builds, but won't import (missing _Py_ReprEnter) + EXT_MODULES.remove(CORE) + + # This uses PyWeakReference and doesn't compile on PyPy + EXT_MODULES.remove(IDENT) + + _to_cythonize.remove(LOCAL) + _to_cythonize.remove(GREENLET) + _to_cythonize.remove(SEMAPHORE) + _to_cythonize.remove(IDENT) + _to_cythonize.remove(ABSTRACT_LINKABLE) + + EXT_MODULES.remove(IMAP) + _to_cythonize.remove(IMAP) + + EXT_MODULES.remove(EVENT) + _to_cythonize.remove(EVENT) + + EXT_MODULES.remove(QUEUE) + _to_cythonize.remove(QUEUE) + + EXT_MODULES.remove(HUB_LOCAL) + _to_cythonize.remove(HUB_LOCAL) + + EXT_MODULES.remove(WAITER) + _to_cythonize.remove(WAITER) + + EXT_MODULES.remove(GLT_PRIMITIVES) + _to_cythonize.remove(GLT_PRIMITIVES) + + EXT_MODULES.remove(HUB_PRIMITIVES) + _to_cythonize.remove(HUB_PRIMITIVES) + + EXT_MODULES.remove(TRACER) + _to_cythonize.remove(TRACER) + + +for mod in _to_cythonize: + EXT_MODULES.remove(mod) + EXT_MODULES.append(cythonize1(mod)) +del _to_cythonize + + +if IGNORE_CFFI and not PYPY: + # Allow distributors to turn off CFFI builds + # even if it's available, because CFFI always embeds + # our copy of libev/libuv and they may not want that. + del cffi_modules[:] + +# If we are running info / help commands, or we're being imported by +# tools like pyroma, we don't need to build anything +_BUILDING = True +if ((len(sys.argv) >= 2 + and ('--help' in sys.argv[1:] + or sys.argv[1] in ('--help-commands', + 'egg_info', + '--version', + 'clean', + '--long-description'))) + or __name__ != '__main__'): + _BUILDING = False + +def make_long_description(): + readme = read('README.rst') + about = read('doc', '_about.rst') + install = read('doc', 'install.rst') + readme = readme.replace('.. include:: doc/_about.rst', + about) + readme = readme.replace('.. include:: doc/install.rst', + install) + + return readme + + +def run_setup(ext_modules, run_make): + if run_make: + if (not LIBEV_EMBED and not WIN and cffi_modules) or PYPY: + # We're not embedding libev but we do want + # to build the CFFI module. We need to configure libev + # because the CORE Extension won't. + # TODO: Generalize this. + if LIBEV_CFFI_MODULE in cffi_modules and not WIN: + system(libev_configure_command) + + setup( + name='gevent', + version=__version__, + description='Coroutine-based network library', + long_description=make_long_description(), + license='MIT', + keywords='greenlet coroutine cooperative multitasking light threads monkey', + author='Denis Bilenko', + author_email='denis.bilenko@gmail.com', + maintainer='Jason Madden', + maintainer_email='jason@nextthought.com', + url='http://www.gevent.org/', + project_urls={ + 'Bug Tracker': 'https://github.com/gevent/gevent/issues', + 'Source Code': 'https://github.com/gevent/gevent/', + 'Documentation': 'http://www.gevent.org', + }, + package_dir={'': 'src'}, + packages=find_packages('src'), + include_package_data=True, + ext_modules=ext_modules, + cmdclass=dict(build_ext=ConfiguringBuildExt), + install_requires=install_requires, + setup_requires=setup_requires, + extras_require={ + 'dnspython': [ + 'dnspython', + 'idna', + ], + 'events': [ + 'zope.event', + 'zope.interface', + ], + 'doc': [ + 'repoze.sphinx.autointerface', + ], + 'test': [ + 'zope.interface', + 'zope.event', + + # Makes tests faster + # Fails to build on PyPy on Windows. + 'psutil ; platform_python_implementation == "CPython" or sys_platform != "win32"', + # examples, called from tests, use this + 'requests', + + # We don't run coverage on Windows, and pypy can't build it there + # anyway (coveralls -> cryptopgraphy -> openssl) + 'coverage>=5.0a3 ; sys_platform != "win32"', + 'coveralls>=1.0 ; sys_platform != "win32"', + + 'futures ; python_version == "2.7"', + 'mock ; python_version == "2.7"', + + # leak checks. previously we had a hand-rolled version. + 'objgraph', + ] + }, + # It's always safe to pass the CFFI keyword, even if + # cffi is not installed: it's just ignored in that case. + cffi_modules=cffi_modules, + zip_safe=False, + test_suite="greentest.testrunner", + classifiers=[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX", + "Operating System :: Microsoft :: Windows", + "Topic :: Internet", + "Topic :: Software Development :: Libraries :: Python Modules", + "Intended Audience :: Developers", + "Development Status :: 4 - Beta" + ], + python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", + entry_points={ + 'gevent.plugins.monkey.will_patch_all': [ + "signal_os_incompat = gevent.monkey:_subscribe_signal_os", + ], + }, + ) + +# Tools like pyroma expect the actual call to `setup` to be performed +# at the top-level at import time, so don't stash it away behind 'if +# __name__ == __main__' + +if os.getenv('READTHEDOCS'): + # Sometimes RTD fails to put our virtualenv bin directory + # on the PATH, meaning we can't run cython. Fix that. + new_path = os.environ['PATH'] + os.pathsep + os.path.dirname(sys.executable) + os.environ['PATH'] = new_path + +try: + run_setup(EXT_MODULES, run_make=_BUILDING) +except BuildFailed: + if ARES not in EXT_MODULES or not ARES.optional: + raise + EXT_MODULES.remove(ARES) + run_setup(EXT_MODULES, run_make=_BUILDING) +if ARES not in EXT_MODULES and __name__ == '__main__' and _BUILDING: + sys.stderr.write('\nWARNING: The gevent.ares extension has been disabled.\n') diff --git a/src/gevent/__abstract_linkable.pxd b/src/gevent/__abstract_linkable.pxd new file mode 100644 index 0000000..60d9203 --- /dev/null +++ b/src/gevent/__abstract_linkable.pxd @@ -0,0 +1,53 @@ +cimport cython + +from gevent.__greenlet_primitives cimport SwitchOutGreenletWithLoop +from gevent.__hub_local cimport get_hub_noargs as get_hub + +cdef InvalidSwitchError +cdef Timeout +cdef bint _greenlet_imported + +cdef extern from "greenlet/greenlet.h": + + ctypedef class greenlet.greenlet [object PyGreenlet]: + pass + + # These are actually macros and so much be included + # (defined) in each .pxd, as are the two functions + # that call them. + greenlet PyGreenlet_GetCurrent() + void PyGreenlet_Import() + +cdef inline greenlet getcurrent(): + return PyGreenlet_GetCurrent() + +cdef inline void greenlet_init(): + global _greenlet_imported + if not _greenlet_imported: + PyGreenlet_Import() + _greenlet_imported = True + +cdef void _init() + +cdef class AbstractLinkable(object): + # We declare the __weakref__ here in the base (even though + # that's not really what we want) as a workaround for a Cython + # issue we see reliably on 3.7b4 and sometimes on 3.6. See + # https://github.com/cython/cython/issues/2270 + cdef object __weakref__ + + cdef readonly SwitchOutGreenletWithLoop hub + + cdef _notifier + cdef set _links + cdef bint _notify_all + + cpdef rawlink(self, callback) + cpdef bint ready(self) + cpdef unlink(self, callback) + + cdef _check_and_notify(self) + cpdef _notify_links(self) + cdef _wait_core(self, timeout, catch=*) + cdef _wait_return_value(self, waited, wait_success) + cdef _wait(self, timeout=*) diff --git a/src/gevent/__greenlet_primitives.pxd b/src/gevent/__greenlet_primitives.pxd new file mode 100644 index 0000000..802c756 --- /dev/null +++ b/src/gevent/__greenlet_primitives.pxd @@ -0,0 +1,47 @@ +cimport cython + +# This file must not cimport anything from gevent. +cdef get_objects +cdef wref + +cdef BlockingSwitchOutError + + +cdef extern from "greenlet/greenlet.h": + + ctypedef class greenlet.greenlet [object PyGreenlet]: + pass + + # These are actually macros and so much be included + # (defined) in each .pxd, as are the two functions + # that call them. + greenlet PyGreenlet_GetCurrent() + object PyGreenlet_Switch(greenlet self, void* args, void* kwargs) + void PyGreenlet_Import() + +@cython.final +cdef inline greenlet getcurrent(): + return PyGreenlet_GetCurrent() + +cdef bint _greenlet_imported + +cdef inline void greenlet_init(): + global _greenlet_imported + if not _greenlet_imported: + PyGreenlet_Import() + _greenlet_imported = True + +cdef inline object _greenlet_switch(greenlet self): + return PyGreenlet_Switch(self, NULL, NULL) + +cdef class TrackedRawGreenlet(greenlet): + pass + +cdef class SwitchOutGreenletWithLoop(TrackedRawGreenlet): + cdef public loop + + cpdef switch(self) + cpdef switch_out(self) + + +cpdef list get_reachable_greenlets() diff --git a/src/gevent/__hub_local.pxd b/src/gevent/__hub_local.pxd new file mode 100644 index 0000000..e7df56f --- /dev/null +++ b/src/gevent/__hub_local.pxd @@ -0,0 +1,17 @@ +from gevent.__greenlet_primitives cimport SwitchOutGreenletWithLoop + +cdef _threadlocal + +cpdef get_hub_class() +cpdef SwitchOutGreenletWithLoop get_hub_if_exists() +cpdef set_hub(SwitchOutGreenletWithLoop hub) +cpdef get_loop() +cpdef set_loop(loop) + +# We can't cdef this, it won't do varargs. +# cpdef WaitOperationsGreenlet get_hub(*args, **kwargs) + +# XXX: TODO: Move the definition of TrackedRawGreenlet +# into a file that can be cython compiled so get_hub can +# return that. +cpdef SwitchOutGreenletWithLoop get_hub_noargs() diff --git a/src/gevent/__hub_primitives.pxd b/src/gevent/__hub_primitives.pxd new file mode 100644 index 0000000..35e5ad2 --- /dev/null +++ b/src/gevent/__hub_primitives.pxd @@ -0,0 +1,73 @@ +cimport cython + +from gevent.__greenlet_primitives cimport SwitchOutGreenletWithLoop +from gevent.__hub_local cimport get_hub_noargs as get_hub + +from gevent.__waiter cimport Waiter +from gevent.__waiter cimport MultipleWaiter + +cdef InvalidSwitchError +cdef _waiter +cdef _greenlet_primitives +cdef traceback +cdef _timeout_error +cdef Timeout + + +cdef extern from "greenlet/greenlet.h": + + ctypedef class greenlet.greenlet [object PyGreenlet]: + pass + + # These are actually macros and so much be included + # (defined) in each .pxd, as are the two functions + # that call them. + greenlet PyGreenlet_GetCurrent() + void PyGreenlet_Import() + +@cython.final +cdef inline greenlet getcurrent(): + return PyGreenlet_GetCurrent() + +cdef bint _greenlet_imported + +cdef inline void greenlet_init(): + global _greenlet_imported + if not _greenlet_imported: + PyGreenlet_Import() + _greenlet_imported = True + + +cdef class WaitOperationsGreenlet(SwitchOutGreenletWithLoop): + + cpdef wait(self, watcher) + cpdef cancel_wait(self, watcher, error, close_watcher=*) + cpdef _cancel_wait(self, watcher, error, close_watcher) + +cdef class _WaitIterator: + cdef SwitchOutGreenletWithLoop _hub + cdef MultipleWaiter _waiter + cdef _switch + cdef _timeout + cdef _objects + cdef _timer + cdef Py_ssize_t _count + cdef bint _begun + + + cdef _begin(self) + cdef _cleanup(self) + + cpdef __enter__(self) + cpdef __exit__(self, typ, value, tb) + + +cpdef iwait_on_objects(objects, timeout=*, count=*) +cpdef wait_on_objects(objects=*, timeout=*, count=*) + +cdef _primitive_wait(watcher, timeout, timeout_exc, WaitOperationsGreenlet hub) +cpdef wait_on_watcher(watcher, timeout=*, timeout_exc=*, WaitOperationsGreenlet hub=*) +cpdef wait_read(fileno, timeout=*, timeout_exc=*) +cpdef wait_write(fileno, timeout=*, timeout_exc=*, event=*) +cpdef wait_readwrite(fileno, timeout=*, timeout_exc=*, event=*) +cpdef wait_on_socket(socket, watcher, timeout_exc=*) diff --git a/src/gevent/__ident.pxd b/src/gevent/__ident.pxd new file mode 100644 index 0000000..55e4307 --- /dev/null +++ b/src/gevent/__ident.pxd @@ -0,0 +1,26 @@ +cimport cython + +cdef extern from "Python.h": + + ctypedef class weakref.ref [object PyWeakReference]: + pass + +cdef heappop +cdef heappush +cdef object WeakKeyDictionary +cdef type ref + +@cython.internal +@cython.final +cdef class ValuedWeakRef(ref): + cdef object value + +@cython.final +cdef class IdentRegistry: + cdef object _registry + cdef list _available_idents + + @cython.final + cpdef object get_ident(self, obj) + @cython.final + cpdef _return_ident(self, ValuedWeakRef ref) diff --git a/src/gevent/__imap.pxd b/src/gevent/__imap.pxd new file mode 100644 index 0000000..5e3e930 --- /dev/null +++ b/src/gevent/__imap.pxd @@ -0,0 +1,45 @@ +cimport cython +from gevent._greenlet cimport Greenlet +from gevent.__semaphore cimport Semaphore +from gevent._queue cimport UnboundQueue + +@cython.freelist(100) +@cython.internal +@cython.final +cdef class Failure: + cdef readonly exc + cdef raise_exception + +cdef inline _raise_exc(Failure failure) + +cdef class IMapUnordered(Greenlet): + cdef bint _zipped + cdef func + cdef iterable + cdef spawn + cdef Semaphore _result_semaphore + cdef int _outstanding_tasks + cdef int _max_index + + cdef readonly UnboundQueue queue + cdef readonly bint finished + + cdef _inext(self) + cdef _ispawn(self, func, item, int item_index) + + # Passed to greenlet.link + cpdef _on_result(self, greenlet) + # Called directly + cdef _on_finish(self, exception) + + cdef _iqueue_value_for_success(self, greenlet) + cdef _iqueue_value_for_failure(self, greenlet) + cdef _iqueue_value_for_self_finished(self) + cdef _iqueue_value_for_self_failure(self, exception) + +cdef class IMap(IMapUnordered): + cdef int index + cdef dict _results + + @cython.locals(index=int) + cdef _inext(self) diff --git a/src/gevent/__init__.py b/src/gevent/__init__.py new file mode 100644 index 0000000..22a67e0 --- /dev/null +++ b/src/gevent/__init__.py @@ -0,0 +1,180 @@ +# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details. +""" +gevent is a coroutine-based Python networking library that uses greenlet +to provide a high-level synchronous API on top of libev event loop. + +See http://www.gevent.org/ for the documentation. + +.. versionchanged:: 1.3a2 + Add the `config` object. +""" + +from __future__ import absolute_import + +from collections import namedtuple + +_version_info = namedtuple('version_info', + ('major', 'minor', 'micro', 'releaselevel', 'serial')) + +#: The programatic version identifier. The fields have (roughly) the +#: same meaning as :data:`sys.version_info` +#: .. deprecated:: 1.2 +#: Use ``pkg_resources.parse_version(__version__)`` (or the equivalent +#: ``packaging.version.Version(__version__)``). +version_info = _version_info(1, 4, 0, 'dev', 0) + +#: The human-readable PEP 440 version identifier. +#: Use ``pkg_resources.parse_version(__version__)`` or +#: ``packaging.version.Version(__version__)`` to get a machine-usable +#: value. +__version__ = '1.4.0' + + +__all__ = [ + 'get_hub', + 'Greenlet', + 'GreenletExit', + 'spawn', + 'spawn_later', + 'spawn_raw', + 'iwait', + 'wait', + 'killall', + 'Timeout', + 'with_timeout', + 'getcurrent', + 'sleep', + 'idle', + 'kill', + 'signal', # deprecated + 'signal_handler', + 'fork', + 'reinit', + 'getswitchinterval', + 'setswitchinterval', + # Added in 1.3a2 + 'config', +] + + +import sys +if sys.platform == 'win32': + # trigger WSAStartup call + import socket # pylint:disable=unused-import,useless-suppression + del socket + +try: + # Floating point number, in number of seconds, + # like time.time + getswitchinterval = sys.getswitchinterval + setswitchinterval = sys.setswitchinterval +except AttributeError: + # Running on Python 2 + _switchinterval = 0.005 + + def getswitchinterval(): + return _switchinterval + + def setswitchinterval(interval): + # Weed out None and non-numbers. This is not + # exactly exception compatible with the Python 3 + # versions. + if interval > 0: + global _switchinterval + _switchinterval = interval + +from gevent._config import config +from gevent._hub_local import get_hub +from gevent._hub_primitives import iwait_on_objects as iwait +from gevent._hub_primitives import wait_on_objects as wait + +from gevent.greenlet import Greenlet, joinall, killall +joinall = joinall # export for pylint +spawn = Greenlet.spawn +spawn_later = Greenlet.spawn_later +#: The singleton configuration object for gevent. +config = config + +from gevent.timeout import Timeout, with_timeout +from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit +try: + from gevent.os import fork +except ImportError: + __all__.remove('fork') + +# See https://github.com/gevent/gevent/issues/648 +# A temporary backwards compatibility shim to enable users to continue +# to treat 'from gevent import signal' as a callable, to matter whether +# the 'gevent.signal' module has been imported first +from gevent.hub import signal as _signal_class +signal_handler = _signal_class +from gevent import signal as _signal_module + +# The object 'gevent.signal' must: +# - be callable, returning a gevent.hub.signal; +# - answer True to isinstance(gevent.signal(...), gevent.signal); +# - answer True to isinstance(gevent.signal(...), gevent.hub.signal) +# - have all the attributes of the module 'gevent.signal'; +# - answer True to isinstance(gevent.signal, types.ModuleType) (optional) + +# The only way to do this is to use a metaclass, an instance of which (a class) +# is put in sys.modules and is substituted for gevent.hub.signal. +# This handles everything except the last one. + + +class _signal_metaclass(type): + + def __getattr__(cls, name): + return getattr(_signal_module, name) + + def __setattr__(cls, name, value): + setattr(_signal_module, name, value) + + def __instancecheck__(cls, instance): + return isinstance(instance, _signal_class) + + def __dir__(cls): + return dir(_signal_module) + + +class signal(object): + + __doc__ = _signal_module.__doc__ + + def __new__(cls, *args, **kwargs): + return _signal_class(*args, **kwargs) + + +# The metaclass is applied after the class declaration +# for Python 2/3 compatibility +signal = _signal_metaclass(str("signal"), + (), + dict(signal.__dict__)) + +sys.modules['gevent.signal'] = signal +sys.modules['gevent.hub'].signal = signal + +del sys + + +# the following makes hidden imports visible to freezing tools like +# py2exe. see https://github.com/gevent/gevent/issues/181 +# This is not well maintained or tested, though, so it likely becomes +# outdated on each major release. + +def __dependencies_for_freezing(): # pragma: no cover + # pylint:disable=unused-import + from gevent import core + from gevent import resolver_thread + from gevent import resolver_ares + from gevent import socket as _socket + from gevent import threadpool + from gevent import thread + from gevent import threading + from gevent import select + from gevent import subprocess + import pprint + import traceback + import signal as _signal + +del __dependencies_for_freezing diff --git a/src/gevent/__semaphore.pxd b/src/gevent/__semaphore.pxd new file mode 100644 index 0000000..dc9f11c --- /dev/null +++ b/src/gevent/__semaphore.pxd @@ -0,0 +1,23 @@ +cimport cython + +from gevent.__abstract_linkable cimport AbstractLinkable +cdef Timeout + + +cdef class Semaphore(AbstractLinkable): + cdef public int counter + + cpdef bint locked(self) + cpdef int release(self) except -1000 + # We don't really want this to be public, but + # threadpool uses it + cpdef _start_notify(self) + cpdef int wait(self, object timeout=*) except -1000 + cpdef bint acquire(self, int blocking=*, object timeout=*) except -1000 + cpdef __enter__(self) + cpdef __exit__(self, object t, object v, object tb) + +cdef class BoundedSemaphore(Semaphore): + cdef readonly int _initial_value + + cpdef int release(self) except -1000 diff --git a/src/gevent/__tracer.pxd b/src/gevent/__tracer.pxd new file mode 100644 index 0000000..454147e --- /dev/null +++ b/src/gevent/__tracer.pxd @@ -0,0 +1,43 @@ +cimport cython + +cdef sys +cdef traceback + +cdef settrace +cdef getcurrent + +cdef format_run_info + +cdef perf_counter +cdef gmctime + + +cdef class GreenletTracer: + cpdef readonly object active_greenlet + cpdef readonly object previous_trace_function + cpdef readonly Py_ssize_t greenlet_switch_counter + + cdef bint _killed + + cpdef _trace(self, str event, tuple args) + + @cython.locals(did_switch=bint) + cpdef did_block_hub(self, hub) + + cpdef kill(self) + +@cython.internal +cdef class _HubTracer(GreenletTracer): + cpdef readonly object hub + cpdef readonly double max_blocking_time + + +cdef class HubSwitchTracer(_HubTracer): + cpdef readonly double last_entered_hub + +cdef class MaxSwitchTracer(_HubTracer): + cpdef readonly double max_blocking + cpdef readonly double last_switch + + @cython.locals(switched_at=double) + cpdef _trace(self, str event, tuple args) diff --git a/src/gevent/__waiter.pxd b/src/gevent/__waiter.pxd new file mode 100644 index 0000000..0763bae --- /dev/null +++ b/src/gevent/__waiter.pxd @@ -0,0 +1,48 @@ +cimport cython + +from gevent.__greenlet_primitives cimport SwitchOutGreenletWithLoop +from gevent.__hub_local cimport get_hub_noargs as get_hub + +cdef sys +cdef ConcurrentObjectUseError + + +cdef bint _greenlet_imported +cdef _NONE + +cdef extern from "greenlet/greenlet.h": + + ctypedef class greenlet.greenlet [object PyGreenlet]: + pass + + # These are actually macros and so much be included + # (defined) in each .pxd, as are the two functions + # that call them. + greenlet PyGreenlet_GetCurrent() + void PyGreenlet_Import() + +cdef inline greenlet getcurrent(): + return PyGreenlet_GetCurrent() + +cdef inline void greenlet_init(): + global _greenlet_imported + if not _greenlet_imported: + PyGreenlet_Import() + _greenlet_imported = True + +cdef class Waiter: + cdef readonly SwitchOutGreenletWithLoop hub + cdef readonly greenlet greenlet + cdef readonly value + cdef _exception + + cpdef get(self) + cpdef clear(self) + + # cpdef of switch leads to parameter errors... + #cpdef switch(self, value) + +@cython.final +@cython.internal +cdef class MultipleWaiter(Waiter): + cdef list _values diff --git a/src/gevent/_abstract_linkable.py b/src/gevent/_abstract_linkable.py new file mode 100644 index 0000000..304e75d --- /dev/null +++ b/src/gevent/_abstract_linkable.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False +""" +Internal module, support for the linkable protocol for "event" like objects. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys + +from gevent._hub_local import get_hub_noargs as get_hub + +from gevent.exceptions import InvalidSwitchError +from gevent.timeout import Timeout + +locals()['getcurrent'] = __import__('greenlet').getcurrent +locals()['greenlet_init'] = lambda: None + +__all__ = [ + 'AbstractLinkable', +] + +class AbstractLinkable(object): + # Encapsulates the standard parts of the linking and notifying + # protocol common to both repeatable events (Event, Semaphore) and + # one-time events (AsyncResult). + + __slots__ = ('hub', '_links', '_notifier', '_notify_all', '__weakref__') + + def __init__(self): + # Before this implementation, AsyncResult and Semaphore + # maintained the order of notifications, but Event did not. + + # In gevent 1.3, before Semaphore extended this class, + # that was changed to not maintain the order. It was done because + # Event guaranteed to only call callbacks once (a set) but + # AsyncResult had no such guarantees. + + # Semaphore likes to maintain order of callbacks, though, + # so when it was added we went back to a list implementation + # for storing callbacks. But we want to preserve the unique callback + # property, so we manually check. + + # We generally don't expect to have so many waiters (for any of those + # objects) that testing membership and removing is a bottleneck. + + # In PyPy 2.6.1 with Cython 0.23, `cdef public` or `cdef + # readonly` or simply `cdef` attributes of type `object` can appear to leak if + # a Python subclass is used (this is visible simply + # instantiating this subclass if _links=[]). Our _links and + # _notifier are such attributes, and gevent.thread subclasses + # this class. Thus, we carefully manage the lifetime of the + # objects we put in these attributes so that, in the normal + # case of a semaphore used correctly (deallocated when it's not + # locked and no one is waiting), the leak goes away (because + # these objects are back to None). This can also be solved on PyPy + # by simply not declaring these objects in the pxd file, but that doesn't work for + # CPython ("No attribute...") + # See https://github.com/gevent/gevent/issues/660 + self._links = set() + self._notifier = None + # This is conceptually a class attribute, defined here for ease of access in + # cython. If it's true, when notifiers fire, all existing callbacks are called. + # If its false, we only call callbacks as long as ready() returns true. + self._notify_all = True + # we don't want to do get_hub() here to allow defining module-level objects + # without initializing the hub + self.hub = None + + def linkcount(self): + # For testing: how many objects are linked to this one? + return len(self._links) + + def ready(self): + # Instances must define this + raise NotImplementedError + + def _check_and_notify(self): + # If this object is ready to be notified, begin the process. + if self.ready() and self._links and not self._notifier: + if self.hub is None: + self.hub = get_hub() + + self._notifier = self.hub.loop.run_callback(self._notify_links) + + def rawlink(self, callback): + """ + Register a callback to call when this object is ready. + + *callback* will be called in the :class:`Hub + `, so it must not use blocking gevent API. + *callback* will be passed one argument: this instance. + """ + if not callable(callback): + raise TypeError('Expected callable: %r' % (callback, )) + + self._links.add(callback) + self._check_and_notify() + + def unlink(self, callback): + """Remove the callback set by :meth:`rawlink`""" + self._links.discard(callback) + + if not self._links and self._notifier is not None: + # If we currently have one queued, de-queue it. + # This will break a reference cycle. + # (self._notifier -> self._notify_links -> self) + # But we can't set it to None in case it was actually running. + self._notifier.stop() + + + def _notify_links(self): + # We release self._notifier here. We are called by it + # at the end of the loop, and it is now false in a boolean way (as soon + # as this method returns). + notifier = self._notifier + # We were ready() at the time this callback was scheduled; + # we may not be anymore, and that status may change during + # callback processing. Some of our subclasses will want to + # notify everyone that the status was once true, even though not it + # may not be anymore. + todo = set(self._links) + try: + for link in todo: + if not self._notify_all and not self.ready(): + break + + if link not in self._links: + # Been removed already by some previous link. OK, fine. + continue + try: + link(self) + except: # pylint:disable=bare-except + # We're running in the hub, so getcurrent() returns + # a hub. + self.hub.handle_error((link, self), *sys.exc_info()) # pylint:disable=undefined-variable + finally: + if getattr(link, 'auto_unlink', None): + # This attribute can avoid having to keep a reference to the function + # *in* the function, which is a cycle + self.unlink(link) + finally: + # We should not have created a new notifier even if callbacks + # released us because we loop through *all* of our links on the + # same callback while self._notifier is still true. + assert self._notifier is notifier + self._notifier = None + + # Our set of active links changed, and we were told to stop on the first + # time we went unready. See if we're ready, and if so, go around + # again. + if not self._notify_all and todo != self._links: + self._check_and_notify() + + def _wait_core(self, timeout, catch=Timeout): + # The core of the wait implementation, handling + # switching and linking. If *catch* is set to (), + # a timeout that elapses will be allowed to be raised. + # Returns a true value if the wait succeeded without timing out. + switch = getcurrent().switch # pylint:disable=undefined-variable + self.rawlink(switch) + try: + with Timeout._start_new_or_dummy(timeout) as timer: + try: + if self.hub is None: + self.hub = get_hub() + result = self.hub.switch() + if result is not self: # pragma: no cover + raise InvalidSwitchError('Invalid switch into Event.wait(): %r' % (result, )) + return True + except catch as ex: + if ex is not timer: + raise + # test_set_and_clear and test_timeout in test_threading + # rely on the exact return values, not just truthish-ness + return False + finally: + self.unlink(switch) + + def _wait_return_value(self, waited, wait_success): + # pylint:disable=unused-argument + # Subclasses should override this to return a value from _wait. + # By default we return None. + return None # pragma: no cover all extent subclasses override + + def _wait(self, timeout=None): + if self.ready(): + return self._wait_return_value(False, False) + + gotit = self._wait_core(timeout) + return self._wait_return_value(True, gotit) + +def _init(): + greenlet_init() # pylint:disable=undefined-variable + +_init() + + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__abstract_linkable') diff --git a/src/gevent/_compat.py b/src/gevent/_compat.py new file mode 100644 index 0000000..22bad70 --- /dev/null +++ b/src/gevent/_compat.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +""" +internal gevent python 2/python 3 bridges. Not for external use. +""" + +from __future__ import print_function, absolute_import, division + +import sys +import os + + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] >= 3 +PYPY = hasattr(sys, 'pypy_version_info') +WIN = sys.platform.startswith("win") +LINUX = sys.platform.startswith('linux') +OSX = sys.platform == 'darwin' + + +PURE_PYTHON = PYPY or os.getenv('PURE_PYTHON') + +## Types + +if PY3: + string_types = (str,) + integer_types = (int,) + text_type = str + native_path_types = (str, bytes) + thread_mod_name = '_thread' + +else: + import __builtin__ # pylint:disable=import-error + string_types = (__builtin__.basestring,) + text_type = __builtin__.unicode + integer_types = (int, __builtin__.long) + native_path_types = string_types + thread_mod_name = 'thread' + +def NativeStrIO(): + import io + return io.BytesIO() if str is bytes else io.StringIO() + +## Exceptions +if PY3: + def reraise(t, value, tb=None): # pylint:disable=unused-argument + if value.__traceback__ is not tb and tb is not None: + raise value.with_traceback(tb) + raise value + def exc_clear(): + pass + +else: + from gevent._util_py2 import reraise # pylint:disable=import-error,no-name-in-module + reraise = reraise # export + exc_clear = sys.exc_clear + +## import locks +try: + # In Python 3.4 and newer in CPython and PyPy3, + # imp.acquire_lock and imp.release_lock are delegated to + # '_imp'. (Which is also used by importlib.) 'imp' itself is + # deprecated. Avoid that warning. + import _imp as imp +except ImportError: + import imp +imp_acquire_lock = imp.acquire_lock +imp_release_lock = imp.release_lock + +## Functions +if PY3: + iteritems = dict.items + itervalues = dict.values + xrange = range + izip = zip + +else: + iteritems = dict.iteritems # python 3: pylint:disable=no-member + itervalues = dict.itervalues # python 3: pylint:disable=no-member + xrange = __builtin__.xrange + from itertools import izip # python 3: pylint:disable=no-member,no-name-in-module + izip = izip + +# fspath from 3.6 os.py, but modified to raise the same exceptions as the +# real native implementation. +# Define for testing +def _fspath(path): + """ + Return the path representation of a path-like object. + + If str or bytes is passed in, it is returned unchanged. Otherwise the + os.PathLike interface is used to get the path representation. If the + path representation is not str or bytes, TypeError is raised. If the + provided path is not str, bytes, or os.PathLike, TypeError is raised. + """ + if isinstance(path, native_path_types): + return path + + # Work from the object's type to match method resolution of other magic + # methods. + path_type = type(path) + try: + path_type_fspath = path_type.__fspath__ + except AttributeError: + raise TypeError("expected str, bytes or os.PathLike object, " + "not " + path_type.__name__) + + path_repr = path_type_fspath(path) + if isinstance(path_repr, native_path_types): + return path_repr + + raise TypeError("expected {}.__fspath__() to return str or bytes, " + "not {}".format(path_type.__name__, + type(path_repr).__name__)) +try: + from os import fspath # pylint: disable=unused-import,no-name-in-module +except ImportError: + # if not available, use the Python version as transparently as + # possible + fspath = _fspath + fspath.__name__ = 'fspath' + +try: + from os import fsencode # pylint: disable=unused-import,no-name-in-module +except ImportError: + encoding = sys.getfilesystemencoding() or ('utf-8' if not WIN else 'mbcs') + errors = 'strict' if WIN and encoding == 'mbcs' else 'surrogateescape' + + # Added in 3.2, so this is for Python 2.7. Note that it doesn't have + # sys.getfilesystemencodeerrors(), which was added in 3.6 + def fsencode(filename): + """Encode filename (an os.PathLike, bytes, or str) to the filesystem + encoding with 'surrogateescape' error handler, return bytes unchanged. + On Windows, use 'strict' error handler if the file system encoding is + 'mbcs' (which is the default encoding). + """ + filename = fspath(filename) # Does type-checking of `filename`. + if isinstance(filename, bytes): + return filename + + try: + return filename.encode(encoding, errors) + except LookupError: + # Can't encode it, and the error handler doesn't + # exist. Probably on Python 2 with an astral character. + # Not sure how to handle this. + raise UnicodeEncodeError("Can't encode path to filesystem encoding") + + +## Clocks +try: + # Python 3.3+ (PEP 418) + from time import perf_counter + perf_counter = perf_counter +except ImportError: + import time + + if sys.platform == "win32": + perf_counter = time.clock + else: + perf_counter = time.time diff --git a/src/gevent/_config.py b/src/gevent/_config.py new file mode 100644 index 0000000..5a9990f --- /dev/null +++ b/src/gevent/_config.py @@ -0,0 +1,701 @@ +# Copyright (c) 2018 gevent. See LICENSE for details. +""" +gevent tunables. + +This should be used as ``from gevent import config``. That variable +is an object of :class:`Config`. + +.. versionadded:: 1.3a2 +""" + +from __future__ import print_function, absolute_import, division + +import importlib +import os +import textwrap + +from gevent._compat import string_types +from gevent._compat import WIN + +__all__ = [ + 'config', +] + +ALL_SETTINGS = [] + +class SettingType(type): + # pylint:disable=bad-mcs-classmethod-argument + + def __new__(cls, name, bases, cls_dict): + if name == 'Setting': + return type.__new__(cls, name, bases, cls_dict) + + cls_dict["order"] = len(ALL_SETTINGS) + if 'name' not in cls_dict: + cls_dict['name'] = name.lower() + + if 'environment_key' not in cls_dict: + cls_dict['environment_key'] = 'GEVENT_' + cls_dict['name'].upper() + + + new_class = type.__new__(cls, name, bases, cls_dict) + new_class.fmt_desc(cls_dict.get("desc", "")) + new_class.__doc__ = new_class.desc + ALL_SETTINGS.append(new_class) + + if new_class.document: + setting_name = cls_dict['name'] + + def getter(self): + return self.settings[setting_name].get() + + def setter(self, value): # pragma: no cover + # The setter should never be hit, Config has a + # __setattr__ that would override. But for the sake + # of consistency we provide one. + self.settings[setting_name].set(value) + + prop = property(getter, setter, doc=new_class.__doc__) + + setattr(Config, cls_dict['name'], prop) + return new_class + + def fmt_desc(cls, desc): + desc = textwrap.dedent(desc).strip() + if hasattr(cls, 'shortname_map'): + desc += ( + "\n\nThis is an importable value. It can be " + "given as a string naming an importable object, " + "or a list of strings in preference order and the first " + "successfully importable object will be used. (Separate values " + "in the environment variable with commas.) " + "It can also be given as the callable object itself (in code). " + ) + if cls.shortname_map: + desc += "Shorthand names for default objects are %r" % (list(cls.shortname_map),) + if getattr(cls.validate, '__doc__'): + desc += '\n\n' + textwrap.dedent(cls.validate.__doc__).strip() + if isinstance(cls.default, str) and hasattr(cls, 'shortname_map'): + default = "`%s`" % (cls.default,) + else: + default = "`%r`" % (cls.default,) + desc += "\n\nThe default value is %s" % (default,) + desc += ("\n\nThe environment variable ``%s`` " + "can be used to control this." % (cls.environment_key,)) + setattr(cls, "desc", desc) + return desc + +def validate_invalid(value): + raise ValueError("Not a valid value: %r" % (value,)) + +def validate_bool(value): + """ + This is a boolean value. + + In the environment variable, it may be given as ``1``, ``true``, + ``on`` or ``yes`` for `True`, or ``0``, ``false``, ``off``, or + ``no`` for `False`. + """ + if isinstance(value, string_types): + value = value.lower().strip() + if value in ('1', 'true', 'on', 'yes'): + value = True + elif value in ('0', 'false', 'off', 'no') or not value: + value = False + else: + raise ValueError("Invalid boolean string: %r" % (value,)) + return bool(value) + +def validate_anything(value): + return value + +convert_str_value_as_is = validate_anything + +class Setting(object): + name = None + value = None + validate = staticmethod(validate_invalid) + default = None + environment_key = None + document = True + + desc = """\ + + A long ReST description. + + The first line should be a single sentence. + + """ + + def _convert(self, value): + if isinstance(value, string_types): + return value.split(',') + return value + + def _default(self): + result = os.environ.get(self.environment_key, self.default) + result = self._convert(result) + return result + + def get(self): + # If we've been specifically set, return it + if 'value' in self.__dict__: + return self.value + # Otherwise, read from the environment and reify + # so we return consistent results. + self.value = self.validate(self._default()) + return self.value + + def set(self, val): + self.value = self.validate(self._convert(val)) + + +Setting = SettingType('Setting', (Setting,), dict(Setting.__dict__)) + +def make_settings(): + """ + Return fresh instances of all classes defined in `ALL_SETTINGS`. + """ + settings = {} + for setting_kind in ALL_SETTINGS: + setting = setting_kind() + assert setting.name not in settings + settings[setting.name] = setting + return settings + + +class Config(object): + """ + Global configuration for gevent. + + There is one instance of this object at ``gevent.config``. If you + are going to make changes in code, instead of using the documented + environment variables, you need to make the changes before using + any parts of gevent that might need those settings. For example:: + + >>> from gevent import config + >>> config.fileobject = 'thread' + + >>> from gevent import fileobject + >>> fileobject.FileObject.__name__ + 'FileObjectThread' + + .. versionadded:: 1.3a2 + + """ + + def __init__(self): + self.settings = make_settings() + + def __getattr__(self, name): + if name not in self.settings: + raise AttributeError("No configuration setting for: %r" % name) + return self.settings[name].get() + + def __setattr__(self, name, value): + if name != "settings" and name in self.settings: + self.set(name, value) + else: + super(Config, self).__setattr__(name, value) + + def set(self, name, value): + if name not in self.settings: + raise AttributeError("No configuration setting for: %r" % name) + self.settings[name].set(value) + + def __dir__(self): + return list(self.settings) + + +class ImportableSetting(object): + + def _import_one_of(self, candidates): + assert isinstance(candidates, list) + if not candidates: + raise ImportError('Cannot import from empty list') + + for item in candidates[:-1]: + try: + return self._import_one(item) + except ImportError: + pass + + return self._import_one(candidates[-1]) + + def _import_one(self, path, _MISSING=object()): + if not isinstance(path, string_types): + return path + + if '.' not in path or '/' in path: + raise ImportError("Cannot import %r. " + "Required format: [package.]module.class. " + "Or choose from %r" + % (path, list(self.shortname_map))) + + + module, item = path.rsplit('.', 1) + module = importlib.import_module(module) + x = getattr(module, item, _MISSING) + if x is _MISSING: + raise ImportError('Cannot import %r from %r' % (item, module)) + return x + + shortname_map = {} + + def validate(self, value): + if isinstance(value, type): + return value + return self._import_one_of([self.shortname_map.get(x, x) for x in value]) + + def get_options(self): + result = {} + for name, val in self.shortname_map.items(): + try: + result[name] = self._import_one(val) + except ImportError as e: + result[name] = e + return result + + +class BoolSettingMixin(object): + validate = staticmethod(validate_bool) + # Don't do string-to-list conversion. + _convert = staticmethod(convert_str_value_as_is) + + +class IntSettingMixin(object): + # Don't do string-to-list conversion. + def _convert(self, value): + if value: + return int(value) + + validate = staticmethod(validate_anything) + + +class _PositiveValueMixin(object): + + def validate(self, value): + if value is not None and value <= 0: + raise ValueError("Must be positive") + return value + + +class FloatSettingMixin(_PositiveValueMixin): + def _convert(self, value): + if value: + return float(value) + + +class ByteCountSettingMixin(_PositiveValueMixin): + + _MULTIPLES = { + # All keys must be the same size. + 'kb': 1024, + 'mb': 1024 * 1024, + 'gb': 1024 * 1024 * 1024, + } + + _SUFFIX_SIZE = 2 + + def _convert(self, value): + if not value or not isinstance(value, str): + return value + value = value.lower() + for s, m in self._MULTIPLES.items(): + if value[-self._SUFFIX_SIZE:] == s: + return int(value[:-self._SUFFIX_SIZE]) * m + return int(value) + + +class Resolver(ImportableSetting, Setting): + + desc = """\ + The callable that will be used to create + :attr:`gevent.hub.Hub.resolver`. + + See :doc:`dns` for more information. + """ + + default = [ + 'thread', + 'dnspython', + 'ares', + 'block', + ] + + shortname_map = { + 'ares': 'gevent.resolver.ares.Resolver', + 'thread': 'gevent.resolver.thread.Resolver', + 'block': 'gevent.resolver.blocking.Resolver', + 'dnspython': 'gevent.resolver.dnspython.Resolver', + } + + + +class Threadpool(ImportableSetting, Setting): + + desc = """\ + The kind of threadpool we use. + """ + + default = 'gevent.threadpool.ThreadPool' + + +class Loop(ImportableSetting, Setting): + + desc = """\ + The kind of the loop we use. + + On Windows, this defaults to libuv, while on + other platforms it defaults to libev. + + """ + + default = [ + 'libev-cext', + 'libev-cffi', + 'libuv-cffi', + ] if not WIN else [ + 'libuv-cffi', + 'libev-cext', + 'libev-cffi', + ] + + shortname_map = { + 'libev-cext': 'gevent.libev.corecext.loop', + 'libev-cffi': 'gevent.libev.corecffi.loop', + 'libuv-cffi': 'gevent.libuv.loop.loop', + } + + shortname_map['libuv'] = shortname_map['libuv-cffi'] + + +class FormatContext(ImportableSetting, Setting): + name = 'format_context' + + # using pprint.pformat can override custom __repr__ methods on dict/list + # subclasses, which can be a security concern + default = 'pprint.saferepr' + + +class LibevBackend(Setting): + name = 'libev_backend' + environment_key = 'GEVENT_BACKEND' + + desc = """\ + The backend for libev, such as 'select' + """ + + default = None + + validate = staticmethod(validate_anything) + + +class FileObject(ImportableSetting, Setting): + desc = """\ + The kind of ``FileObject`` we will use. + + See :mod:`gevent.fileobject` for a detailed description. + + """ + environment_key = 'GEVENT_FILE' + + default = [ + 'posix', + 'thread', + ] + + shortname_map = { + 'thread': 'gevent._fileobjectcommon.FileObjectThread', + 'posix': 'gevent._fileobjectposix.FileObjectPosix', + 'block': 'gevent._fileobjectcommon.FileObjectBlock' + } + + +class WatchChildren(BoolSettingMixin, Setting): + desc = """\ + Should we *not* watch children with the event loop watchers? + + This is an advanced setting. + + See :mod:`gevent.os` for a detailed description. + """ + name = 'disable_watch_children' + environment_key = 'GEVENT_NOWAITPID' + default = False + + +class TraceMalloc(IntSettingMixin, Setting): + name = 'trace_malloc' + environment_key = 'PYTHONTRACEMALLOC' + default = False + + desc = """\ + Should FFI objects track their allocation? + + This is only useful for low-level debugging. + + On Python 3, this environment variable is built in to the + interpreter, and it may also be set with the ``-X + tracemalloc`` command line argument. + + On Python 2, gevent interprets this argument and adds extra + tracking information for FFI objects. + """ + + +class TrackGreenletTree(BoolSettingMixin, Setting): + name = 'track_greenlet_tree' + environment_key = 'GEVENT_TRACK_GREENLET_TREE' + default = True + + desc = """\ + Should `Greenlet` objects track their spawning tree? + + Setting this to a false value will make spawning `Greenlet` + objects and using `spawn_raw` faster, but the + ``spawning_greenlet``, ``spawn_tree_locals`` and ``spawning_stack`` + will not be captured. + + .. versionadded:: 1.3b1 + """ + + +## Monitoring settings +# All env keys should begin with GEVENT_MONITOR + +class MonitorThread(BoolSettingMixin, Setting): + name = 'monitor_thread' + environment_key = 'GEVENT_MONITOR_THREAD_ENABLE' + default = False + + desc = """\ + Should each hub start a native OS thread to monitor + for problems? + + Such a thread will periodically check to see if the event loop + is blocked for longer than `max_blocking_time`, producing output on + the hub's exception stream (stderr by default) if it detects this condition. + + If this setting is true, then this thread will be created + the first time the hub is switched to, + or you can call :meth:`gevent.hub.Hub.start_periodic_monitoring_thread` at any + time to create it (from the same thread that will run the hub). That function + will return an instance of :class:`gevent.events.IPeriodicMonitorThread` + to which you can add your own monitoring functions. That function + also emits an event of :class:`gevent.events.PeriodicMonitorThreadStartedEvent`. + + .. seealso:: `max_blocking_time` + + .. versionadded:: 1.3b1 + """ + +class MaxBlockingTime(FloatSettingMixin, Setting): + name = 'max_blocking_time' + # This environment key doesn't follow the convention because it's + # meant to match a key used by existing projects + environment_key = 'GEVENT_MAX_BLOCKING_TIME' + default = 0.1 + + desc = """\ + If the `monitor_thread` is enabled, this is + approximately how long (in seconds) + the event loop will be allowed to block before a warning is issued. + + This function depends on using `greenlet.settrace`, so installing + your own trace function after starting the monitoring thread will + cause this feature to misbehave unless you call the function + returned by `greenlet.settrace`. If you install a tracing function *before* + the monitoring thread is started, it will still be called. + + .. note:: In the unlikely event of creating and using multiple different + gevent hubs in the same native thread in a short period of time, + especially without destroying the hubs, false positives may be reported. + + .. versionadded:: 1.3b1 + """ + +class MonitorMemoryPeriod(FloatSettingMixin, Setting): + name = 'memory_monitor_period' + + environment_key = 'GEVENT_MONITOR_MEMORY_PERIOD' + default = 5 + + desc = """\ + If `monitor_thread` is enabled, this is approximately how long + (in seconds) we will go between checking the processes memory usage. + + Checking the memory usage is relatively expensive on some operating + systems, so this should not be too low. gevent will place a floor + value on it. + """ + +class MonitorMemoryMaxUsage(ByteCountSettingMixin, Setting): + name = 'max_memory_usage' + + environment_key = 'GEVENT_MONITOR_MEMORY_MAX' + default = None + + desc = """\ + If `monitor_thread` is enabled, + then if memory usage exceeds this amount (in bytes), events will + be emitted. See `gevent.events`. In the environment variable, you can use + a suffix of 'kb', 'mb' or 'gb' to specify the value in kilobytes, megabytes + or gigibytes. + + There is no default value for this setting. If you wish to + cap memory usage, you must choose a value. + """ + +# The ares settings are all interpreted by +# gevent/resolver/ares.pyx, so we don't do +# any validation here. + +class AresSettingMixin(object): + + document = False + + @property + def kwarg_name(self): + return self.name[5:] + + validate = staticmethod(validate_anything) + + _convert = staticmethod(convert_str_value_as_is) + +class AresFlags(AresSettingMixin, Setting): + name = 'ares_flags' + default = None + environment_key = 'GEVENTARES_FLAGS' + +class AresTimeout(AresSettingMixin, Setting): + document = True + name = 'ares_timeout' + default = None + environment_key = 'GEVENTARES_TIMEOUT' + desc = """\ + + .. deprecated:: 1.3a2 + Prefer the :attr:`resolver_timeout` setting. If both are set, + the results are not defined. + """ + +class AresTries(AresSettingMixin, Setting): + name = 'ares_tries' + default = None + environment_key = 'GEVENTARES_TRIES' + +class AresNdots(AresSettingMixin, Setting): + name = 'ares_ndots' + default = None + environment_key = 'GEVENTARES_NDOTS' + +class AresUDPPort(AresSettingMixin, Setting): + name = 'ares_udp_port' + default = None + environment_key = 'GEVENTARES_UDP_PORT' + +class AresTCPPort(AresSettingMixin, Setting): + name = 'ares_tcp_port' + default = None + environment_key = 'GEVENTARES_TCP_PORT' + +class AresServers(AresSettingMixin, Setting): + document = True + name = 'ares_servers' + default = None + environment_key = 'GEVENTARES_SERVERS' + desc = """\ + A list of strings giving the IP addresses of nameservers for the ares resolver. + + In the environment variable, these strings are separated by commas. + + .. deprecated:: 1.3a2 + Prefer the :attr:`resolver_nameservers` setting. If both are set, + the results are not defined. + """ + +# Generic nameservers, works for dnspython and ares. +class ResolverNameservers(AresSettingMixin, Setting): + document = True + name = 'resolver_nameservers' + default = None + environment_key = 'GEVENT_RESOLVER_NAMESERVERS' + desc = """\ + A list of strings giving the IP addresses of nameservers for the (non-system) resolver. + + In the environment variable, these strings are separated by commas. + + .. rubric:: Resolver Behaviour + + * blocking + + Ignored + + * Threaded + + Ignored + + * dnspython + + If this setting is not given, the dnspython resolver will + load nameservers to use from ``/etc/resolv.conf`` + or the Windows registry. This setting replaces any nameservers read + from those means. Note that the file and registry are still read + for other settings. + + .. caution:: dnspython does not validate the members of the list. + An improper address (such as a hostname instead of IP) has + undefined results, including hanging the process. + + * ares + + Similar to dnspython, but with more platform and compile-time + options. ares validates that the members of the list are valid + addresses. + """ + + # Normal string-to-list rules. But still validate_anything. + _convert = Setting._convert + + # TODO: In the future, support reading a resolv.conf file + # *other* than /etc/resolv.conf, and do that both on Windows + # and other platforms. Also offer the option to disable the system + # configuration entirely. + + @property + def kwarg_name(self): + return 'servers' + +# Generic timeout, works for dnspython and ares +class ResolverTimeout(FloatSettingMixin, AresSettingMixin, Setting): + document = True + name = 'resolver_timeout' + environment_key = 'GEVENT_RESOLVER_TIMEOUT' + desc = """\ + The total amount of time that the DNS resolver will spend making queries. + + Only the ares and dnspython resolvers support this. + + .. versionadded:: 1.3a2 + """ + + @property + def kwarg_name(self): + return 'timeout' + +config = Config() + +# Go ahead and attempt to import the loop when this class is +# instantiated. The hub won't work if the loop can't be found. This +# can solve problems with the class being imported from multiple +# threads at once, leading to one of the imports failing. +# factories are themselves handled lazily. See #687. + +# Don't cache it though, in case the user re-configures through the +# API. + +try: + Loop().get() +except ImportError: # pragma: no cover + pass diff --git a/src/gevent/_event.pxd b/src/gevent/_event.pxd new file mode 100644 index 0000000..f08d2d7 --- /dev/null +++ b/src/gevent/_event.pxd @@ -0,0 +1,30 @@ +cimport cython + +from gevent.__hub_local cimport get_hub_noargs as get_hub +from gevent.__abstract_linkable cimport AbstractLinkable + +cdef _None +cdef reraise +cdef dump_traceback +cdef load_traceback + +cdef Timeout + +cdef class Event(AbstractLinkable): + cdef bint _flag + +cdef class AsyncResult(AbstractLinkable): + cdef readonly _value + cdef readonly tuple _exc_info + + # For the use of _imap.py + cdef public int _imap_task_index + + cpdef get(self, block=*, timeout=*) + cpdef bint successful(self) + + cpdef wait(self, timeout=*) + cpdef bint done(self) + + cpdef bint cancel(self) + cpdef bint cancelled(self) diff --git a/src/gevent/_ffi/__init__.py b/src/gevent/_ffi/__init__.py new file mode 100644 index 0000000..56f1e96 --- /dev/null +++ b/src/gevent/_ffi/__init__.py @@ -0,0 +1,27 @@ +""" +Internal helpers for FFI implementations. +""" +from __future__ import print_function, absolute_import + +import os +import sys + +def _dbg(*args, **kwargs): + # pylint:disable=unused-argument + pass + +#_dbg = print + +def _pid_dbg(*args, **kwargs): + kwargs['file'] = sys.stderr + print(os.getpid(), *args, **kwargs) + +CRITICAL = 1 +ERROR = 3 +DEBUG = 5 +TRACE = 9 + +GEVENT_DEBUG_LEVEL = vars()[os.getenv("GEVENT_DEBUG", 'CRITICAL').upper()] + +if GEVENT_DEBUG_LEVEL >= TRACE: + _dbg = _pid_dbg diff --git a/src/gevent/_ffi/callback.py b/src/gevent/_ffi/callback.py new file mode 100644 index 0000000..df59a9f --- /dev/null +++ b/src/gevent/_ffi/callback.py @@ -0,0 +1,58 @@ +from __future__ import absolute_import, print_function + +__all__ = [ + 'callback', +] + + +# For times when *args is captured but often not passed (empty), +# we can avoid keeping the new tuple that was created for *args +# around by using a constant. +_NOARGS = () + + +class callback(object): + + __slots__ = ('callback', 'args') + + def __init__(self, cb, args): + self.callback = cb + self.args = args or _NOARGS + + def stop(self): + self.callback = None + self.args = None + + close = stop + + # Note that __nonzero__ and pending are different + # bool() is used in contexts where we need to know whether to schedule another callback, + # so it's true if it's pending or currently running + # 'pending' has the same meaning as libev watchers: it is cleared before actually + # running the callback + + def __nonzero__(self): + # it's nonzero if it's pending or currently executing + # NOTE: This depends on loop._run_callbacks setting the args property + # to None. + return self.args is not None + __bool__ = __nonzero__ + + @property + def pending(self): + return self.callback is not None + + def _format(self): + return '' + + def __repr__(self): + result = "<%s at 0x%x" % (self.__class__.__name__, id(self)) + if self.pending: + result += " pending" + if self.callback is not None: + result += " callback=%r" % (self.callback, ) + if self.args is not None: + result += " args=%r" % (self.args, ) + if self.callback is None and self.args is None: + result += " stopped" + return result + ">" diff --git a/src/gevent/_ffi/loop.py b/src/gevent/_ffi/loop.py new file mode 100644 index 0000000..f77c25a --- /dev/null +++ b/src/gevent/_ffi/loop.py @@ -0,0 +1,713 @@ +""" +Basic loop implementation for ffi-based cores. +""" +# pylint: disable=too-many-lines, protected-access, redefined-outer-name, not-callable +from __future__ import absolute_import, print_function + +from collections import deque +import sys +import os +import traceback + +from gevent._ffi import _dbg +from gevent._ffi import GEVENT_DEBUG_LEVEL +from gevent._ffi import TRACE +from gevent._ffi.callback import callback +from gevent._compat import PYPY + +from gevent import getswitchinterval + +__all__ = [ + 'AbstractLoop', + 'assign_standard_callbacks', +] + + +class _EVENTSType(object): + def __repr__(self): + return 'gevent.core.EVENTS' + +EVENTS = GEVENT_CORE_EVENTS = _EVENTSType() + + +##### +## Note on CFFI objects, callbacks and the lifecycle of watcher objects +# +# Each subclass of `watcher` allocates a C structure of the +# appropriate type e.g., struct gevent_ev_io and holds this pointer in +# its `_gwatcher` attribute. When that watcher instance is garbage +# collected, then the C structure is also freed. The C structure is +# passed to libev from the watcher's start() method and then to the +# appropriate C callback function, e.g., _gevent_ev_io_callback, which +# passes it back to python's _python_callback where we need the +# watcher instance. Therefore, as long as that callback is active (the +# watcher is started), the watcher instance must not be allowed to get +# GC'd---any access at the C level or even the FFI level to the freed +# memory could crash the process. +# +# However, the typical idiom calls for writing something like this: +# loop.io(fd, python_cb).start() +# thus forgetting the newly created watcher subclass and allowing it to be immediately +# GC'd. To combat this, when the watcher is started, it places itself into the loop's +# `_keepaliveset`, and it only removes itself when the watcher's `stop()` method is called. +# Often, this is the *only* reference keeping the watcher object, and hence its C structure, +# alive. +# +# This is slightly complicated by the fact that the python-level +# callback, called from the C callback, could choose to manually stop +# the watcher. When we return to the C level callback, we now have an +# invalid pointer, and attempting to pass it back to Python (e.g., to +# handle an error) could crash. Hence, _python_callback, +# _gevent_io_callback, and _python_handle_error cooperate to make sure +# that the watcher instance stays in the loops `_keepaliveset` while +# the C code could be running---and if it gets removed, to not call back +# to Python again. +# See also https://github.com/gevent/gevent/issues/676 +#### +class AbstractCallbacks(object): + + + def __init__(self, ffi): + self.ffi = ffi + self.callbacks = [] + if GEVENT_DEBUG_LEVEL < TRACE: + self.from_handle = ffi.from_handle + + def from_handle(self, handle): # pylint:disable=method-hidden + x = self.ffi.from_handle(handle) + return x + + def python_callback(self, handle, revents): + """ + Returns an integer having one of three values: + + - -1 + An exception occurred during the callback and you must call + :func:`_python_handle_error` to deal with it. The Python watcher + object will have the exception tuple saved in ``_exc_info``. + - 1 + Everything went according to plan. You should check to see if the libev + watcher is still active, and call :func:`python_stop` if it is not. This will + clean up the memory. Finding the watcher still active at the event loop level, + but not having stopped itself at the gevent level is a buggy scenario and + shouldn't happen. + - 2 + Everything went according to plan, but the watcher has already + been stopped. Its memory may no longer be valid. + + This function should never return 0, as that's the default value that + Python exceptions will produce. + """ + #print("Running callback", handle) + orig_ffi_watcher = None + try: + # Even dereferencing the handle needs to be inside the try/except; + # if we don't return normally (e.g., a signal) then we wind up going + # to the 'onerror' handler (unhandled_onerror), which + # is not what we want; that can permanently wedge the loop depending + # on which callback was executing. + # XXX: See comments in that function. We may be able to restart and do better? + if not handle: + # Hmm, a NULL handle. That's not supposed to happen. + # We can easily get into a loop if we deref it and allow that + # to raise. + _dbg("python_callback got null handle") + return 1 + the_watcher = self.from_handle(handle) + orig_ffi_watcher = the_watcher._watcher + args = the_watcher.args + if args is None: + # Legacy behaviour from corecext: convert None into () + # See test__core_watcher.py + args = _NOARGS + if args and args[0] == GEVENT_CORE_EVENTS: + args = (revents, ) + args[1:] + #print("Calling function", the_watcher.callback, args) + the_watcher.callback(*args) + except: # pylint:disable=bare-except + _dbg("Got exception servicing watcher with handle", handle, sys.exc_info()) + # It's possible for ``the_watcher`` to be undefined (UnboundLocalError) + # if we threw an exception (signal) on the line that created that variable. + # This is typically the case with a signal under libuv + try: + the_watcher + except UnboundLocalError: + the_watcher = self.from_handle(handle) + the_watcher._exc_info = sys.exc_info() + # Depending on when the exception happened, the watcher + # may or may not have been stopped. We need to make sure its + # memory stays valid so we can stop it at the ev level if needed. + # If its loop is gone, it has already been stopped, + # see https://github.com/gevent/gevent/issues/1295 for a case where + # that happened + if the_watcher.loop is not None: + the_watcher.loop._keepaliveset.add(the_watcher) + return -1 + else: + if (the_watcher.loop is not None + and the_watcher in the_watcher.loop._keepaliveset + and the_watcher._watcher is orig_ffi_watcher): + # It didn't stop itself, *and* it didn't stop itself, reset + # its watcher, and start itself again. libuv's io watchers MAY + # do that. + # The normal, expected scenario when we find the watcher still + # in the keepaliveset is that it is still active at the event loop + # level, so we don't expect that python_stop gets called. + #_dbg("The watcher has not stopped itself, possibly still active", the_watcher) + return 1 + return 2 # it stopped itself + + def python_handle_error(self, handle, _revents): + _dbg("Handling error for handle", handle) + if not handle: + return + try: + watcher = self.from_handle(handle) + exc_info = watcher._exc_info + del watcher._exc_info + # In the past, we passed the ``watcher`` itself as the context, + # which typically meant that the Hub would just print + # the exception. This is a problem because sometimes we can't + # detect signals until late in ``python_callback``; specifically, + # test_selectors.py:DefaultSelectorTest.test_select_interrupt_exc + # installs a SIGALRM handler that raises an exception. That exception can happen + # before we enter ``python_callback`` or at any point within it because of the way + # libuv swallows signals. By passing None, we get the exception prapagated into + # the main greenlet (which is probably *also* not what we always want, but + # I see no way to distinguish the cases). + watcher.loop.handle_error(None, *exc_info) + finally: + # XXX Since we're here on an error condition, and we + # made sure that the watcher object was put in loop._keepaliveset, + # what about not stopping the watcher? Looks like a possible + # memory leak? + # XXX: This used to do "if revents & (libev.EV_READ | libev.EV_WRITE)" + # before stopping. Why? + try: + watcher.stop() + except: # pylint:disable=bare-except + watcher.loop.handle_error(watcher, *sys.exc_info()) + return # pylint:disable=lost-exception + + def unhandled_onerror(self, t, v, tb): + # This is supposed to be called for signals, etc. + # This is the onerror= value for CFFI. + # If we return None, C will get a value of 0/NULL; + # if we raise, CFFI will print the exception and then + # return 0/NULL; (unless error= was configured) + # If things go as planned, we return the value that asks + # C to call back and check on if the watcher needs to be closed or + # not. + + # XXX: TODO: Could this cause events to be lost? Maybe we need to return + # a value that causes the C loop to try the callback again? + # at least for signals under libuv, which are delivered at very odd times. + # Hopefully the event still shows up when we poll the next time. + watcher = None + handle = tb.tb_frame.f_locals['handle'] if tb is not None else None + if handle: # handle could be NULL + watcher = self.from_handle(handle) + if watcher is not None: + watcher.loop.handle_error(None, t, v, tb) + return 1 + + # Raising it causes a lot of noise from CFFI + print("WARNING: gevent: Unhandled error with no watcher", + file=sys.stderr) + traceback.print_exception(t, v, tb) + + def python_stop(self, handle): + if not handle: # pragma: no cover + print( + "WARNING: gevent: Unable to dereference handle; not stopping watcher. " + "Native resources may leak. This is most likely a bug in gevent.", + file=sys.stderr) + # The alternative is to crash with no helpful information + # NOTE: Raising exceptions here does nothing, they're swallowed by CFFI. + # Since the C level passed in a null pointer, even dereferencing the handle + # will just produce some exceptions. + return + watcher = self.from_handle(handle) + watcher.stop() + + if not PYPY: + def python_check_callback(self, watcher_ptr): # pylint:disable=unused-argument + # If we have the onerror callback, this is a no-op; all the real + # work to rethrow the exception is done by the onerror callback + + # NOTE: Unlike the rest of the functions, this is called with a pointer + # to the C level structure, *not* a pointer to the void* that represents a + # for the Python Watcher object. + pass + else: # PyPy + # On PyPy, we need the function to have some sort of body, otherwise + # the signal exceptions don't always get caught, *especially* with + # libuv (however, there's no reason to expect this to only be a libuv + # issue; it's just that we don't depend on the periodic signal timer + # under libev, so the issue is much more pronounced under libuv) + # test_socket's test_sendall_interrupted can hang. + # See https://github.com/gevent/gevent/issues/1112 + + def python_check_callback(self, watcher_ptr): # pylint:disable=unused-argument + # Things we've tried that *don't* work: + # greenlet.getcurrent() + # 1 + 1 + try: + raise MemoryError() + except MemoryError: + pass + + def python_prepare_callback(self, watcher_ptr): + loop = self._find_loop_from_c_watcher(watcher_ptr) + if loop is None: # pragma: no cover + print("WARNING: gevent: running prepare callbacks from a destroyed handle: ", + watcher_ptr) + return + loop._run_callbacks() + + def check_callback_onerror(self, t, v, tb): + watcher_ptr = tb.tb_frame.f_locals['watcher_ptr'] if tb is not None else None + if watcher_ptr: + loop = self._find_loop_from_c_watcher(watcher_ptr) + if loop is not None: + # None as the context argument causes the exception to be raised + # in the main greenlet. + loop.handle_error(None, t, v, tb) + return None + raise v # Let CFFI print + + def _find_loop_from_c_watcher(self, watcher_ptr): + raise NotImplementedError() + + + +def assign_standard_callbacks(ffi, lib, callbacks_class, extras=()): # pylint:disable=unused-argument + # callbacks keeps these cdata objects alive at the python level + callbacks = callbacks_class(ffi) + extras = tuple([(getattr(callbacks, name), error) for name, error in extras]) + for (func, error_func) in ((callbacks.python_callback, None), + (callbacks.python_handle_error, None), + (callbacks.python_stop, None), + (callbacks.python_check_callback, + callbacks.check_callback_onerror), + (callbacks.python_prepare_callback, + callbacks.check_callback_onerror)) + extras: + # The name of the callback function matches the 'extern Python' declaration. + error_func = error_func or callbacks.unhandled_onerror + callback = ffi.def_extern(onerror=error_func)(func) + # keep alive the cdata + # (def_extern returns the original function, and it requests that + # the function be "global", so maybe it keeps a hard reference to it somewhere now + # unlike ffi.callback(), and we don't need to do this?) + callbacks.callbacks.append(callback) + + # At this point, the library C variable (static function, actually) + # is filled in. + + return callbacks + + +if sys.version_info[0] >= 3: + basestring = (bytes, str) + integer_types = (int,) +else: + import __builtin__ # pylint:disable=import-error + basestring = (__builtin__.basestring,) + integer_types = (int, __builtin__.long) + + + + +_NOARGS = () + +CALLBACK_CHECK_COUNT = 50 + +class AbstractLoop(object): + # pylint:disable=too-many-public-methods,too-many-instance-attributes + + error_handler = None + + _CHECK_POINTER = None + + _TIMER_POINTER = None + _TIMER_CALLBACK_SIG = None + + _PREPARE_POINTER = None + + starting_timer_may_update_loop_time = False + + # Subclasses should set this in __init__ to reflect + # whether they were the default loop. + _default = None + + def __init__(self, ffi, lib, watchers, flags=None, default=None): + self._ffi = ffi + self._lib = lib + self._ptr = None + self._handle_to_self = self._ffi.new_handle(self) # XXX: Reference cycle? + self._watchers = watchers + self._in_callback = False + self._callbacks = deque() + # Stores python watcher objects while they are started + self._keepaliveset = set() + self._init_loop_and_aux_watchers(flags, default) + + + def _init_loop_and_aux_watchers(self, flags=None, default=None): + + self._ptr = self._init_loop(flags, default) + + + # self._check is a watcher that runs in each iteration of the + # mainloop, just after the blocking call. It's point is to handle + # signals. It doesn't run watchers or callbacks, it just exists to give + # CFFI a chance to raise signal exceptions so we can handle them. + self._check = self._ffi.new(self._CHECK_POINTER) + self._check.data = self._handle_to_self + self._init_and_start_check() + + # self._prepare is a watcher that runs in each iteration of the mainloop, + # just before the blocking call. It's where we run deferred callbacks + # from self.run_callback. This cooperates with _setup_for_run_callback() + # to schedule self._timer0 if needed. + self._prepare = self._ffi.new(self._PREPARE_POINTER) + self._prepare.data = self._handle_to_self + self._init_and_start_prepare() + + # A timer we start and stop on demand. If we have callbacks, + # too many to run in one iteration of _run_callbacks, we turn this + # on so as to have the next iteration of the run loop return to us + # as quickly as possible. + # TODO: There may be a more efficient way to do this using ev_timer_again; + # see the "ev_timer" section of the ev manpage (http://linux.die.net/man/3/ev) + # Alternatively, setting the ev maximum block time may also work. + self._timer0 = self._ffi.new(self._TIMER_POINTER) + self._timer0.data = self._handle_to_self + self._init_callback_timer() + + # TODO: We may be able to do something nicer and use the existing python_callback + # combined with onerror and the class check/timer/prepare to simplify things + # and unify our handling + + def _init_loop(self, flags, default): + """ + Called by __init__ to create or find the loop. The return value + is assigned to self._ptr. + """ + raise NotImplementedError() + + def _init_and_start_check(self): + raise NotImplementedError() + + def _init_and_start_prepare(self): + raise NotImplementedError() + + def _init_callback_timer(self): + raise NotImplementedError() + + def _stop_callback_timer(self): + raise NotImplementedError() + + def _start_callback_timer(self): + raise NotImplementedError() + + def _check_callback_handle_error(self, t, v, tb): + self.handle_error(None, t, v, tb) + + def _run_callbacks(self): # pylint:disable=too-many-branches + # When we're running callbacks, its safe for timers to + # update the notion of the current time (because if we're here, + # we're not running in a timer callback that may let other timers + # run; this is mostly an issue for libuv). + + # That's actually a bit of a lie: on libev, self._timer0 really is + # a timer, and so sometimes this is running in a timer callback, not + # a prepare callback. But that's OK, libev doesn't suffer from cascading + # timer expiration and its safe to update the loop time at any + # moment there. + self.starting_timer_may_update_loop_time = True + try: + count = CALLBACK_CHECK_COUNT + now = self.now() + expiration = now + getswitchinterval() + self._stop_callback_timer() + while self._callbacks: + cb = self._callbacks.popleft() # pylint:disable=assignment-from-no-return + count -= 1 + self.unref() # XXX: libuv doesn't have a global ref count! + callback = cb.callback + cb.callback = None + args = cb.args + if callback is None or args is None: + # it's been stopped + continue + + try: + callback(*args) + except: # pylint:disable=bare-except + # If we allow an exception to escape this method (while we are running the ev callback), + # then CFFI will print the error and libev will continue executing. + # There are two problems with this. The first is that the code after + # the loop won't run. The second is that any remaining callbacks scheduled + # for this loop iteration will be silently dropped; they won't run, but they'll + # also not be *stopped* (which is not a huge deal unless you're looking for + # consistency or checking the boolean/pending status; the loop doesn't keep + # a reference to them like it does to watchers...*UNLESS* the callback itself had + # a reference to a watcher; then I don't know what would happen, it depends on + # the state of the watcher---a leak or crash is not totally inconceivable). + # The Cython implementation in core.ppyx uses gevent_call from callbacks.c + # to run the callback, which uses gevent_handle_error to handle any errors the + # Python callback raises...it unconditionally simply prints any error raised + # by loop.handle_error and clears it, so callback handling continues. + # We take a similar approach (but are extra careful about printing) + try: + self.handle_error(cb, *sys.exc_info()) + except: # pylint:disable=bare-except + try: + print("Exception while handling another error", file=sys.stderr) + traceback.print_exc() + except: # pylint:disable=bare-except + pass # Nothing we can do here + finally: + # NOTE: this must be reset here, because cb.args is used as a flag in + # the callback class so that bool(cb) of a callback that has been run + # becomes False + cb.args = None + + # We've finished running one group of callbacks + # but we may have more, so before looping check our + # switch interval. + if count == 0 and self._callbacks: + count = CALLBACK_CHECK_COUNT + self.update_now() + if self.now() >= expiration: + now = 0 + break + + # Update the time before we start going again, if we didn't + # just do so. + if now != 0: + self.update_now() + + if self._callbacks: + self._start_callback_timer() + finally: + self.starting_timer_may_update_loop_time = False + + def _stop_aux_watchers(self): + raise NotImplementedError() + + def destroy(self): + if self._ptr: + try: + if not self._can_destroy_loop(self._ptr): + return False + self._stop_aux_watchers() + self._destroy_loop(self._ptr) + finally: + # not ffi.NULL, we don't want something that can be + # passed to C and crash later. This will create nice friendly + # TypeError from CFFI. + self._ptr = None + del self._handle_to_self + del self._callbacks + del self._keepaliveset + + return True + + def _can_destroy_loop(self, ptr): + raise NotImplementedError() + + def _destroy_loop(self, ptr): + raise NotImplementedError() + + @property + def ptr(self): + return self._ptr + + @property + def WatcherType(self): + return self._watchers.watcher + + @property + def MAXPRI(self): + return 1 + + @property + def MINPRI(self): + return 1 + + def _handle_syserr(self, message, errno): + try: + errno = os.strerror(errno) + except: # pylint:disable=bare-except + traceback.print_exc() + try: + message = '%s: %s' % (message, errno) + except: # pylint:disable=bare-except + traceback.print_exc() + self.handle_error(None, SystemError, SystemError(message), None) + + def handle_error(self, context, type, value, tb): + handle_error = None + error_handler = self.error_handler + if error_handler is not None: + # we do want to do getattr every time so that setting Hub.handle_error property just works + handle_error = getattr(error_handler, 'handle_error', error_handler) + handle_error(context, type, value, tb) + else: + self._default_handle_error(context, type, value, tb) + + def _default_handle_error(self, context, type, value, tb): # pylint:disable=unused-argument + # note: Hub sets its own error handler so this is not used by gevent + # this is here to make core.loop usable without the rest of gevent + # Should cause the loop to stop running. + traceback.print_exception(type, value, tb) + + + def run(self, nowait=False, once=False): + raise NotImplementedError() + + def reinit(self): + raise NotImplementedError() + + def ref(self): + # XXX: libuv doesn't do it this way + raise NotImplementedError() + + def unref(self): + raise NotImplementedError() + + def break_(self, how=None): + raise NotImplementedError() + + def verify(self): + pass + + def now(self): + raise NotImplementedError() + + def update_now(self): + raise NotImplementedError() + + def update(self): + import warnings + warnings.warn("'update' is deprecated; use 'update_now'", + DeprecationWarning, + stacklevel=2) + self.update_now() + + def __repr__(self): + return '<%s at 0x%x %s>' % (self.__class__.__name__, id(self), self._format()) + + @property + def default(self): + return self._default if self._ptr else False + + @property + def iteration(self): + return -1 + + @property + def depth(self): + return -1 + + @property + def backend_int(self): + return 0 + + @property + def backend(self): + return "default" + + @property + def pendingcnt(self): + return 0 + + def io(self, fd, events, ref=True, priority=None): + return self._watchers.io(self, fd, events, ref, priority) + + def timer(self, after, repeat=0.0, ref=True, priority=None): + return self._watchers.timer(self, after, repeat, ref, priority) + + def signal(self, signum, ref=True, priority=None): + return self._watchers.signal(self, signum, ref, priority) + + def idle(self, ref=True, priority=None): + return self._watchers.idle(self, ref, priority) + + def prepare(self, ref=True, priority=None): + return self._watchers.prepare(self, ref, priority) + + def check(self, ref=True, priority=None): + return self._watchers.check(self, ref, priority) + + def fork(self, ref=True, priority=None): + return self._watchers.fork(self, ref, priority) + + def async_(self, ref=True, priority=None): + return self._watchers.async_(self, ref, priority) + + # Provide BWC for those that can use 'async' as is + locals()['async'] = async_ + + if sys.platform != "win32": + + def child(self, pid, trace=0, ref=True): + return self._watchers.child(self, pid, trace, ref) + + def install_sigchld(self): + pass + + def stat(self, path, interval=0.0, ref=True, priority=None): + return self._watchers.stat(self, path, interval, ref, priority) + + def callback(self, priority=None): + return callback(self, priority) + + def _setup_for_run_callback(self): + raise NotImplementedError() + + def run_callback(self, func, *args): + # If we happen to already be running callbacks (inside + # _run_callbacks), this could happen almost immediately, + # without the loop cycling. + cb = callback(func, args) + self._callbacks.append(cb) + self._setup_for_run_callback() + + return cb + + def _format(self): + if not self._ptr: + return 'destroyed' + msg = self.backend + if self.default: + msg += ' default' + msg += ' pending=%s' % self.pendingcnt + msg += self._format_details() + return msg + + def _format_details(self): + msg = '' + fileno = self.fileno() # pylint:disable=assignment-from-none + try: + activecnt = self.activecnt + except AttributeError: + activecnt = None + if activecnt is not None: + msg += ' ref=' + repr(activecnt) + if fileno is not None: + msg += ' fileno=' + repr(fileno) + #if sigfd is not None and sigfd != -1: + # msg += ' sigfd=' + repr(sigfd) + return msg + + def fileno(self): + return None + + @property + def activecnt(self): + if not self._ptr: + raise ValueError('operation on destroyed loop') + return 0 diff --git a/src/gevent/_ffi/watcher.py b/src/gevent/_ffi/watcher.py new file mode 100644 index 0000000..3f880ce --- /dev/null +++ b/src/gevent/_ffi/watcher.py @@ -0,0 +1,641 @@ +""" +Useful base classes for watchers. The available +watchers will depend on the specific event loop. +""" +# pylint:disable=not-callable +from __future__ import absolute_import, print_function + +import signal as signalmodule +import functools +import warnings + +from gevent._config import config + +try: + from tracemalloc import get_object_traceback + + def tracemalloc(init): + # PYTHONTRACEMALLOC env var controls this on Python 3. + return init +except ImportError: # Python < 3.4 + + if config.trace_malloc: + # Use the same env var to turn this on for Python 2 + import traceback + + class _TB(object): + __slots__ = ('lines',) + + def __init__(self, lines): + # These end in newlines, which we don't want for consistency + self.lines = [x.rstrip() for x in lines] + + def format(self): + return self.lines + + def tracemalloc(init): + @functools.wraps(init) + def traces(self, *args, **kwargs): + init(self, *args, **kwargs) + self._captured_malloc = _TB(traceback.format_stack()) + return traces + + def get_object_traceback(obj): + return obj._captured_malloc + + else: + def get_object_traceback(_obj): + return None + + def tracemalloc(init): + return init + +from gevent._compat import fsencode + +from gevent._ffi import _dbg # pylint:disable=unused-import +from gevent._ffi import GEVENT_DEBUG_LEVEL +from gevent._ffi import DEBUG +from gevent._ffi.loop import GEVENT_CORE_EVENTS +from gevent._ffi.loop import _NOARGS + +ALLOW_WATCHER_DEL = GEVENT_DEBUG_LEVEL >= DEBUG + +__all__ = [ + +] + +try: + ResourceWarning +except NameError: + class ResourceWarning(Warning): + "Python 2 fallback" + +class _NoWatcherResult(int): + + def __repr__(self): + return "" + +_NoWatcherResult = _NoWatcherResult(0) + +def events_to_str(event_field, all_events): + result = [] + for (flag, string) in all_events: + c_flag = flag + if event_field & c_flag: + result.append(string) + event_field = event_field & (~c_flag) + if not event_field: + break + if event_field: + result.append(hex(event_field)) + return '|'.join(result) + + +def not_while_active(func): + @functools.wraps(func) + def nw(self, *args, **kwargs): + if self.active: + raise ValueError("not while active") + func(self, *args, **kwargs) + return nw + +def only_if_watcher(func): + @functools.wraps(func) + def if_w(self): + if self._watcher: + return func(self) + return _NoWatcherResult + return if_w + + +class LazyOnClass(object): + + @classmethod + def lazy(cls, cls_dict, func): + "Put a LazyOnClass object in *cls_dict* with the same name as *func*" + cls_dict[func.__name__] = cls(func) + + def __init__(self, func, name=None): + self.name = name or func.__name__ + self.func = func + + def __get__(self, inst, klass): + if inst is None: # pragma: no cover + return self + + val = self.func(inst) + setattr(klass, self.name, val) + return val + + +class AbstractWatcherType(type): + """ + Base metaclass for watchers. + + To use, you will: + + - subclass the watcher class defined from this type. + - optionally subclass this type + """ + # pylint:disable=bad-mcs-classmethod-argument + + _FFI = None + _LIB = None + + def __new__(cls, name, bases, cls_dict): + if name != 'watcher' and not cls_dict.get('_watcher_skip_ffi'): + cls._fill_watcher(name, bases, cls_dict) + if '__del__' in cls_dict and not ALLOW_WATCHER_DEL: # pragma: no cover + raise TypeError("CFFI watchers are not allowed to have __del__") + return type.__new__(cls, name, bases, cls_dict) + + @classmethod + def _fill_watcher(cls, name, bases, cls_dict): + # TODO: refactor smaller + # pylint:disable=too-many-locals + if name.endswith('_'): + # Strip trailing _ added to avoid keyword duplications + # e.g., async_ + name = name[:-1] + + def _mro_get(attr, bases, error=True): + for b in bases: + try: + return getattr(b, attr) + except AttributeError: + continue + if error: # pragma: no cover + raise AttributeError(attr) + _watcher_prefix = cls_dict.get('_watcher_prefix') or _mro_get('_watcher_prefix', bases) + + if '_watcher_type' not in cls_dict: + watcher_type = _watcher_prefix + '_' + name + cls_dict['_watcher_type'] = watcher_type + elif not cls_dict['_watcher_type'].startswith(_watcher_prefix): + watcher_type = _watcher_prefix + '_' + cls_dict['_watcher_type'] + cls_dict['_watcher_type'] = watcher_type + + active_name = _watcher_prefix + '_is_active' + + def _watcher_is_active(self): + return getattr(self._LIB, active_name) + + LazyOnClass.lazy(cls_dict, _watcher_is_active) + + watcher_struct_name = cls_dict.get('_watcher_struct_name') + if not watcher_struct_name: + watcher_struct_pattern = (cls_dict.get('_watcher_struct_pattern') + or _mro_get('_watcher_struct_pattern', bases, False) + or 'struct %s') + watcher_struct_name = watcher_struct_pattern % (watcher_type,) + + def _watcher_struct_pointer_type(self): + return self._FFI.typeof(watcher_struct_name + ' *') + + LazyOnClass.lazy(cls_dict, _watcher_struct_pointer_type) + + callback_name = (cls_dict.get('_watcher_callback_name') + or _mro_get('_watcher_callback_name', bases, False) + or '_gevent_generic_callback') + + def _watcher_callback(self): + return self._FFI.addressof(self._LIB, callback_name) + + LazyOnClass.lazy(cls_dict, _watcher_callback) + + def _make_meth(name, watcher_name): + def meth(self): + lib_name = self._watcher_type + '_' + name + return getattr(self._LIB, lib_name) + meth.__name__ = watcher_name + return meth + + for meth_name in 'start', 'stop', 'init': + watcher_name = '_watcher' + '_' + meth_name + if watcher_name not in cls_dict: + LazyOnClass.lazy(cls_dict, _make_meth(meth_name, watcher_name)) + + def new_handle(cls, obj): + return cls._FFI.new_handle(obj) + + def new(cls, kind): + return cls._FFI.new(kind) + +class watcher(object): + + _callback = None + _args = None + _watcher = None + # self._handle has a reference to self, keeping it alive. + # We must keep self._handle alive for ffi.from_handle() to be + # able to work. We only fill this in when we are started, + # and when we are stopped we destroy it. + # NOTE: This is a GC cycle, so we keep it around for as short + # as possible. + _handle = None + + @tracemalloc + def __init__(self, _loop, ref=True, priority=None, args=_NOARGS): + self.loop = _loop + self.__init_priority = priority + self.__init_args = args + self.__init_ref = ref + self._watcher_full_init() + + + def _watcher_full_init(self): + priority = self.__init_priority + ref = self.__init_ref + args = self.__init_args + + self._watcher_create(ref) + + if priority is not None: + self._watcher_ffi_set_priority(priority) + + try: + self._watcher_ffi_init(args) + except: + # Let these be GC'd immediately. + # If we keep them around to when *we* are gc'd, + # they're probably invalid, meaning any native calls + # we do then to close() them are likely to fail + self._watcher = None + raise + self._watcher_ffi_set_init_ref(ref) + + @classmethod + def _watcher_ffi_close(cls, ffi_watcher): + pass + + def _watcher_create(self, ref): # pylint:disable=unused-argument + self._watcher = self._watcher_new() + + def _watcher_new(self): + return type(self).new(self._watcher_struct_pointer_type) # pylint:disable=no-member + + def _watcher_ffi_set_init_ref(self, ref): + pass + + def _watcher_ffi_set_priority(self, priority): + pass + + def _watcher_ffi_init(self, args): + raise NotImplementedError() + + def _watcher_ffi_start(self): + raise NotImplementedError() + + def _watcher_ffi_stop(self): + self._watcher_stop(self.loop._ptr, self._watcher) + + def _watcher_ffi_ref(self): + raise NotImplementedError() + + def _watcher_ffi_unref(self): + raise NotImplementedError() + + def _watcher_ffi_start_unref(self): + # While a watcher is active, we don't keep it + # referenced. This allows a timer, for example, to be started, + # and still allow the loop to end if there is nothing + # else to do. see test__order.TestSleep0 for one example. + self._watcher_ffi_unref() + + def _watcher_ffi_stop_ref(self): + self._watcher_ffi_ref() + + # A string identifying the type of libev object we watch, e.g., 'ev_io' + # This should be a class attribute. + _watcher_type = None + # A class attribute that is the callback on the libev object that init's the C struct, + # e.g., libev.ev_io_init. If None, will be set by _init_subclasses. + _watcher_init = None + # A class attribute that is the callback on the libev object that starts the C watcher, + # e.g., libev.ev_io_start. If None, will be set by _init_subclasses. + _watcher_start = None + # A class attribute that is the callback on the libev object that stops the C watcher, + # e.g., libev.ev_io_stop. If None, will be set by _init_subclasses. + _watcher_stop = None + # A cffi ctype object identifying the struct pointer we create. + # This is a class attribute set based on the _watcher_type + _watcher_struct_pointer_type = None + # The attribute of the libev object identifying the custom + # callback function for this type of watcher. This is a class + # attribute set based on the _watcher_type in _init_subclasses. + _watcher_callback = None + _watcher_is_active = None + + def close(self): + if self._watcher is None: + return + + self.stop() + _watcher = self._watcher + self._watcher = None + self._watcher_set_data(_watcher, self._FFI.NULL) # pylint: disable=no-member + self._watcher_ffi_close(_watcher) + self.loop = None + + def _watcher_set_data(self, the_watcher, data): + # This abstraction exists for the sole benefit of + # libuv.watcher.stat, which "subclasses" uv_handle_t. + # Can we do something to avoid this extra function call? + the_watcher.data = data + return data + + def __enter__(self): + return self + + def __exit__(self, t, v, tb): + self.close() + + if ALLOW_WATCHER_DEL: + def __del__(self): + if self._watcher: + tb = get_object_traceback(self) + tb_msg = '' + if tb is not None: + tb_msg = '\n'.join(tb.format()) + tb_msg = '\nTraceback:\n' + tb_msg + warnings.warn("Failed to close watcher %r%s" % (self, tb_msg), + ResourceWarning) + + # may fail if __init__ did; will be harmlessly printed + self.close() + + + def __repr__(self): + formats = self._format() + result = "<%s at 0x%x%s" % (self.__class__.__name__, id(self), formats) + if self.pending: + result += " pending" + if self.callback is not None: + fself = getattr(self.callback, '__self__', None) + if fself is self: + result += " callback=" % (self.callback.__name__) + else: + result += " callback=%r" % (self.callback, ) + if self.args is not None: + result += " args=%r" % (self.args, ) + if self.callback is None and self.args is None: + result += " stopped" + result += " watcher=%s" % (self._watcher) + result += " handle=%s" % (self._watcher_handle) + result += " ref=%s" % (self.ref) + return result + ">" + + @property + def _watcher_handle(self): + if self._watcher: + return self._watcher.data + + def _format(self): + return '' + + @property + def ref(self): + raise NotImplementedError() + + def _get_callback(self): + return self._callback + + def _set_callback(self, cb): + if not callable(cb) and cb is not None: + raise TypeError("Expected callable, not %r" % (cb, )) + if cb is None: + if '_callback' in self.__dict__: + del self._callback + else: + self._callback = cb + callback = property(_get_callback, _set_callback) + + def _get_args(self): + return self._args + + def _set_args(self, args): + if not isinstance(args, tuple) and args is not None: + raise TypeError("args must be a tuple or None") + if args is None: + if '_args' in self.__dict__: + del self._args + else: + self._args = args + + args = property(_get_args, _set_args) + + def start(self, callback, *args): + if callback is None: + raise TypeError('callback must be callable, not None') + self.callback = callback + self.args = args or _NOARGS + self.loop._keepaliveset.add(self) + self._handle = self._watcher_set_data(self._watcher, type(self).new_handle(self)) # pylint:disable=no-member + self._watcher_ffi_start() + self._watcher_ffi_start_unref() + + def stop(self): + if self._callback is None: + assert self.loop is None or self not in self.loop._keepaliveset + return + self._watcher_ffi_stop_ref() + self._watcher_ffi_stop() + self.loop._keepaliveset.discard(self) + self._handle = None + self._watcher_set_data(self._watcher, self._FFI.NULL) # pylint:disable=no-member + self.callback = None + self.args = None + + def _get_priority(self): + return None + + @not_while_active + def _set_priority(self, priority): + pass + + priority = property(_get_priority, _set_priority) + + + @property + def active(self): + if self._watcher is not None and self._watcher_is_active(self._watcher): + return True + return False + + @property + def pending(self): + return False + +watcher = AbstractWatcherType('watcher', (object,), dict(watcher.__dict__)) + +class IoMixin(object): + + EVENT_MASK = 0 + + def __init__(self, loop, fd, events, ref=True, priority=None, _args=None): + # Win32 only works with sockets, and only when we use libuv, because + # we don't use _open_osfhandle. See libuv/watchers.py:io for a description. + if fd < 0: + raise ValueError('fd must be non-negative: %r' % fd) + if events & ~self.EVENT_MASK: + raise ValueError('illegal event mask: %r' % events) + self._fd = fd + super(IoMixin, self).__init__(loop, ref=ref, priority=priority, + args=_args or (fd, events)) + + def start(self, callback, *args, **kwargs): + args = args or _NOARGS + if kwargs.get('pass_events'): + args = (GEVENT_CORE_EVENTS, ) + args + super(IoMixin, self).start(callback, *args) + + def _format(self): + return ' fd=%d' % self._fd + +class TimerMixin(object): + _watcher_type = 'timer' + + def __init__(self, loop, after=0.0, repeat=0.0, ref=True, priority=None): + if repeat < 0.0: + raise ValueError("repeat must be positive or zero: %r" % repeat) + self._after = after + self._repeat = repeat + super(TimerMixin, self).__init__(loop, ref=ref, priority=priority, args=(after, repeat)) + + def start(self, callback, *args, **kw): + update = kw.get("update", self.loop.starting_timer_may_update_loop_time) + if update: + # Quoth the libev doc: "This is a costly operation and is + # usually done automatically within ev_run(). This + # function is rarely useful, but when some event callback + # runs for a very long time without entering the event + # loop, updating libev's idea of the current time is a + # good idea." + + # 1.3 changed the default for this to False *unless* the loop is + # running a callback; see libuv for details. Note that + # starting Timeout objects still sets this to true. + + self.loop.update_now() + super(TimerMixin, self).start(callback, *args) + + def again(self, callback, *args, **kw): + raise NotImplementedError() + + +class SignalMixin(object): + _watcher_type = 'signal' + + def __init__(self, loop, signalnum, ref=True, priority=None): + if signalnum < 1 or signalnum >= signalmodule.NSIG: + raise ValueError('illegal signal number: %r' % signalnum) + # still possible to crash on one of libev's asserts: + # 1) "libev: ev_signal_start called with illegal signal number" + # EV_NSIG might be different from signal.NSIG on some platforms + # 2) "libev: a signal must not be attached to two different loops" + # we probably could check that in LIBEV_EMBED mode, but not in general + self._signalnum = signalnum + super(SignalMixin, self).__init__(loop, ref=ref, priority=priority, args=(signalnum, )) + + +class IdleMixin(object): + _watcher_type = 'idle' + + +class PrepareMixin(object): + _watcher_type = 'prepare' + + +class CheckMixin(object): + _watcher_type = 'check' + + +class ForkMixin(object): + _watcher_type = 'fork' + + +class AsyncMixin(object): + _watcher_type = 'async' + + def send(self): + raise NotImplementedError() + + @property + def pending(self): + raise NotImplementedError() + + +class ChildMixin(object): + + # hack for libuv which doesn't extend watcher + _CALL_SUPER_INIT = True + + def __init__(self, loop, pid, trace=0, ref=True): + if not loop.default: + raise TypeError('child watchers are only available on the default loop') + loop.install_sigchld() + self._pid = pid + if self._CALL_SUPER_INIT: + super(ChildMixin, self).__init__(loop, ref=ref, args=(pid, trace)) + + def _format(self): + return ' pid=%r rstatus=%r' % (self.pid, self.rstatus) + + @property + def pid(self): + return self._pid + + @property + def rpid(self): + # The received pid, the result of the waitpid() call. + return self._rpid + + _rpid = None + _rstatus = 0 + + @property + def rstatus(self): + return self._rstatus + +class StatMixin(object): + + @staticmethod + def _encode_path(path): + return fsencode(path) + + def __init__(self, _loop, path, interval=0.0, ref=True, priority=None): + # Store the encoded path in the same attribute that corecext does + self._paths = self._encode_path(path) + + # Keep the original path to avoid re-encoding, especially on Python 3 + self._path = path + + # Although CFFI would automatically convert a bytes object into a char* when + # calling ev_stat_init(..., char*, ...), on PyPy the char* pointer is not + # guaranteed to live past the function call. On CPython, only with a constant/interned + # bytes object is the pointer guaranteed to last path the function call. (And since + # Python 3 is pretty much guaranteed to produce a newly-encoded bytes object above, thats + # rarely the case). Therefore, we must keep a reference to the produced cdata object + # so that the struct ev_stat_watcher's `path` pointer doesn't become invalid/deallocated + self._cpath = self._FFI.new('char[]', self._paths) + + self._interval = interval + super(StatMixin, self).__init__(_loop, ref=ref, priority=priority, + args=(self._cpath, + interval)) + + @property + def path(self): + return self._path + + @property + def attr(self): + raise NotImplementedError + + @property + def prev(self): + raise NotImplementedError + + @property + def interval(self): + return self._interval diff --git a/src/gevent/_fileobjectcommon.py b/src/gevent/_fileobjectcommon.py new file mode 100644 index 0000000..99404f3 --- /dev/null +++ b/src/gevent/_fileobjectcommon.py @@ -0,0 +1,281 @@ +from __future__ import absolute_import, print_function, division + +try: + from errno import EBADF +except ImportError: + EBADF = 9 + +import os +from io import TextIOWrapper +import functools +import sys + + +from gevent.hub import _get_hub_noargs as get_hub +from gevent._compat import integer_types +from gevent._compat import reraise +from gevent.lock import Semaphore, DummySemaphore + +class cancel_wait_ex(IOError): + + def __init__(self): + super(cancel_wait_ex, self).__init__( + EBADF, 'File descriptor was closed in another greenlet') + + +class FileObjectClosed(IOError): + + def __init__(self): + super(FileObjectClosed, self).__init__( + EBADF, 'Bad file descriptor (FileObject was closed)') + +class FileObjectBase(object): + """ + Internal base class to ensure a level of consistency + between FileObjectPosix and FileObjectThread + """ + + # List of methods we delegate to the wrapping IO object, if they + # implement them and we do not. + _delegate_methods = ( + # General methods + 'flush', + 'fileno', + 'writable', + 'readable', + 'seek', + 'seekable', + 'tell', + + # Read + 'read', + 'readline', + 'readlines', + 'read1', + + # Write + 'write', + 'writelines', + 'truncate', + ) + + + # Whether we are translating universal newlines or not. + _translate = False + + _translate_encoding = None + _translate_errors = None + + def __init__(self, io, closefd): + """ + :param io: An io.IOBase-like object. + """ + self._io = io + # We don't actually use this property ourself, but we save it (and + # pass it along) for compatibility. + self._close = closefd + + if self._translate: + # This automatically handles delegation by assigning to + # self.io + self.translate_newlines(None, self._translate_encoding, self._translate_errors) + else: + self._do_delegate_methods() + + + io = property(lambda s: s._io, + # Historically we either hand-wrote all the delegation methods + # to use self.io, or we simply used __getattr__ to look them up at + # runtime. This meant people could change the io attribute on the fly + # and it would mostly work (subprocess.py used to do that). We don't recommend + # that, but we still support it. + lambda s, nv: setattr(s, '_io', nv) or s._do_delegate_methods()) + + def _do_delegate_methods(self): + for meth_name in self._delegate_methods: + meth = getattr(self._io, meth_name, None) + implemented_by_class = hasattr(type(self), meth_name) + if meth and not implemented_by_class: + setattr(self, meth_name, self._wrap_method(meth)) + elif hasattr(self, meth_name) and not implemented_by_class: + delattr(self, meth_name) + + def _wrap_method(self, method): + """ + Wrap a method we're copying into our dictionary from the underlying + io object to do something special or different, if necessary. + """ + return method + + def translate_newlines(self, mode, *text_args, **text_kwargs): + wrapper = TextIOWrapper(self._io, *text_args, **text_kwargs) + if mode: + wrapper.mode = mode + self.io = wrapper + self._translate = True + + @property + def closed(self): + """True if the file is closed""" + return self._io is None + + def close(self): + if self._io is None: + return + + io = self._io + self._io = None + self._do_close(io, self._close) + + def _do_close(self, fobj, closefd): + raise NotImplementedError() + + def __getattr__(self, name): + if self._io is None: + raise FileObjectClosed() + return getattr(self._io, name) + + def __repr__(self): + return '<%s _fobj=%r%s>' % (self.__class__.__name__, self.io, self._extra_repr()) + + def _extra_repr(self): + return '' + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + +class FileObjectBlock(FileObjectBase): + + def __init__(self, fobj, *args, **kwargs): + closefd = kwargs.pop('close', True) + if kwargs: + raise TypeError('Unexpected arguments: %r' % kwargs.keys()) + if isinstance(fobj, integer_types): + if not closefd: + # we cannot do this, since fdopen object will close the descriptor + raise TypeError('FileObjectBlock does not support close=False on an fd.') + fobj = os.fdopen(fobj, *args) + super(FileObjectBlock, self).__init__(fobj, closefd) + + def _do_close(self, fobj, closefd): + fobj.close() + +class FileObjectThread(FileObjectBase): + """ + A file-like object wrapping another file-like object, performing all blocking + operations on that object in a background thread. + + .. caution:: + Attempting to change the threadpool or lock of an existing FileObjectThread + has undefined consequences. + + .. versionchanged:: 1.1b1 + The file object is closed using the threadpool. Note that whether or + not this action is synchronous or asynchronous is not documented. + + """ + + def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True): + """ + :param fobj: The underlying file-like object to wrap, or an integer fileno + that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*. + :keyword bool lock: If True (the default) then all operations will + be performed one-by-one. Note that this does not guarantee that, if using + this file object from multiple threads/greenlets, operations will be performed + in any particular order, only that no two operations will be attempted at the + same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize + file operations with an external resource. + :keyword bool close: If True (the default) then when this object is closed, + the underlying object is closed as well. + """ + closefd = close + self.threadpool = threadpool or get_hub().threadpool + self.lock = lock + if self.lock is True: + self.lock = Semaphore() + elif not self.lock: + self.lock = DummySemaphore() + if not hasattr(self.lock, '__enter__'): + raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock)) + if isinstance(fobj, integer_types): + if not closefd: + # we cannot do this, since fdopen object will close the descriptor + raise TypeError('FileObjectThread does not support close=False on an fd.') + if mode is None: + assert bufsize == -1, "If you use the default mode, you can't choose a bufsize" + fobj = os.fdopen(fobj) + else: + fobj = os.fdopen(fobj, mode, bufsize) + + self.__io_holder = [fobj] # signal for _wrap_method + super(FileObjectThread, self).__init__(fobj, closefd) + + def _do_close(self, fobj, closefd): + self.__io_holder[0] = None # for _wrap_method + try: + with self.lock: + self.threadpool.apply(fobj.flush) + finally: + if closefd: + # Note that we're not taking the lock; older code + # did fobj.close() without going through the threadpool at all, + # so acquiring the lock could potentially introduce deadlocks + # that weren't present before. Avoiding the lock doesn't make + # the existing race condition any worse. + # We wrap the close in an exception handler and re-raise directly + # to avoid the (common, expected) IOError from being logged by the pool + def close(_fobj=fobj): + try: + _fobj.close() + except: # pylint:disable=bare-except + return sys.exc_info() + finally: + _fobj = None + del fobj + + exc_info = self.threadpool.apply(close) + del close + + if exc_info: + reraise(*exc_info) + + def _do_delegate_methods(self): + super(FileObjectThread, self)._do_delegate_methods() + if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''): + self.read1 = self.read + self.__io_holder[0] = self._io + + def _extra_repr(self): + return ' threadpool=%r' % (self.threadpool,) + + def __iter__(self): + return self + + def next(self): + line = self.readline() + if line: + return line + raise StopIteration + __next__ = next + + def _wrap_method(self, method): + # NOTE: We are careful to avoid introducing a refcycle + # within self. Our wrapper cannot refer to self. + io_holder = self.__io_holder + lock = self.lock + threadpool = self.threadpool + + @functools.wraps(method) + def thread_method(*args, **kwargs): + if io_holder[0] is None: + # This is different than FileObjectPosix, etc, + # because we want to save the expensive trip through + # the threadpool. + raise FileObjectClosed() + with lock: + return threadpool.apply(method, args, kwargs) + + return thread_method diff --git a/src/gevent/_fileobjectposix.py b/src/gevent/_fileobjectposix.py new file mode 100644 index 0000000..699743c --- /dev/null +++ b/src/gevent/_fileobjectposix.py @@ -0,0 +1,357 @@ +from __future__ import absolute_import +import os +import sys +import io +from io import BufferedReader +from io import BufferedWriter +from io import BytesIO +from io import DEFAULT_BUFFER_SIZE +from io import RawIOBase +from io import UnsupportedOperation + +from gevent._compat import reraise +from gevent._fileobjectcommon import cancel_wait_ex +from gevent._fileobjectcommon import FileObjectBase +from gevent.hub import get_hub +from gevent.os import _read +from gevent.os import _write +from gevent.os import ignored_errors +from gevent.os import make_nonblocking + + +class GreenFileDescriptorIO(RawIOBase): + + # Note that RawIOBase has a __del__ method that calls + # self.close(). (In C implementations like CPython, this is + # the type's tp_dealloc slot; prior to Python 3, the object doesn't + # appear to have a __del__ method, even though it functionally does) + + _read_event = None + _write_event = None + _closed = False + _seekable = None + + def __init__(self, fileno, mode='r', closefd=True): + RawIOBase.__init__(self) # Python 2: pylint:disable=no-member,non-parent-init-called + + self._closefd = closefd + self._fileno = fileno + make_nonblocking(fileno) + readable = 'r' in mode + writable = 'w' in mode + + self.hub = get_hub() + io_watcher = self.hub.loop.io + try: + if readable: + self._read_event = io_watcher(fileno, 1) + + if writable: + self._write_event = io_watcher(fileno, 2) + except: + # If anything goes wrong, it's important to go ahead and + # close these watchers *now*, especially under libuv, so + # that they don't get eventually reclaimed by the garbage + # collector at some random time, thanks to the C level + # slot (even though we don't seem to have any actual references + # at the Python level). Previously, if we didn't close now, + # that random close in the future would cause issues if we had duplicated + # the fileno (if a wrapping with statement had closed an open fileobject, + # for example) + + # test__fileobject can show a failure if this doesn't happen + # TRAVIS=true GEVENT_LOOP=libuv python -m gevent.tests.test__fileobject \ + # TestFileObjectPosix.test_seek TestFileObjectThread.test_bufsize_0 + self.close() + raise + + def readable(self): + return self._read_event is not None + + def writable(self): + return self._write_event is not None + + def seekable(self): + if self._seekable is None: + try: + os.lseek(self._fileno, 0, os.SEEK_CUR) + except OSError: + self._seekable = False + else: + self._seekable = True + return self._seekable + + def fileno(self): + return self._fileno + + @property + def closed(self): + return self._closed + + def __destroy_events(self): + read_event = self._read_event + write_event = self._write_event + hub = self.hub + self.hub = self._read_event = self._write_event = None + + if read_event is not None: + hub.cancel_wait(read_event, cancel_wait_ex, True) + if write_event is not None: + hub.cancel_wait(write_event, cancel_wait_ex, True) + + def close(self): + if self._closed: + return + self.flush() + # TODO: Can we use 'read_event is not None and write_event is + # not None' to mean _closed? + self._closed = True + self.__destroy_events() + fileno = self._fileno + if self._closefd: + self._fileno = None + os.close(fileno) + + # RawIOBase provides a 'read' method that will call readall() if + # the `size` was missing or -1 and otherwise call readinto(). We + # want to take advantage of this to avoid single byte reads when + # possible. This is highlighted by a bug in BufferedIOReader that + # calls read() in a loop when its readall() method is invoked; + # this was fixed in Python 3.3, but we still need our workaround for 2.7. See + # https://github.com/gevent/gevent/issues/675) + def __read(self, n): + if self._read_event is None: + raise UnsupportedOperation('read') + while True: + try: + return _read(self._fileno, n) + except (IOError, OSError) as ex: + if ex.args[0] not in ignored_errors: + raise + self.hub.wait(self._read_event) + + def readall(self): + ret = BytesIO() + while True: + data = self.__read(DEFAULT_BUFFER_SIZE) + if not data: + break + ret.write(data) + return ret.getvalue() + + def readinto(self, b): + data = self.__read(len(b)) + n = len(data) + try: + b[:n] = data + except TypeError as err: + import array + if not isinstance(b, array.array): + raise err + b[:n] = array.array(b'b', data) + return n + + def write(self, b): + if self._write_event is None: + raise UnsupportedOperation('write') + while True: + try: + return _write(self._fileno, b) + except (IOError, OSError) as ex: + if ex.args[0] not in ignored_errors: + raise + self.hub.wait(self._write_event) + + def seek(self, offset, whence=0): + try: + return os.lseek(self._fileno, offset, whence) + except IOError: # pylint:disable=try-except-raise + raise + except OSError as ex: # pylint:disable=duplicate-except + # Python 2.x + # make sure on Python 2.x we raise an IOError + # as documented for RawIOBase. + # See https://github.com/gevent/gevent/issues/1323 + reraise(IOError, IOError(*ex.args), sys.exc_info()[2]) + + +class FlushingBufferedWriter(BufferedWriter): + + def write(self, b): + ret = BufferedWriter.write(self, b) + self.flush() + return ret + + +class FileObjectPosix(FileObjectBase): + """ + A file-like object that operates on non-blocking files but + provides a synchronous, cooperative interface. + + .. caution:: + This object is only effective wrapping files that can be used meaningfully + with :func:`select.select` such as sockets and pipes. + + In general, on most platforms, operations on regular files + (e.g., ``open('a_file.txt')``) are considered non-blocking + already, even though they can take some time to complete as + data is copied to the kernel and flushed to disk: this time + is relatively bounded compared to sockets or pipes, though. + A :func:`~os.read` or :func:`~os.write` call on such a file + will still effectively block for some small period of time. + Therefore, wrapping this class around a regular file is + unlikely to make IO gevent-friendly: reading or writing large + amounts of data could still block the event loop. + + If you'll be working with regular files and doing IO in large + chunks, you may consider using + :class:`~gevent.fileobject.FileObjectThread` or + :func:`~gevent.os.tp_read` and :func:`~gevent.os.tp_write` to bypass this + concern. + + .. note:: + Random read/write (e.g., ``mode='rwb'``) is not supported. + For that, use :class:`io.BufferedRWPair` around two instance of this + class. + + .. tip:: + Although this object provides a :meth:`fileno` method and so + can itself be passed to :func:`fcntl.fcntl`, setting the + :data:`os.O_NONBLOCK` flag will have no effect (reads will + still block the greenlet, although other greenlets can run). + However, removing that flag *will cause this object to no + longer be cooperative* (other greenlets will no longer run). + + You can use the internal ``fileio`` attribute of this object + (a :class:`io.RawIOBase`) to perform non-blocking byte reads. + Note, however, that once you begin directly using this + attribute, the results from using methods of *this* object + are undefined, especially in text mode. (See :issue:`222`.) + + .. versionchanged:: 1.1 + Now uses the :mod:`io` package internally. Under Python 2, previously + used the undocumented class :class:`socket._fileobject`. This provides + better file-like semantics (and portability to Python 3). + .. versionchanged:: 1.2a1 + Document the ``fileio`` attribute for non-blocking reads. + """ + + #: platform specific default for the *bufsize* parameter + default_bufsize = io.DEFAULT_BUFFER_SIZE + + def __init__(self, fobj, mode='rb', bufsize=-1, close=True): + """ + :param fobj: Either an integer fileno, or an object supporting the + usual :meth:`socket.fileno` method. The file *will* be + put in non-blocking mode using :func:`gevent.os.make_nonblocking`. + :keyword str mode: The manner of access to the file, one of "rb", "rU" or "wb" + (where the "b" or "U" can be omitted). + If "U" is part of the mode, universal newlines will be used. On Python 2, + if 't' is not in the mode, this will result in returning byte (native) strings; + putting 't' in the mode will return text strings. This may cause + :exc:`UnicodeDecodeError` to be raised. + :keyword int bufsize: If given, the size of the buffer to use. The default + value means to use a platform-specific default + Other values are interpreted as for the :mod:`io` package. + Buffering is ignored in text mode. + + .. versionchanged:: 1.3a1 + + On Python 2, enabling universal newlines no longer forces unicode + IO. + + .. versionchanged:: 1.2a1 + + A bufsize of 0 in write mode is no longer forced to be 1. + Instead, the underlying buffer is flushed after every write + operation to simulate a bufsize of 0. In gevent 1.0, a + bufsize of 0 was flushed when a newline was written, while + in gevent 1.1 it was flushed when more than one byte was + written. Note that this may have performance impacts. + """ + + if isinstance(fobj, int): + fileno = fobj + fobj = None + else: + fileno = fobj.fileno() + if not isinstance(fileno, int): + raise TypeError('fileno must be int: %r' % fileno) + + orig_mode = mode + mode = (mode or 'rb').replace('b', '') + if 'U' in mode: + self._translate = True + if bytes is str and 't' not in mode: + # We're going to be producing unicode objects, but + # universal newlines doesn't do that in the stdlib, + # so fix that to return str objects. The fix is two parts: + # first, set an encoding on the stream that can round-trip + # all bytes, and second, decode all bytes once they've been read. + self._translate_encoding = 'latin-1' + import functools + + def wrap_method(m): + if m.__name__.startswith("read"): + @functools.wraps(m) + def wrapped(*args, **kwargs): + result = m(*args, **kwargs) + assert isinstance(result, unicode) # pylint:disable=undefined-variable + return result.encode('latin-1') + return wrapped + return m + self._wrap_method = wrap_method + mode = mode.replace('U', '') + else: + self._translate = False + + mode = mode.replace('t', '') + + if len(mode) != 1 and mode not in 'rw': # pragma: no cover + # Python 3 builtin `open` raises a ValueError for invalid modes; + # Python 2 ignores it. In the past, we raised an AssertionError, if __debug__ was + # enabled (which it usually was). Match Python 3 because it makes more sense + # and because __debug__ may not be enabled. + # NOTE: This is preventing a mode like 'rwb' for binary random access; + # that code was never tested and was explicitly marked as "not used" + raise ValueError('mode can only be [rb, rU, wb], not %r' % (orig_mode,)) + + + self._orig_bufsize = bufsize + if bufsize < 0 or bufsize == 1: + bufsize = self.default_bufsize + elif bufsize == 0: + bufsize = 1 + + if mode == 'r': + IOFamily = BufferedReader + else: + assert mode == 'w' + IOFamily = BufferedWriter + if self._orig_bufsize == 0: + # We could also simply pass self.fileio as *io*, but this way + # we at least consistently expose a BufferedWriter in our *io* + # attribute. + IOFamily = FlushingBufferedWriter + + + self._fobj = fobj + # This attribute is documented as available for non-blocking reads. + self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close) + + buffered_fobj = IOFamily(self.fileio, bufsize) + + super(FileObjectPosix, self).__init__(buffered_fobj, close) + + def _do_close(self, fobj, closefd): + try: + fobj.close() + # self.fileio already knows whether or not to close the + # file descriptor + self.fileio.close() + finally: + self._fobj = None + self.fileio = None + + def __iter__(self): + return self._io diff --git a/src/gevent/_greenlet.pxd b/src/gevent/_greenlet.pxd new file mode 100644 index 0000000..bb9de08 --- /dev/null +++ b/src/gevent/_greenlet.pxd @@ -0,0 +1,177 @@ +# cython: auto_pickle=False + +cimport cython +from gevent.__ident cimport IdentRegistry +from gevent.__hub_local cimport get_hub_noargs as get_hub +from gevent.__waiter cimport Waiter + +cdef bint _PYPY +cdef sys_getframe +cdef sys_exc_info +cdef Timeout +cdef GreenletExit +cdef InvalidSwitchError + +cdef extern from "greenlet/greenlet.h": + + ctypedef class greenlet.greenlet [object PyGreenlet]: + pass + + # These are actually macros and so much be included + # (defined) in each .pxd, as are the two functions + # that call them. + greenlet PyGreenlet_GetCurrent() + void PyGreenlet_Import() + +@cython.final +cdef inline greenlet getcurrent(): + return PyGreenlet_GetCurrent() + +cdef bint _greenlet_imported + +cdef inline void greenlet_init(): + global _greenlet_imported + if not _greenlet_imported: + PyGreenlet_Import() + _greenlet_imported = True + +cdef extern from "Python.h": + + ctypedef class types.CodeType [object PyCodeObject]: + pass + +cdef extern from "frameobject.h": + + ctypedef class types.FrameType [object PyFrameObject]: + cdef CodeType f_code + cdef int f_lineno + # We can't declare this in the object, because it's + # allowed to be NULL, and Cython can't handle that. + # We have to go through the python machinery to get a + # proper None instead. + # cdef FrameType f_back + +cdef void _init() + +cdef class SpawnedLink: + cdef public object callback + + +@cython.final +cdef class SuccessSpawnedLink(SpawnedLink): + pass + +@cython.final +cdef class FailureSpawnedLink(SpawnedLink): + pass + +@cython.final +@cython.internal +@cython.freelist(1000) +cdef class _Frame: + cdef readonly CodeType f_code + cdef readonly int f_lineno + cdef readonly _Frame f_back + + +@cython.final +@cython.locals(frames=list,frame=FrameType) +cdef inline list _extract_stack(int limit) + +@cython.final +@cython.locals(previous=_Frame, frame=tuple, f=_Frame) +cdef _Frame _Frame_from_list(list frames) + + +cdef class Greenlet(greenlet): + cdef readonly object value + cdef readonly tuple args + cdef readonly dict kwargs + cdef readonly object spawning_greenlet + cdef public dict spawn_tree_locals + + # This is accessed with getattr() dynamically so it + # must be visible to Python + cdef readonly list _spawning_stack_frames + + cdef list _links + cdef tuple _exc_info + cdef object _notifier + cdef object _start_event + cdef str _formatted_info + cdef object _ident + + cpdef bint has_links(self) + cpdef join(self, timeout=*) + cpdef bint ready(self) + cpdef bint successful(self) + cpdef rawlink(self, object callback) + cpdef str _formatinfo(self) + + @cython.locals(reg=IdentRegistry) + cdef _get_minimal_ident(self) + + + cdef bint __started_but_aborted(self) + cdef bint __start_cancelled_by_kill(self) + cdef bint __start_pending(self) + cdef bint __never_started_or_killed(self) + cdef bint __start_completed(self) + cdef __handle_death_before_start(self, tuple args) + + cdef __cancel_start(self) + + cdef _report_result(self, object result) + cdef _report_error(self, tuple exc_info) + # This is used as the target of a callback + # from the loop, and so needs to be a cpdef + cpdef _notify_links(self) + + # Hmm, declaring _raise_exception causes issues when _imap + # is also compiled. + # TypeError: wrap() takes exactly one argument (0 given) + # cpdef _raise_exception(self) + + + +# Declare a bunch of imports as cdefs so they can +# be accessed directly as static vars without +# doing a module global lookup. This is especially important +# for spawning greenlets. +cdef _greenlet__init__ +cdef _threadlocal +cdef get_hub_class +cdef wref + +cdef dump_traceback +cdef load_traceback +cdef Waiter +cdef wait +cdef iwait +cdef reraise +cpdef GEVENT_CONFIG + + +@cython.final +@cython.internal +cdef class _dummy_event: + cdef readonly bint pending + cdef readonly bint active + + cpdef stop(self) + cpdef start(self, cb) + cpdef close(self) + +cdef _dummy_event _cancelled_start_event +cdef _dummy_event _start_completed_event + + +@cython.locals(diehards=list) +cdef _killall3(list greenlets, object exception, object waiter) +cdef _killall(list greenlets, object exception) + +@cython.locals(done=list) +cpdef joinall(greenlets, timeout=*, raise_error=*, count=*) + +cdef set _spawn_callbacks +cdef void _call_spawn_callbacks(Greenlet gr) except * diff --git a/src/gevent/_greenlet_primitives.py b/src/gevent/_greenlet_primitives.py new file mode 100644 index 0000000..b982bc1 --- /dev/null +++ b/src/gevent/_greenlet_primitives.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# copyright (c) 2018 gevent. See LICENSE. +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False +""" +A collection of primitives used by the hub, and suitable for +compilation with Cython because of their frequency of use. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from weakref import ref as wref +from gc import get_objects + +from greenlet import greenlet + +from gevent.exceptions import BlockingSwitchOutError + + +# In Cython, we define these as 'cdef inline' functions. The +# compilation unit cannot have a direct assignment to them (import +# is assignment) without generating a 'lvalue is not valid target' +# error. +locals()['getcurrent'] = __import__('greenlet').getcurrent +locals()['greenlet_init'] = lambda: None +locals()['_greenlet_switch'] = greenlet.switch + +__all__ = [ + 'TrackedRawGreenlet', + 'SwitchOutGreenletWithLoop', +] + +class TrackedRawGreenlet(greenlet): + + def __init__(self, function, parent): + greenlet.__init__(self, function, parent) + # See greenlet.py's Greenlet class. We capture the cheap + # parts to maintain the tree structure, but we do not capture + # the stack because that's too expensive for 'spawn_raw'. + + current = getcurrent() # pylint:disable=undefined-variable + self.spawning_greenlet = wref(current) + # See Greenlet for how trees are maintained. + try: + self.spawn_tree_locals = current.spawn_tree_locals + except AttributeError: + self.spawn_tree_locals = {} + if current.parent: + current.spawn_tree_locals = self.spawn_tree_locals + + +class SwitchOutGreenletWithLoop(TrackedRawGreenlet): + # Subclasses must define: + # - self.loop + + # This class defines loop in its .pxd for Cython. This lets us avoid + # circular dependencies with the hub. + + def switch(self): + switch_out = getattr(getcurrent(), 'switch_out', None) # pylint:disable=undefined-variable + if switch_out is not None: + switch_out() + return _greenlet_switch(self) # pylint:disable=undefined-variable + + def switch_out(self): + raise BlockingSwitchOutError('Impossible to call blocking function in the event loop callback') + + +def get_reachable_greenlets(): + # We compile this loop with Cython so that it's faster, and so that + # the GIL isn't dropped at unpredictable times during the loop. + # Dropping the GIL could lead to accessing partly constructed objects + # in undefined states (particularly, tuples). This helps close a hole + # where a `SystemError: Objects/tupleobject.c bad argument to internal function` + # could get raised. (Note that this probably doesn't completely close the hole, + # if other threads have dropped the GIL, but hopefully the speed makes that + # more rare.) See https://github.com/gevent/gevent/issues/1302 + return [ + x for x in get_objects() + if isinstance(x, greenlet) and not getattr(x, 'greenlet_tree_is_ignored', False) + ] + +def _init(): + greenlet_init() # pylint:disable=undefined-variable + +_init() + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__greenlet_primitives') diff --git a/src/gevent/_hub_local.py b/src/gevent/_hub_local.py new file mode 100644 index 0000000..622062a --- /dev/null +++ b/src/gevent/_hub_local.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# copyright 2018 gevent. See LICENSE +""" +Maintains the thread local hub. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from gevent._compat import thread_mod_name + +__all__ = [ + 'get_hub', + 'get_hub_noargs', + 'get_hub_if_exists', +] + +# These must be the "real" native thread versions, +# not monkey-patched. +# We are imported early enough (by gevent/__init__) that +# we can rely on not being monkey-patched in any way yet. +class _Threadlocal(__import__(thread_mod_name)._local): + + def __init__(self): + # Use a class with an initializer so that we can test + # for 'is None' instead of catching AttributeError, making + # the code cleaner and possibly solving some corner cases + # (like #687) + super(_Threadlocal, self).__init__() + self.Hub = None + self.loop = None + self.hub = None + +_threadlocal = _Threadlocal() + +Hub = None # Set when gevent.hub is imported + +def get_hub_class(): + """Return the type of hub to use for the current thread. + + If there's no type of hub for the current thread yet, 'gevent.hub.Hub' is used. + """ + hubtype = _threadlocal.Hub + if hubtype is None: + hubtype = _threadlocal.Hub = Hub + return hubtype + +def set_default_hub_class(hubtype): + global Hub + Hub = hubtype + +def get_hub(*args, **kwargs): + """ + Return the hub for the current thread. + + If a hub does not exist in the current thread, a new one is + created of the type returned by :func:`get_hub_class`. + + .. deprecated:: 1.3b1 + The ``*args`` and ``**kwargs`` arguments are deprecated. They were + only used when the hub was created, and so were non-deterministic---to be + sure they were used, *all* callers had to pass them, or they were order-dependent. + Use ``set_hub`` instead. + """ + hub = _threadlocal.hub + if hub is None: + hubtype = get_hub_class() + hub = _threadlocal.hub = hubtype(*args, **kwargs) + return hub + +def get_hub_noargs(): + # Just like get_hub, but cheaper to call because it + # takes no arguments or kwargs. See also a copy in + # gevent/greenlet.py + hub = _threadlocal.hub + if hub is None: + hubtype = get_hub_class() + hub = _threadlocal.hub = hubtype() + return hub + +def get_hub_if_exists(): + """Return the hub for the current thread. + + Return ``None`` if no hub has been created yet. + """ + return _threadlocal.hub + + +def set_hub(hub): + _threadlocal.hub = hub + +def get_loop(): + return _threadlocal.loop + +def set_loop(loop): + _threadlocal.loop = loop + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__hub_local') diff --git a/src/gevent/_hub_primitives.py b/src/gevent/_hub_primitives.py new file mode 100644 index 0000000..490c7a6 --- /dev/null +++ b/src/gevent/_hub_primitives.py @@ -0,0 +1,394 @@ +# -*- coding: utf-8 -*- +# copyright (c) 2018 gevent. See LICENSE. +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False,binding=True +""" +A collection of primitives used by the hub, and suitable for +compilation with Cython because of their frequency of use. + + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import traceback + +from gevent.exceptions import InvalidSwitchError +from gevent.exceptions import ConcurrentObjectUseError + +from gevent import _greenlet_primitives +from gevent import _waiter +from gevent._util import _NONE +from gevent._hub_local import get_hub_noargs as get_hub +from gevent.timeout import Timeout + +# In Cython, we define these as 'cdef inline' functions. The +# compilation unit cannot have a direct assignment to them (import +# is assignment) without generating a 'lvalue is not valid target' +# error. +locals()['getcurrent'] = __import__('greenlet').getcurrent +locals()['greenlet_init'] = lambda: None +locals()['Waiter'] = _waiter.Waiter +locals()['MultipleWaiter'] = _waiter.MultipleWaiter +locals()['SwitchOutGreenletWithLoop'] = _greenlet_primitives.SwitchOutGreenletWithLoop + +__all__ = [ + 'WaitOperationsGreenlet', + 'iwait_on_objects', + 'wait_on_objects', + 'wait_read', + 'wait_write', + 'wait_readwrite', +] + +class WaitOperationsGreenlet(SwitchOutGreenletWithLoop): # pylint:disable=undefined-variable + + def wait(self, watcher): + """ + Wait until the *watcher* (which must not be started) is ready. + + The current greenlet will be unscheduled during this time. + """ + waiter = Waiter(self) # pylint:disable=undefined-variable + watcher.start(waiter.switch, waiter) + try: + result = waiter.get() + if result is not waiter: + raise InvalidSwitchError('Invalid switch into %s: %r (expected %r)' % ( + getcurrent(), # pylint:disable=undefined-variable + result, waiter)) + finally: + watcher.stop() + + def cancel_wait(self, watcher, error, close_watcher=False): + """ + Cancel an in-progress call to :meth:`wait` by throwing the given *error* + in the waiting greenlet. + + .. versionchanged:: 1.3a1 + Added the *close_watcher* parameter. If true, the watcher + will be closed after the exception is thrown. The watcher should then + be discarded. Closing the watcher is important to release native resources. + .. versionchanged:: 1.3a2 + Allow the *watcher* to be ``None``. No action is taken in that case. + """ + if watcher is None: + # Presumably already closed. + # See https://github.com/gevent/gevent/issues/1089 + return + if watcher.callback is not None: + self.loop.run_callback(self._cancel_wait, watcher, error, close_watcher) + elif close_watcher: + watcher.close() + + def _cancel_wait(self, watcher, error, close_watcher): + # We have to check again to see if it was still active by the time + # our callback actually runs. + active = watcher.active + cb = watcher.callback + if close_watcher: + watcher.close() + if active: + # The callback should be greenlet.switch(). It may or may not be None. + glet = getattr(cb, '__self__', None) + if glet is not None: + glet.throw(error) + + +class _WaitIterator(object): + + def __init__(self, objects, hub, timeout, count): + self._hub = hub + self._waiter = MultipleWaiter(hub) # pylint:disable=undefined-variable + self._switch = self._waiter.switch + self._timeout = timeout + self._objects = objects + + self._timer = None + self._begun = False + + # Even if we're only going to return 1 object, + # we must still rawlink() *all* of them, so that no + # matter which one finishes first we find it. + self._count = len(objects) if count is None else min(count, len(objects)) + + def _begin(self): + if self._begun: + return + + self._begun = True + + # XXX: If iteration doesn't actually happen, we + # could leave these links around! + for obj in self._objects: + obj.rawlink(self._switch) + + if self._timeout is not None: + self._timer = self._hub.loop.timer(self._timeout, priority=-1) + self._timer.start(self._switch, self) + + def __iter__(self): + return self + + def __next__(self): + self._begin() + + if self._count == 0: + # Exhausted + self._cleanup() + raise StopIteration() + + self._count -= 1 + try: + item = self._waiter.get() + self._waiter.clear() + if item is self: + # Timer expired, no more + self._cleanup() + raise StopIteration() + return item + except: + self._cleanup() + raise + + next = __next__ + + def _cleanup(self): + if self._timer is not None: + self._timer.close() + self._timer = None + + objs = self._objects + self._objects = () + for aobj in objs: + unlink = getattr(aobj, 'unlink', None) + if unlink is not None: + try: + unlink(self._switch) + except: # pylint:disable=bare-except + traceback.print_exc() + + def __enter__(self): + return self + + def __exit__(self, typ, value, tb): + self._cleanup() + + +def iwait_on_objects(objects, timeout=None, count=None): + """ + Iteratively yield *objects* as they are ready, until all (or *count*) are ready + or *timeout* expired. + + If you will only be consuming a portion of the *objects*, you should + do so inside a ``with`` block on this object to avoid leaking resources:: + + with gevent.iwait((a, b, c)) as it: + for i in it: + if i is a: + break + + :param objects: A sequence (supporting :func:`len`) containing objects + implementing the wait protocol (rawlink() and unlink()). + :keyword int count: If not `None`, then a number specifying the maximum number + of objects to wait for. If ``None`` (the default), all objects + are waited for. + :keyword float timeout: If given, specifies a maximum number of seconds + to wait. If the timeout expires before the desired waited-for objects + are available, then this method returns immediately. + + .. seealso:: :func:`wait` + + .. versionchanged:: 1.1a1 + Add the *count* parameter. + .. versionchanged:: 1.1a2 + No longer raise :exc:`LoopExit` if our caller switches greenlets + in between items yielded by this function. + .. versionchanged:: 1.4 + Add support to use the returned object as a context manager. + """ + # QQQ would be nice to support iterable here that can be generated slowly (why?) + hub = get_hub() + if objects is None: + return [hub.join(timeout=timeout)] + return _WaitIterator(objects, hub, timeout, count) + + +def wait_on_objects(objects=None, timeout=None, count=None): + """ + Wait for ``objects`` to become ready or for event loop to finish. + + If ``objects`` is provided, it must be a list containing objects + implementing the wait protocol (rawlink() and unlink() methods): + + - :class:`gevent.Greenlet` instance + - :class:`gevent.event.Event` instance + - :class:`gevent.lock.Semaphore` instance + - :class:`gevent.subprocess.Popen` instance + + If ``objects`` is ``None`` (the default), ``wait()`` blocks until + the current event loop has nothing to do (or until ``timeout`` passes): + + - all greenlets have finished + - all servers were stopped + - all event loop watchers were stopped. + + If ``count`` is ``None`` (the default), wait for all ``objects`` + to become ready. + + If ``count`` is a number, wait for (up to) ``count`` objects to become + ready. (For example, if count is ``1`` then the function exits + when any object in the list is ready). + + If ``timeout`` is provided, it specifies the maximum number of + seconds ``wait()`` will block. + + Returns the list of ready objects, in the order in which they were + ready. + + .. seealso:: :func:`iwait` + """ + if objects is None: + hub = get_hub() + return hub.join(timeout=timeout) # pylint:disable= + return list(iwait_on_objects(objects, timeout, count)) + +_timeout_error = Exception + +def set_default_timeout_error(e): + global _timeout_error + _timeout_error = e + +def _primitive_wait(watcher, timeout, timeout_exc, hub): + if watcher.callback is not None: + raise ConcurrentObjectUseError('This socket is already used by another greenlet: %r' + % (watcher.callback, )) + + if hub is None: + hub = get_hub() + + if timeout is None: + hub.wait(watcher) + return + + timeout = Timeout._start_new_or_dummy( + timeout, + (timeout_exc + if timeout_exc is not _NONE or timeout is None + else _timeout_error('timed out'))) + + with timeout: + hub.wait(watcher) + +# Suitable to be bound as an instance method +def wait_on_socket(socket, watcher, timeout_exc=None): + if socket is None or watcher is None: + # test__hub TestCloseSocketWhilePolling, on Python 2; Python 3 + # catches the EBADF differently. + raise ConcurrentObjectUseError("The socket has already been closed by another greenlet") + _primitive_wait(watcher, socket.timeout, + timeout_exc if timeout_exc is not None else _NONE, + socket.hub) + +def wait_on_watcher(watcher, timeout=None, timeout_exc=_NONE, hub=None): + """ + wait(watcher, timeout=None, [timeout_exc=None]) -> None + + Block the current greenlet until *watcher* is ready. + + If *timeout* is non-negative, then *timeout_exc* is raised after + *timeout* second has passed. + + If :func:`cancel_wait` is called on *io* by another greenlet, + raise an exception in this blocking greenlet + (``socket.error(EBADF, 'File descriptor was closed in another + greenlet')`` by default). + + :param io: An event loop watcher, most commonly an IO watcher obtained from + :meth:`gevent.core.loop.io` + :keyword timeout_exc: The exception to raise if the timeout expires. + By default, a :class:`socket.timeout` exception is raised. + If you pass a value for this keyword, it is interpreted as for + :class:`gevent.timeout.Timeout`. + + :raises ~gevent.hub.ConcurrentObjectUseError: If the *watcher* is + already started. + """ + _primitive_wait(watcher, timeout, timeout_exc, hub) + + +def wait_read(fileno, timeout=None, timeout_exc=_NONE): + """ + wait_read(fileno, timeout=None, [timeout_exc=None]) -> None + + Block the current greenlet until *fileno* is ready to read. + + For the meaning of the other parameters and possible exceptions, + see :func:`wait`. + + .. seealso:: :func:`cancel_wait` + """ + hub = get_hub() + io = hub.loop.io(fileno, 1) + try: + return wait_on_watcher(io, timeout, timeout_exc, hub) + finally: + io.close() + + +def wait_write(fileno, timeout=None, timeout_exc=_NONE, event=_NONE): + """ + wait_write(fileno, timeout=None, [timeout_exc=None]) -> None + + Block the current greenlet until *fileno* is ready to write. + + For the meaning of the other parameters and possible exceptions, + see :func:`wait`. + + .. deprecated:: 1.1 + The keyword argument *event* is ignored. Applications should not pass this parameter. + In the future, doing so will become an error. + + .. seealso:: :func:`cancel_wait` + """ + # pylint:disable=unused-argument + hub = get_hub() + io = hub.loop.io(fileno, 2) + try: + return wait_on_watcher(io, timeout, timeout_exc, hub) + finally: + io.close() + + +def wait_readwrite(fileno, timeout=None, timeout_exc=_NONE, event=_NONE): + """ + wait_readwrite(fileno, timeout=None, [timeout_exc=None]) -> None + + Block the current greenlet until *fileno* is ready to read or + write. + + For the meaning of the other parameters and possible exceptions, + see :func:`wait`. + + .. deprecated:: 1.1 + The keyword argument *event* is ignored. Applications should not pass this parameter. + In the future, doing so will become an error. + + .. seealso:: :func:`cancel_wait` + """ + # pylint:disable=unused-argument + hub = get_hub() + io = hub.loop.io(fileno, 3) + try: + return wait_on_watcher(io, timeout, timeout_exc, hub) + finally: + io.close() + + +def _init(): + greenlet_init() # pylint:disable=undefined-variable + +_init() + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__hub_primitives') diff --git a/src/gevent/_ident.py b/src/gevent/_ident.py new file mode 100644 index 0000000..97c6055 --- /dev/null +++ b/src/gevent/_ident.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 gevent contributors. See LICENSE for details. +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from weakref import WeakKeyDictionary +from weakref import ref + +from heapq import heappop +from heapq import heappush + +__all__ = [ + 'IdentRegistry', +] + +class ValuedWeakRef(ref): + """ + A weak ref with an associated value. + """ + + __slots__ = ('value',) + + +class IdentRegistry(object): + """ + Maintains a unique mapping of (small) positive integer identifiers + to objects that can be weakly referenced. + + It is guaranteed that no two objects will have the the same + identifier at the same time, as long as those objects are + also uniquely hashable. + """ + + def __init__(self): + # {obj -> (ident, wref(obj))} + self._registry = WeakKeyDictionary() + + # A heap of numbers that have been used and returned + self._available_idents = [] + + def get_ident(self, obj): + """ + Retrieve the identifier for *obj*, creating one + if necessary. + """ + + try: + return self._registry[obj][0] + except KeyError: + pass + + if self._available_idents: + # Take the smallest free number + ident = heappop(self._available_idents) + else: + # Allocate a bigger one + ident = len(self._registry) + + vref = ValuedWeakRef(obj, self._return_ident) + vref.value = ident # pylint:disable=assigning-non-slot,attribute-defined-outside-init + self._registry[obj] = (ident, vref) + return ident + + def _return_ident(self, vref): + # By the time this is called, self._registry has been + # updated + if heappush is not None: + # Under some circumstances we can get called + # when the interpreter is shutting down, and globals + # aren't available any more. + heappush(self._available_idents, vref.value) + + def __len__(self): + return len(self._registry) + + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__ident') diff --git a/src/gevent/_imap.py b/src/gevent/_imap.py new file mode 100644 index 0000000..e976b67 --- /dev/null +++ b/src/gevent/_imap.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 gevent +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False,infer_types=True + +""" +Iterators across greenlets or AsyncResult objects. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from gevent import _semaphore +from gevent import queue + + +__all__ = [ + 'IMapUnordered', + 'IMap', +] + +locals()['Greenlet'] = __import__('gevent').Greenlet +locals()['Semaphore'] = _semaphore.Semaphore +locals()['UnboundQueue'] = queue.UnboundQueue + + +class Failure(object): + __slots__ = ('exc', 'raise_exception') + + def __init__(self, exc, raise_exception=None): + self.exc = exc + self.raise_exception = raise_exception + + +def _raise_exc(failure): + # For cython. + if failure.raise_exception: + failure.raise_exception() + else: + raise failure.exc + +class IMapUnordered(Greenlet): # pylint:disable=undefined-variable + """ + At iterator of map results. + """ + + def __init__(self, func, iterable, spawn, maxsize=None, _zipped=False): + """ + An iterator that. + + :param callable spawn: The function we use to create new greenlets. + :keyword int maxsize: If given and not-None, specifies the maximum number of + finished results that will be allowed to accumulated awaiting the reader; + more than that number of results will cause map function greenlets to begin + to block. This is most useful is there is a great disparity in the speed of + the mapping code and the consumer and the results consume a great deal of resources. + Using a bound is more computationally expensive than not using a bound. + + .. versionchanged:: 1.1b3 + Added the *maxsize* parameter. + """ + Greenlet.__init__(self) # pylint:disable=undefined-variable + self.spawn = spawn + self._zipped = _zipped + self.func = func + self.iterable = iterable + self.queue = UnboundQueue() # pylint:disable=undefined-variable + + + if maxsize: + # Bounding the queue is not enough if we want to keep from + # accumulating objects; the result value will be around as + # the greenlet's result, blocked on self.queue.put(), and + # we'll go on to spawn another greenlet, which in turn can + # create the result. So we need a semaphore to prevent a + # greenlet from exiting while the queue is full so that we + # don't spawn the next greenlet (assuming that self.spawn + # is of course bounded). (Alternatively we could have the + # greenlet itself do the insert into the pool, but that + # takes some rework). + # + # Given the use of a semaphore at this level, sizing the queue becomes + # redundant, and that lets us avoid having to use self.link() instead + # of self.rawlink() to avoid having blocking methods called in the + # hub greenlet. + self._result_semaphore = Semaphore(maxsize) # pylint:disable=undefined-variable + else: + self._result_semaphore = None + + self._outstanding_tasks = 0 + # The index (zero based) of the maximum number of + # results we will have. + self._max_index = -1 + self.finished = False + + + # We're iterating in a different greenlet than we're running. + def __iter__(self): + return self + + def __next__(self): + if self._result_semaphore is not None: + self._result_semaphore.release() + value = self._inext() + if isinstance(value, Failure): + _raise_exc(value) + return value + + next = __next__ # Py2 + + def _inext(self): + return self.queue.get() + + def _ispawn(self, func, item, item_index): + if self._result_semaphore is not None: + self._result_semaphore.acquire() + self._outstanding_tasks += 1 + g = self.spawn(func, item) if not self._zipped else self.spawn(func, *item) + g._imap_task_index = item_index + g.rawlink(self._on_result) + return g + + def _run(self): # pylint:disable=method-hidden + try: + func = self.func + for item in self.iterable: + self._max_index += 1 + self._ispawn(func, item, self._max_index) + self._on_finish(None) + except BaseException as e: + self._on_finish(e) + raise + finally: + self.spawn = None + self.func = None + self.iterable = None + self._result_semaphore = None + + def _on_result(self, greenlet): + # This method will be called in the hub greenlet (we rawlink) + self._outstanding_tasks -= 1 + count = self._outstanding_tasks + finished = self.finished + ready = self.ready() + put_finished = False + + if ready and count <= 0 and not finished: + finished = self.finished = True + put_finished = True + + if greenlet.successful(): + self.queue.put(self._iqueue_value_for_success(greenlet)) + else: + self.queue.put(self._iqueue_value_for_failure(greenlet)) + + if put_finished: + self.queue.put(self._iqueue_value_for_self_finished()) + + def _on_finish(self, exception): + # Called in this greenlet. + if self.finished: + return + + if exception is not None: + self.finished = True + self.queue.put(self._iqueue_value_for_self_failure(exception)) + return + + if self._outstanding_tasks <= 0: + self.finished = True + self.queue.put(self._iqueue_value_for_self_finished()) + + def _iqueue_value_for_success(self, greenlet): + return greenlet.value + + def _iqueue_value_for_failure(self, greenlet): + return Failure(greenlet.exception, getattr(greenlet, '_raise_exception')) + + def _iqueue_value_for_self_finished(self): + return Failure(StopIteration()) + + def _iqueue_value_for_self_failure(self, exception): + return Failure(exception, self._raise_exception) + + +class IMap(IMapUnordered): + # A specialization of IMapUnordered that returns items + # in the order in which they were generated, not + # the order in which they finish. + + def __init__(self, *args, **kwargs): + # The result dictionary: {index: value} + self._results = {} + + # The index of the result to return next. + self.index = 0 + IMapUnordered.__init__(self, *args, **kwargs) + + def _inext(self): + try: + value = self._results.pop(self.index) + except KeyError: + # Wait for our index to finish. + while 1: + index, value = self.queue.get() + if index == self.index: + break + else: + self._results[index] = value + self.index += 1 + return value + + def _iqueue_value_for_success(self, greenlet): + return (greenlet._imap_task_index, IMapUnordered._iqueue_value_for_success(self, greenlet)) + + def _iqueue_value_for_failure(self, greenlet): + return (greenlet._imap_task_index, IMapUnordered._iqueue_value_for_failure(self, greenlet)) + + def _iqueue_value_for_self_finished(self): + return (self._max_index + 1, IMapUnordered._iqueue_value_for_self_finished(self)) + + def _iqueue_value_for_self_failure(self, exception): + return (self._max_index + 1, IMapUnordered._iqueue_value_for_self_failure(self, exception)) + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__imap') diff --git a/src/gevent/_interfaces.py b/src/gevent/_interfaces.py new file mode 100644 index 0000000..8ee2020 --- /dev/null +++ b/src/gevent/_interfaces.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2018 gevent contributors. See LICENSE for details. +""" +Interfaces gevent uses that don't belong any one place. + +This is not a public module, these interfaces are not +currently exposed to the public, they mostly exist for +documentation and testing purposes. + +.. versionadded:: 1.3b2 + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys + +from gevent._util import Interface +from gevent._util import Attribute + +# pylint:disable=no-method-argument, unused-argument, no-self-argument + +__all__ = [ + 'ILoop', + 'IWatcher', +] + +class ILoop(Interface): + """ + The common interface expected for all event loops. + + .. caution:: + This is an internal, low-level interface. It may change + between minor versions of gevent. + + .. rubric:: Watchers + + The methods that create event loop watchers are `io`, `timer`, + `signal`, `idle`, `prepare`, `check`, `fork`, `async_`, `child`, + `stat`. These all return various types of :class:`IWatcher`. + + All of those methods have one or two common arguments. *ref* is a + boolean saying whether the event loop is allowed to exit even if + this watcher is still started. *priority* is event loop specific. + """ + + default = Attribute("Boolean indicating whether this is the default loop") + + approx_timer_resolution = Attribute( + "Floating point number of seconds giving (approximately) the minimum " + "resolution of a timer (and hence the minimun value the sleep can sleep for). " + "On libuv, this is fixed by the library, but on libev it is just a guess " + "and the actual value is system dependent." + ) + + def run(nowait=False, once=False): + """ + Run the event loop. + + This is usually called automatically by the hub greenlet, but + in special cases (when the hub is *not* running) you can use + this to control how the event loop runs (for example, to integrate + it with another event loop). + """ + + def now(): + """ + now() -> float + + Return the loop's notion of the current time. + + This may not necessarily be related to :func:`time.time` (it + may have a different starting point), but it must be expressed + in fractional seconds (the same *units* used by :func:`time.time`). + """ + + def update_now(): + """ + Update the loop's notion of the current time. + + .. versionadded:: 1.3 + In the past, this available as ``update``. This is still available as + an alias but will be removed in the future. + """ + + def destroy(): + """ + Clean up resources used by this loop. + + If you create loops + (especially loops that are not the default) you *should* call + this method when you are done with the loop. + + .. caution:: + + As an implementation note, the libev C loop implementation has a + finalizer (``__del__``) that destroys the object, but the libuv + and libev CFFI implementations do not. The C implementation may change. + + """ + + def io(fd, events, ref=True, priority=None): + """ + Create and return a new IO watcher for the given *fd*. + + *events* is a bitmask specifying which events to watch + for. 1 means read, and 2 means write. + """ + + def timer(after, repeat=0.0, ref=True, priority=None): + """ + Create and return a timer watcher that will fire after *after* seconds. + + If *repeat* is given, the timer will continue to fire every *repeat* seconds. + """ + + def signal(signum, ref=True, priority=None): + """ + Create and return a signal watcher for the signal *signum*, + one of the constants defined in :mod:`signal`. + + This is platform and event loop specific. + """ + + def idle(ref=True, priority=None): + """ + Create and return a watcher that fires when the event loop is idle. + """ + + def prepare(ref=True, priority=None): + """ + Create and return a watcher that fires before the event loop + polls for IO. + + .. caution:: This method is not supported by libuv. + """ + + def check(ref=True, priority=None): + """ + Create and return a watcher that fires after the event loop + polls for IO. + """ + + def fork(ref=True, priority=None): + """ + Create a watcher that fires when the process forks. + + Availability: POSIX + """ + + def async_(ref=True, priority=None): + """ + Create a watcher that fires when triggered, possibly + from another thread. + + .. versionchanged:: 1.3 + This was previously just named ``async``; for compatibility + with Python 3.7 where ``async`` is a keyword it was renamed. + On older versions of Python the old name is still around, but + it will be removed in the future. + """ + + if sys.platform != "win32": + + def child(pid, trace=0, ref=True): + """ + Create a watcher that fires for events on the child with process ID *pid*. + + This is platform specific and not available on Windows. + """ + + def stat(path, interval=0.0, ref=True, priority=None): + """ + Create a watcher that monitors the filesystem item at *path*. + + If the operating system doesn't support event notifications + from the filesystem, poll for changes every *interval* seconds. + """ + + def run_callback(func, *args): + """ + Run the *func* passing it *args* at the next opportune moment. + + This is a way of handing control to the event loop and deferring + an action. + """ + +class IWatcher(Interface): + """ + An event loop watcher. + + These objects call their *callback* function when the event + loop detects the event has happened. + + .. important:: You *must* call :meth:`close` when you are + done with this object to avoid leaking native resources. + """ + + def start(callback, *args, **kwargs): + """ + Have the event loop begin watching for this event. + + When the event is detected, *callback* will be called with + *args*. + + .. caution:: + + Not all watchers accept ``**kwargs``, + and some watchers define special meanings for certain keyword args. + """ + + def stop(): + """ + Have the event loop stop watching this event. + + In the future you may call :meth:`start` to begin watching + again. + """ + + def close(): + """ + Dispose of any native resources associated with the watcher. + + If we were active, stop. + + Attempting to operate on this object after calling close is + undefined. You should dispose of any references you have to it + after calling this method. + """ diff --git a/src/gevent/_local.pxd b/src/gevent/_local.pxd new file mode 100644 index 0000000..331ce28 --- /dev/null +++ b/src/gevent/_local.pxd @@ -0,0 +1,113 @@ +# cython: auto_pickle=False + +cimport cython +from gevent._greenlet cimport Greenlet + +cdef bint _PYPY +cdef ref +cdef copy + +cdef object _marker +cdef str key_prefix +cdef bint _greenlet_imported + + +cdef extern from "greenlet/greenlet.h": + + ctypedef class greenlet.greenlet [object PyGreenlet]: + pass + + # These are actually macros and so much be included + # (defined) in each .pxd, as are the two functions + # that call them. + greenlet PyGreenlet_GetCurrent() + void PyGreenlet_Import() + +cdef inline greenlet getcurrent(): + return PyGreenlet_GetCurrent() + +cdef inline void greenlet_init(): + global _greenlet_imported + if not _greenlet_imported: + PyGreenlet_Import() + _greenlet_imported = True + + +cdef void _init() + +@cython.final +@cython.internal +cdef class _wrefdict(dict): + cdef object __weakref__ + +@cython.final +@cython.internal +cdef class _greenlet_deleted: + cdef object idt + cdef object wrdicts + + +@cython.final +@cython.internal +cdef class _local_deleted: + cdef str key + cdef object wrthread + cdef _greenlet_deleted greenlet_deleted + +@cython.final +@cython.internal +cdef class _localimpl: + cdef str key + cdef dict dicts + cdef tuple localargs + cdef dict localkwargs + cdef tuple localtypeid + cdef object __weakref__ + + +@cython.final +@cython.internal +cdef class _localimpl_dict_entry: + cdef object wrgreenlet + cdef dict localdict + +@cython.locals(localdict=dict, key=str, + greenlet_deleted=_greenlet_deleted, + local_deleted=_local_deleted) +cdef dict _localimpl_create_dict(_localimpl self, + greenlet greenlet, + object idt) + +cdef set _local_attrs + +cdef class local: + cdef _localimpl _local__impl + cdef set _local_type_get_descriptors + cdef set _local_type_set_or_del_descriptors + cdef set _local_type_del_descriptors + cdef set _local_type_set_descriptors + cdef set _local_type_vars + cdef type _local_type + + @cython.locals(entry=_localimpl_dict_entry, + dct=dict, duplicate=dict, + instance=local) + cpdef local __copy__(local self) + + +@cython.locals(impl=_localimpl,dct=dict, + dct=dict, entry=_localimpl_dict_entry) +cdef inline dict _local_get_dict(local self) + +@cython.locals(entry=_localimpl_dict_entry) +cdef _local__copy_dict_from(local self, _localimpl impl, dict duplicate) + +@cython.locals(mro=list, gets=set, dels=set, set_or_del=set, + type_self=type, type_attr=type, + sets=set) +cdef tuple _local_find_descriptors(local self) + +@cython.locals(result=list, local_impl=_localimpl, + entry=_localimpl_dict_entry, k=str, + greenlet_dict=dict) +cpdef all_local_dicts_for_greenlet(greenlet greenlet) diff --git a/src/gevent/_monitor.py b/src/gevent/_monitor.py new file mode 100644 index 0000000..2c4ef03 --- /dev/null +++ b/src/gevent/_monitor.py @@ -0,0 +1,325 @@ +# Copyright (c) 2018 gevent. See LICENSE for details. +from __future__ import print_function, absolute_import, division + +import os +import sys + +from weakref import ref as wref + +from greenlet import getcurrent + +from gevent import config as GEVENT_CONFIG +from gevent.monkey import get_original +from gevent.events import notify +from gevent.events import EventLoopBlocked +from gevent.events import MemoryUsageThresholdExceeded +from gevent.events import MemoryUsageUnderThreshold +from gevent.events import IPeriodicMonitorThread +from gevent.events import implementer + +from gevent._tracer import GreenletTracer +from gevent._compat import thread_mod_name +from gevent._compat import perf_counter + + + +__all__ = [ + 'PeriodicMonitoringThread', +] + +get_thread_ident = get_original(thread_mod_name, 'get_ident') +start_new_thread = get_original(thread_mod_name, 'start_new_thread') +thread_sleep = get_original('time', 'sleep') + + + +class MonitorWarning(RuntimeWarning): + """The type of warnings we emit.""" + + +class _MonitorEntry(object): + + __slots__ = ('function', 'period', 'last_run_time') + + def __init__(self, function, period): + self.function = function + self.period = period + self.last_run_time = 0 + + def __eq__(self, other): + return self.function == other.function and self.period == other.period + + def __repr__(self): + return repr((self.function, self.period, self.last_run_time)) + + +@implementer(IPeriodicMonitorThread) +class PeriodicMonitoringThread(object): + # This doesn't extend threading.Thread because that gets monkey-patched. + # We use the low-level 'start_new_thread' primitive instead. + + # The amount of seconds we will sleep when we think we have nothing + # to do. + inactive_sleep_time = 2.0 + + # The absolute minimum we will sleep, regardless of + # what particular monitoring functions want to say. + min_sleep_time = 0.005 + + # The minimum period in seconds at which we will check memory usage. + # Getting memory usage is fairly expensive. + min_memory_monitor_period = 2 + + # A list of _MonitorEntry objects: [(function(hub), period, last_run_time))] + # The first entry is always our entry for self.monitor_blocking + _monitoring_functions = None + + # The calculated min sleep time for the monitoring functions list. + _calculated_sleep_time = None + + # A boolean value that also happens to capture the + # memory usage at the time we exceeded the threshold. Reset + # to 0 when we go back below. + _memory_exceeded = 0 + + # The instance of GreenletTracer we're using + _greenlet_tracer = None + + def __init__(self, hub): + self._hub_wref = wref(hub, self._on_hub_gc) + self.should_run = True + + # Must be installed in the thread that the hub is running in; + # the trace function is threadlocal + assert get_thread_ident() == hub.thread_ident + self._greenlet_tracer = GreenletTracer() + + self._monitoring_functions = [_MonitorEntry(self.monitor_blocking, + GEVENT_CONFIG.max_blocking_time)] + self._calculated_sleep_time = GEVENT_CONFIG.max_blocking_time + # Create the actual monitoring thread. This is effectively a "daemon" + # thread. + self.monitor_thread_ident = start_new_thread(self, ()) + + # We must track the PID to know if your thread has died after a fork + self.pid = os.getpid() + + def _on_fork(self): + # Pseudo-standard method that resolver_ares and threadpool + # also have, called by hub.reinit() + pid = os.getpid() + if pid != self.pid: + self.pid = pid + self.monitor_thread_ident = start_new_thread(self, ()) + + @property + def hub(self): + return self._hub_wref() + + + def monitoring_functions(self): + # Return a list of _MonitorEntry objects + + # Update max_blocking_time each time. + mbt = GEVENT_CONFIG.max_blocking_time # XXX: Events so we know when this changes. + if mbt != self._monitoring_functions[0].period: + self._monitoring_functions[0].period = mbt + self._calculated_sleep_time = min(x.period for x in self._monitoring_functions) + return self._monitoring_functions + + def add_monitoring_function(self, function, period): + if not callable(function): + raise ValueError("function must be callable") + + if period is None: + # Remove. + self._monitoring_functions = [ + x for x in self._monitoring_functions + if x.function != function + ] + elif period <= 0: + raise ValueError("Period must be positive.") + else: + # Add or update period + entry = _MonitorEntry(function, period) + self._monitoring_functions = [ + x if x.function != function else entry + for x in self._monitoring_functions + ] + if entry not in self._monitoring_functions: + self._monitoring_functions.append(entry) + self._calculated_sleep_time = min(x.period for x in self._monitoring_functions) + + def calculate_sleep_time(self): + min_sleep = self._calculated_sleep_time + if min_sleep <= 0: + # Everyone wants to be disabled. Sleep for a longer period of + # time than usual so we don't spin unnecessarily. We might be + # enabled again in the future. + return self.inactive_sleep_time + return max((min_sleep, self.min_sleep_time)) + + def kill(self): + if not self.should_run: + # Prevent overwriting trace functions. + return + # Stop this monitoring thread from running. + self.should_run = False + # Uninstall our tracing hook + self._greenlet_tracer.kill() + + def _on_hub_gc(self, _): + self.kill() + + def __call__(self): + # The function that runs in the monitoring thread. + # We cannot use threading.current_thread because it would + # create an immortal DummyThread object. + getcurrent().gevent_monitoring_thread = wref(self) + + try: + while self.should_run: + functions = self.monitoring_functions() + assert functions + sleep_time = self.calculate_sleep_time() + + thread_sleep(sleep_time) + + # Make sure the hub is still around, and still active, + # and keep it around while we are here. + hub = self.hub + if not hub: + self.kill() + + if self.should_run: + this_run = perf_counter() + for entry in functions: + f = entry.function + period = entry.period + last_run = entry.last_run_time + if period and last_run + period <= this_run: + entry.last_run_time = this_run + f(hub) + del hub # break our reference to hub while we sleep + + except SystemExit: + pass + except: # pylint:disable=bare-except + # We're a daemon thread, so swallow any exceptions that get here + # during interpreter shutdown. + if not sys or not sys.stderr: # pragma: no cover + # Interpreter is shutting down + pass + else: + hub = self.hub + if hub is not None: + # XXX: This tends to do bad things like end the process, because we + # try to switch *threads*, which can't happen. Need something better. + hub.handle_error(self, *sys.exc_info()) + + def monitor_blocking(self, hub): + # Called periodically to see if the trace function has + # fired to switch greenlets. If not, we will print + # the greenlet tree. + + # For tests, we return a true value when we think we found something + # blocking + + did_block = self._greenlet_tracer.did_block_hub(hub) + if not did_block: + return + + active_greenlet = did_block[1] + report = self._greenlet_tracer.did_block_hub_report( + hub, active_greenlet, + dict(greenlet_stacks=False, current_thread_ident=self.monitor_thread_ident)) + + stream = hub.exception_stream + for line in report: + # Printing line by line may interleave with other things, + # but it should also prevent a "reentrant call to print" + # when the report is large. + print(line, file=stream) + + notify(EventLoopBlocked(active_greenlet, GEVENT_CONFIG.max_blocking_time, report)) + return (active_greenlet, report) + + def ignore_current_greenlet_blocking(self): + self._greenlet_tracer.ignore_current_greenlet_blocking() + + def monitor_current_greenlet_blocking(self): + self._greenlet_tracer.monitor_current_greenlet_blocking() + + def _get_process(self): # pylint:disable=method-hidden + try: + # The standard library 'resource' module doesn't provide + # a standard way to get the RSS measure, only the maximum. + # You might be tempted to try to compute something by adding + # together text and data sizes, but on many systems those come back + # zero. So our only option is psutil. + from psutil import Process, AccessDenied + # Make sure it works (why would we be denied access to our own process?) + try: + proc = Process() + proc.memory_full_info() + except AccessDenied: # pragma: no cover + proc = None + except ImportError: + proc = None + + self._get_process = lambda: proc + return proc + + def can_monitor_memory_usage(self): + return self._get_process() is not None + + def install_monitor_memory_usage(self): + # Start monitoring memory usage, if possible. + # If not possible, emit a warning. + if not self.can_monitor_memory_usage(): + import warnings + warnings.warn("Unable to monitor memory usage. Install psutil.", + MonitorWarning) + return + + self.add_monitoring_function(self.monitor_memory_usage, + max(GEVENT_CONFIG.memory_monitor_period, + self.min_memory_monitor_period)) + + def monitor_memory_usage(self, _hub): + max_allowed = GEVENT_CONFIG.max_memory_usage + if not max_allowed: + # They disabled it. + return -1 # value for tests + + rusage = self._get_process().memory_full_info() + # uss only documented available on Windows, Linux, and OS X. + # If not available, fall back to rss as an aproximation. + mem_usage = getattr(rusage, 'uss', 0) or rusage.rss + + event = None # Return value for tests + + if mem_usage > max_allowed: + if mem_usage > self._memory_exceeded: + # We're still growing + event = MemoryUsageThresholdExceeded( + mem_usage, max_allowed, rusage) + notify(event) + self._memory_exceeded = mem_usage + else: + # we're below. Were we above it last time? + if self._memory_exceeded: + event = MemoryUsageUnderThreshold( + mem_usage, max_allowed, rusage, self._memory_exceeded) + notify(event) + self._memory_exceeded = 0 + + return event + + def __repr__(self): + return '<%s at %s in thread %s greenlet %r for %r>' % ( + self.__class__.__name__, + hex(id(self)), + hex(self.monitor_thread_ident), + getcurrent(), + self._hub_wref()) diff --git a/src/gevent/_patcher.py b/src/gevent/_patcher.py new file mode 100644 index 0000000..2ced50c --- /dev/null +++ b/src/gevent/_patcher.py @@ -0,0 +1,127 @@ +# Copyright 2018 gevent. See LICENSE for details. + +# Portions of the following are inspired by code from eventlet. I +# believe they are distinct enough that no eventlet copyright would +# apply (they are not a copy or substantial portion of the eventlot +# code). + +# Added in gevent 1.3a2. Not public in that release. + +from __future__ import absolute_import, print_function + +import importlib +import sys + +from gevent._compat import PY3 +from gevent._compat import iteritems +from gevent._compat import imp_acquire_lock +from gevent._compat import imp_release_lock + + +from gevent.builtins import __import__ as _import + + +MAPPING = { + 'gevent.local': '_threading_local', + 'gevent.socket': 'socket', + 'gevent.select': 'select', + 'gevent.ssl': 'ssl', + 'gevent.thread': '_thread' if PY3 else 'thread', + 'gevent.subprocess': 'subprocess', + 'gevent.os': 'os', + 'gevent.threading': 'threading', + 'gevent.builtins': 'builtins' if PY3 else '__builtin__', + 'gevent.signal': 'signal', + 'gevent.time': 'time', + 'gevent.queue': 'queue' if PY3 else 'Queue', +} + +_PATCH_PREFIX = '__g_patched_module_' + +class _SysModulesPatcher(object): + + def __init__(self, importing): + self._saved = {} + self.importing = importing + self.green_modules = { + stdlib_name: importlib.import_module(gevent_name) + for gevent_name, stdlib_name + in iteritems(MAPPING) + } + self.orig_imported = frozenset(sys.modules) + + def _save(self): + for modname in self.green_modules: + self._saved[modname] = sys.modules.get(modname, None) + + self._saved[self.importing] = sys.modules.get(self.importing, None) + # Anything we've already patched regains its original name during this + # process + for mod_name, mod in iteritems(sys.modules): + if mod_name.startswith(_PATCH_PREFIX): + orig_mod_name = mod_name[len(_PATCH_PREFIX):] + self._saved[mod_name] = sys.modules.get(orig_mod_name, None) + self.green_modules[orig_mod_name] = mod + + def _replace(self): + # Cover the target modules so that when you import the module it + # sees only the patched versions + for name, mod in iteritems(self.green_modules): + sys.modules[name] = mod + + def _restore(self): + for modname, mod in iteritems(self._saved): + if mod is not None: + sys.modules[modname] = mod + else: + try: + del sys.modules[modname] + except KeyError: + pass + # Anything from the same package tree we imported this time + # needs to be saved so we can restore it later, and so it doesn't + # leak into the namespace. + pkg_prefix = self.importing.split('.', 1)[0] + for modname, mod in list(iteritems(sys.modules)): + if (modname not in self.orig_imported + and modname != self.importing + and not modname.startswith(_PATCH_PREFIX) + and modname.startswith(pkg_prefix)): + sys.modules[_PATCH_PREFIX + modname] = mod + del sys.modules[modname] + + def __exit__(self, t, v, tb): + try: + self._restore() + finally: + imp_release_lock() + + def __enter__(self): + imp_acquire_lock() + self._save() + self._replace() + + +def import_patched(module_name): + """ + Import *module_name* with gevent monkey-patches active, + and return the greened module. + + Any sub-modules that were imported by the package are also + saved. + + """ + patched_name = _PATCH_PREFIX + module_name + if patched_name in sys.modules: + return sys.modules[patched_name] + + + # Save the current module state, and restore on exit, + # capturing desirable changes in the modules package. + with _SysModulesPatcher(module_name): + sys.modules.pop(module_name, None) + + module = _import(module_name, {}, {}, module_name.split('.')[:-1]) + sys.modules[patched_name] = module + + return module diff --git a/src/gevent/_queue.pxd b/src/gevent/_queue.pxd new file mode 100644 index 0000000..d4d9da7 --- /dev/null +++ b/src/gevent/_queue.pxd @@ -0,0 +1,74 @@ +cimport cython +from gevent.__waiter cimport Waiter +from gevent._event cimport Event + +cdef _heappush +cdef _heappop +cdef _heapify + +@cython.final +cdef _safe_remove(deq, item) + +@cython.final +@cython.internal +cdef class ItemWaiter(Waiter): + cdef readonly item + cdef readonly queue + +cdef class Queue: + cdef __weakref__ + cdef readonly hub + cdef readonly queue + + cdef getters + cdef putters + + cdef _event_unlock + cdef Py_ssize_t _maxsize + + cpdef _get(self) + cpdef _put(self, item) + cpdef _peek(self) + + cpdef Py_ssize_t qsize(self) + cpdef bint empty(self) + cpdef bint full(self) + + cpdef put(self, item, block=*, timeout=*) + cpdef put_nowait(self, item) + + cdef __get_or_peek(self, method, block, timeout) + + cpdef get(self, block=*, timeout=*) + cpdef get_nowait(self) + cpdef peek(self, block=*, timeout=*) + cpdef peek_nowait(self) + + cdef _schedule_unlock(self) + +@cython.final +cdef class UnboundQueue(Queue): + pass + +cdef class PriorityQueue(Queue): + pass + +cdef class LifoQueue(Queue): + pass + +cdef class JoinableQueue(Queue): + cdef Event _cond + cdef readonly int unfinished_tasks + + +cdef class Channel: + cdef __weakref__ + cdef readonly getters + cdef readonly putters + cdef readonly hub + cdef _event_unlock + + cpdef get(self, block=*, timeout=*) + cpdef get_nowait(self) + + cdef _schedule_unlock(self) diff --git a/src/gevent/_semaphore.py b/src/gevent/_semaphore.py new file mode 100644 index 0000000..d55132e --- /dev/null +++ b/src/gevent/_semaphore.py @@ -0,0 +1,188 @@ +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False +from __future__ import print_function, absolute_import, division + +__all__ = [ + 'Semaphore', + 'BoundedSemaphore', +] + +def _get_linkable(): + x = __import__('gevent._abstract_linkable') + return x._abstract_linkable.AbstractLinkable +locals()['AbstractLinkable'] = _get_linkable() +del _get_linkable + + +class Semaphore(AbstractLinkable): # pylint:disable=undefined-variable + """ + Semaphore(value=1) -> Semaphore + + A semaphore manages a counter representing the number of release() + calls minus the number of acquire() calls, plus an initial value. + The acquire() method blocks if necessary until it can return + without making the counter negative. + + If not given, ``value`` defaults to 1. + + The semaphore is a context manager and can be used in ``with`` statements. + + This Semaphore's ``__exit__`` method does not call the trace function + on CPython, but does under PyPy. + + .. seealso:: :class:`BoundedSemaphore` for a safer version that prevents + some classes of bugs. + + .. versionchanged:: 1.4.0 + + The order in which waiters are awakened is not specified. It was not + specified previously, but usually went in FIFO order. + """ + + def __init__(self, value=1): + if value < 0: + raise ValueError("semaphore initial value must be >= 0") + super(Semaphore, self).__init__() + self.counter = value + self._notify_all = False + + def __str__(self): + params = (self.__class__.__name__, self.counter, self.linkcount()) + return '<%s counter=%s _links[%s]>' % params + + def locked(self): + """Return a boolean indicating whether the semaphore can be acquired. + Most useful with binary semaphores.""" + return self.counter <= 0 + + def release(self): + """ + Release the semaphore, notifying any waiters if needed. + """ + self.counter += 1 + self._check_and_notify() + return self.counter + + def ready(self): + return self.counter > 0 + + def _start_notify(self): + self._check_and_notify() + + def _wait_return_value(self, waited, wait_success): + if waited: + return wait_success + # We didn't even wait, we must be good to go. + # XXX: This is probably dead code, we're careful not to go into the wait + # state if we don't expect to need to + return True + + def wait(self, timeout=None): + """ + wait(timeout=None) -> int + + Wait until it is possible to acquire this semaphore, or until the optional + *timeout* elapses. + + .. caution:: If this semaphore was initialized with a size of 0, + this method will block forever if no timeout is given. + + :keyword float timeout: If given, specifies the maximum amount of seconds + this method will block. + :return: A number indicating how many times the semaphore can be acquired + before blocking. + """ + if self.counter > 0: + return self.counter + + self._wait(timeout) # return value irrelevant, whether we got it or got a timeout + return self.counter + + def acquire(self, blocking=True, timeout=None): + """ + acquire(blocking=True, timeout=None) -> bool + + Acquire the semaphore. + + .. caution:: If this semaphore was initialized with a size of 0, + this method will block forever (unless a timeout is given or blocking is + set to false). + + :keyword bool blocking: If True (the default), this function will block + until the semaphore is acquired. + :keyword float timeout: If given, specifies the maximum amount of seconds + this method will block. + :return: A boolean indicating whether the semaphore was acquired. + If ``blocking`` is True and ``timeout`` is None (the default), then + (so long as this semaphore was initialized with a size greater than 0) + this will always return True. If a timeout was given, and it expired before + the semaphore was acquired, False will be returned. (Note that this can still + raise a ``Timeout`` exception, if some other caller had already started a timer.) + """ + if self.counter > 0: + self.counter -= 1 + return True + + if not blocking: + return False + + success = self._wait(timeout) + if not success: + # Our timer expired. + return False + + # Neither our timer no another one expired, so we blocked until + # awoke. Therefore, the counter is ours + self.counter -= 1 + assert self.counter >= 0 + return True + + _py3k_acquire = acquire # PyPy needs this; it must be static for Cython + + def __enter__(self): + self.acquire() + + def __exit__(self, t, v, tb): + self.release() + + +class BoundedSemaphore(Semaphore): + """ + BoundedSemaphore(value=1) -> BoundedSemaphore + + A bounded semaphore checks to make sure its current value doesn't + exceed its initial value. If it does, :class:`ValueError` is + raised. In most situations semaphores are used to guard resources + with limited capacity. If the semaphore is released too many times + it's a sign of a bug. + + If not given, *value* defaults to 1. + """ + + #: For monkey-patching, allow changing the class of error we raise + _OVER_RELEASE_ERROR = ValueError + + def __init__(self, *args, **kwargs): + Semaphore.__init__(self, *args, **kwargs) + self._initial_value = self.counter + + def release(self): + if self.counter >= self._initial_value: + raise self._OVER_RELEASE_ERROR("Semaphore released too many times") + Semaphore.release(self) + + + +# By building the semaphore with Cython under PyPy, we get +# atomic operations (specifically, exiting/releasing), at the +# cost of some speed (one trivial semaphore micro-benchmark put the pure-python version +# at around 1s and the compiled version at around 4s). Some clever subclassing +# and having only the bare minimum be in cython might help reduce that penalty. +# NOTE: You must use version 0.23.4 or later to avoid a memory leak. +# https://mail.python.org/pipermail/cython-devel/2015-October/004571.html +# However, that's all for naught on up to and including PyPy 4.0.1 which +# have some serious crashing bugs with GC interacting with cython. +# It hasn't been tested since then, and PURE_PYTHON is assumed to be true +# for PyPy in all cases anyway, so this does nothing. + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__semaphore') diff --git a/src/gevent/_socket2.py b/src/gevent/_socket2.py new file mode 100644 index 0000000..5acca41 --- /dev/null +++ b/src/gevent/_socket2.py @@ -0,0 +1,473 @@ +# Copyright (c) 2009-2014 Denis Bilenko and gevent contributors. See LICENSE for details. +""" +Python 2 socket module. +""" +from __future__ import absolute_import + +# Our import magic sadly makes this warning useless +# pylint: disable=undefined-variable + +from gevent import _socketcommon +from gevent._util import copy_globals +from gevent._compat import PYPY +from gevent.timeout import Timeout + +copy_globals(_socketcommon, globals(), + names_to_ignore=_socketcommon.__py3_imports__ + _socketcommon.__extensions__, + dunder_names_to_keep=()) + +__socket__ = _socketcommon.__socket__ +__implements__ = _socketcommon._implements +__extensions__ = _socketcommon.__extensions__ +__imports__ = [i for i in _socketcommon.__imports__ if i not in _socketcommon.__py3_imports__] +__dns__ = _socketcommon.__dns__ +try: + _fileobject = __socket__._fileobject + _socketmethods = __socket__._socketmethods +except AttributeError: + # Allow this module to be imported under Python 3 + # for building the docs + _fileobject = object + _socketmethods = ('bind', 'connect', 'connect_ex', + 'fileno', 'listen', 'getpeername', + 'getsockname', 'getsockopt', + 'setsockopt', 'sendall', + 'setblocking', 'settimeout', + 'gettimeout', 'shutdown') +else: + # Python 2 doesn't natively support with statements on _fileobject; + # but it eases our test cases if we can do the same with on both Py3 + # and Py2. Implementation copied from Python 3 + assert not hasattr(_fileobject, '__enter__') + # we could either patch in place: + #_fileobject.__enter__ = lambda self: self + #_fileobject.__exit__ = lambda self, *args: self.close() if not self.closed else None + # or we could subclass. subclassing has the benefit of not + # changing the behaviour of the stdlib if we're just imported; OTOH, + # under Python 2.6/2.7, test_urllib2net.py asserts that the class IS + # socket._fileobject (sigh), so we have to work around that. + + # We also make it call our custom socket closing method that disposes + # if IO watchers but not the actual socket itself. + + # Python 2 relies on reference counting to close sockets, so this is all + # very ugly and fragile. + + class _fileobject(_fileobject): # pylint:disable=function-redefined + + def __enter__(self): + return self + + def __exit__(self, *args): + if not self.closed: + self.close() + + def close(self): + if self._sock is not None: + self._sock._drop_events() + super(_fileobject, self).close() + + +def _get_memory(data): + try: + mv = memoryview(data) + if mv.shape: + return mv + # No shape, probably working with a ctypes object, + # or something else exotic that supports the buffer interface + return mv.tobytes() + except TypeError: + # fixes "python2.7 array.array doesn't support memoryview used in + # gevent.socket.send" issue + # (http://code.google.com/p/gevent/issues/detail?id=94) + return buffer(data) + + +class _closedsocket(object): + __slots__ = [] + + def _dummy(*args, **kwargs): # pylint:disable=no-method-argument,unused-argument + raise error(EBADF, 'Bad file descriptor') + # All _delegate_methods must also be initialized here. + send = recv = recv_into = sendto = recvfrom = recvfrom_into = _dummy + + if PYPY: + + def _drop(self): + pass + + def _reuse(self): + pass + + __getattr__ = _dummy + + +timeout_default = object() + +from gevent._hub_primitives import wait_on_socket as _wait_on_socket + +class socket(object): + """ + gevent `socket.socket `_ + for Python 2. + + This object should have the same API as the standard library socket linked to above. Not all + methods are specifically documented here; when they are they may point out a difference + to be aware of or may document a method the standard library does not. + """ + + # pylint:disable=too-many-public-methods + + def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None): + if _sock is None: + self._sock = _realsocket(family, type, proto) + self.timeout = _socket.getdefaulttimeout() + else: + if hasattr(_sock, '_sock'): + # passed a gevent socket + self._sock = _sock._sock + self.timeout = getattr(_sock, 'timeout', False) + if self.timeout is False: + self.timeout = _socket.getdefaulttimeout() + else: + # passed a native socket + self._sock = _sock + self.timeout = _socket.getdefaulttimeout() + if PYPY: + self._sock._reuse() + self._sock.setblocking(0) + fileno = self._sock.fileno() + self.hub = get_hub() + io = self.hub.loop.io + self._read_event = io(fileno, 1) + self._write_event = io(fileno, 2) + + def __repr__(self): + return '<%s at %s %s>' % (type(self).__name__, hex(id(self)), self._formatinfo()) + + def __str__(self): + return '<%s %s>' % (type(self).__name__, self._formatinfo()) + + def _formatinfo(self): + # pylint:disable=broad-except + try: + fileno = self.fileno() + except Exception as ex: + fileno = str(ex) + try: + sockname = self.getsockname() + sockname = '%s:%s' % sockname + except Exception: + sockname = None + try: + peername = self.getpeername() + peername = '%s:%s' % peername + except Exception: + peername = None + result = 'fileno=%s' % fileno + if sockname is not None: + result += ' sock=' + str(sockname) + if peername is not None: + result += ' peer=' + str(peername) + if getattr(self, 'timeout', None) is not None: + result += ' timeout=' + str(self.timeout) + return result + + def _get_ref(self): + return self._read_event.ref or self._write_event.ref + + def _set_ref(self, value): + self._read_event.ref = value + self._write_event.ref = value + + ref = property(_get_ref, _set_ref) + + _wait = _wait_on_socket + + def accept(self): + while 1: + try: + client_socket, address = self._sock.accept() + break + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event) + sockobj = socket(_sock=client_socket) + if PYPY: + client_socket._drop() + return sockobj, address + + def _drop_events(self, cancel_wait_ex=cancel_wait_ex): + if self._read_event is not None: + self.hub.cancel_wait(self._read_event, cancel_wait_ex, True) + self._read_event = None + if self._write_event is not None: + self.hub.cancel_wait(self._write_event, cancel_wait_ex, True) + self._write_event = None + + + def close(self, _closedsocket=_closedsocket): + # This function should not reference any globals. See Python issue #808164. + + # Also break any reference to the loop.io objects. Our fileno, + # which they were tied to, is now free to be reused, so these + # objects are no longer functional. + self._drop_events() + s = self._sock + + # Note that we change self._sock at this point. Methods *must not* + # cache `self._sock` separately from self._write_event/self._read_event, + # or they will be out of sync and we may get inappropriate errors. + # (See test__hub:TestCloseSocketWhilePolling for an example). + + self._sock = _closedsocket() + if PYPY: + s._drop() + + @property + def closed(self): + return isinstance(self._sock, _closedsocket) + + def connect(self, address): + if self.timeout == 0.0: + return self._sock.connect(address) + + address = _socketcommon._resolve_addr(self._sock, address) + + timer = Timeout._start_new_or_dummy(self.timeout, timeout('timed out')) + try: + while 1: + err = self._sock.getsockopt(SOL_SOCKET, SO_ERROR) + if err: + raise error(err, strerror(err)) + result = self._sock.connect_ex(address) + if not result or result == EISCONN: + break + elif (result in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (result == EINVAL and is_windows): + self._wait(self._write_event) + else: + raise error(result, strerror(result)) + finally: + timer.close() + + def connect_ex(self, address): + try: + return self.connect(address) or 0 + except timeout: + return EAGAIN + except error as ex: + if type(ex) is error: # pylint:disable=unidiomatic-typecheck + return ex.args[0] + raise # gaierror is not silenced by connect_ex + + def dup(self): + """dup() -> socket object + + Return a new socket object connected to the same system resource. + Note, that the new socket does not inherit the timeout.""" + return socket(_sock=self._sock) + + def makefile(self, mode='r', bufsize=-1): + # Two things to look out for: + # 1) Closing the original socket object should not close the + # fileobject (hence creating a new socket instance); + # An alternate approach is what _socket3.py does, which is to + # keep count of the times makefile objects have been opened (Py3's + # SocketIO helps with that). But the newly created socket, which + # has its own read/write watchers, does need those to be closed + # when the fileobject is; our custom subclass does that. Note that + # we can't pass the 'close=True' argument, as that causes reference counts + # to get screwed up, and Python2 sockets rely on those. + # 2) The resulting fileobject must keep the timeout in order + # to be compatible with the stdlib's socket.makefile. + # Pass self as _sock to preserve timeout. + fobj = _fileobject(type(self)(_sock=self), mode, bufsize) + if PYPY: + self._sock._drop() + return fobj + + def recv(self, *args): + while 1: + try: + return self._sock.recv(*args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + # QQQ without clearing exc_info test__refcount.test_clean_exit fails + sys.exc_clear() + self._wait(self._read_event) + + def recvfrom(self, *args): + while 1: + try: + return self._sock.recvfrom(*args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event) + + def recvfrom_into(self, *args): + while 1: + try: + return self._sock.recvfrom_into(*args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event) + + def recv_into(self, *args): + while 1: + try: + return self._sock.recv_into(*args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event) + + def send(self, data, flags=0, timeout=timeout_default): + if timeout is timeout_default: + timeout = self.timeout + try: + return self._sock.send(data, flags) + except error as ex: + if ex.args[0] not in _socketcommon.GSENDAGAIN or timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._write_event) + try: + return self._sock.send(data, flags) + except error as ex2: + if ex2.args[0] == EWOULDBLOCK: + return 0 + raise + + def sendall(self, data, flags=0): + if isinstance(data, unicode): + data = data.encode() + # this sendall is also reused by gevent.ssl.SSLSocket subclass, + # so it should not call self._sock methods directly + data_memory = _get_memory(data) + return _socketcommon._sendall(self, data_memory, flags) + + def sendto(self, *args): + try: + return self._sock.sendto(*args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._write_event) + try: + return self._sock.sendto(*args) + except error as ex2: + if ex2.args[0] == EWOULDBLOCK: + return 0 + raise + + def setblocking(self, flag): + if flag: + self.timeout = None + else: + self.timeout = 0.0 + + def settimeout(self, howlong): + if howlong is not None: + try: + f = howlong.__float__ + except AttributeError: + raise TypeError('a float is required') + howlong = f() + if howlong < 0.0: + raise ValueError('Timeout value out of range') + self.__dict__['timeout'] = howlong # avoid recursion with any property on self.timeout + + def gettimeout(self): + return self.__dict__['timeout'] # avoid recursion with any property on self.timeout + + def shutdown(self, how): + if how == 0: # SHUT_RD + self.hub.cancel_wait(self._read_event, cancel_wait_ex) + elif how == 1: # SHUT_WR + self.hub.cancel_wait(self._write_event, cancel_wait_ex) + else: + self.hub.cancel_wait(self._read_event, cancel_wait_ex) + self.hub.cancel_wait(self._write_event, cancel_wait_ex) + self._sock.shutdown(how) + + family = property(lambda self: self._sock.family) + type = property(lambda self: self._sock.type) + proto = property(lambda self: self._sock.proto) + + def fileno(self): + return self._sock.fileno() + + def getsockname(self): + return self._sock.getsockname() + + def getpeername(self): + return self._sock.getpeername() + + # delegate the functions that we haven't implemented to the real socket object + + _s = "def %s(self, *args): return self._sock.%s(*args)\n\n" + _m = None + for _m in set(_socketmethods) - set(locals()): + exec(_s % (_m, _m,)) + del _m, _s + + if PYPY: + + def _reuse(self): + self._sock._reuse() + + def _drop(self): + self._sock._drop() + + +SocketType = socket + +if hasattr(_socket, 'socketpair'): + + def socketpair(family=getattr(_socket, 'AF_UNIX', _socket.AF_INET), + type=_socket.SOCK_STREAM, proto=0): + one, two = _socket.socketpair(family, type, proto) + result = socket(_sock=one), socket(_sock=two) + if PYPY: + one._drop() + two._drop() + return result +elif 'socketpair' in __implements__: + __implements__.remove('socketpair') + +if hasattr(_socket, 'fromfd'): + + def fromfd(fd, family, type, proto=0): + s = _socket.fromfd(fd, family, type, proto) + result = socket(_sock=s) + if PYPY: + s._drop() + return result + +elif 'fromfd' in __implements__: + __implements__.remove('fromfd') + +if hasattr(__socket__, 'ssl'): + + def ssl(sock, keyfile=None, certfile=None): + # deprecated in 2.7.9 but still present; + # sometimes backported by distros. See ssl.py + # Note that we import gevent.ssl, not _ssl2, to get the correct + # version. + from gevent import ssl as _sslmod + # wrap_socket is 2.7.9/backport, sslwrap_simple is older. They take + # the same arguments. + wrap = getattr(_sslmod, 'wrap_socket', None) or getattr(_sslmod, 'sslwrap_simple') + return wrap(sock, keyfile, certfile) + __implements__.append('ssl') + +__all__ = __implements__ + __extensions__ + __imports__ diff --git a/src/gevent/_socket3.py b/src/gevent/_socket3.py new file mode 100644 index 0000000..973b5a9 --- /dev/null +++ b/src/gevent/_socket3.py @@ -0,0 +1,756 @@ +# Port of Python 3.3's socket module to gevent +""" +Python 3 socket module. +""" +# Our import magic sadly makes this warning useless +# pylint: disable=undefined-variable +# pylint: disable=too-many-statements,too-many-branches +# pylint: disable=too-many-public-methods,unused-argument +from __future__ import absolute_import +import io +import os +import sys + +from gevent import _socketcommon +from gevent._util import copy_globals +from gevent._compat import PYPY +from gevent.timeout import Timeout +import _socket +from os import dup + + +copy_globals(_socketcommon, globals(), + names_to_ignore=_socketcommon.__extensions__, + dunder_names_to_keep=()) + +try: + from errno import EHOSTUNREACH + from errno import ECONNREFUSED +except ImportError: + EHOSTUNREACH = -1 + ECONNREFUSED = -1 + + +__socket__ = _socketcommon.__socket__ +__implements__ = _socketcommon._implements +__extensions__ = _socketcommon.__extensions__ +__imports__ = _socketcommon.__imports__ +__dns__ = _socketcommon.__dns__ + + +SocketIO = __socket__.SocketIO # pylint:disable=no-member + + +def _get_memory(data): + mv = memoryview(data) + if mv.shape: + return mv + # No shape, probably working with a ctypes object, + # or something else exotic that supports the buffer interface + return mv.tobytes() + +timeout_default = object() + + +class _wrefsocket(_socket.socket): + # Plain stdlib socket.socket objects subclass _socket.socket + # and add weakref ability. The ssl module, for one, counts on this. + # We don't create socket.socket objects (because they may have been + # monkey patched to be the object from this module), but we still + # need to make sure what we do create can be weakrefd. + + __slots__ = ("__weakref__", ) + + if PYPY: + # server.py unwraps the socket object to get the raw _sock; + # it depends on having a timeout property alias, which PyPy does not + # provide. + timeout = property(lambda s: s.gettimeout(), + lambda s, nv: s.settimeout(nv)) + +from gevent._hub_primitives import wait_on_socket as _wait_on_socket + +class socket(object): + """ + gevent `socket.socket `_ + for Python 3. + + This object should have the same API as the standard library socket linked to above. Not all + methods are specifically documented here; when they are they may point out a difference + to be aware of or may document a method the standard library does not. + """ + + # Subclasses can set this to customize the type of the + # native _socket.socket we create. It MUST be a subclass + # of _wrefsocket. (gevent internal usage only) + _gevent_sock_class = _wrefsocket + + _io_refs = 0 + _closed = False + _read_event = None + _write_event = None + + + # Take the same approach as socket2: wrap a real socket object, + # don't subclass it. This lets code that needs the raw _sock (not tied to the hub) + # get it. This shows up in tests like test__example_udp_server. + + if sys.version_info[:2] < (3, 7): + def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None): + self._sock = self._gevent_sock_class(family, type, proto, fileno) + self.timeout = None + self.__init_common() + else: + # In 3.7, socket changed to auto-detecting family, type, and proto + # when given a fileno. + def __init__(self, family=-1, type=-1, proto=-1, fileno=None): + if fileno is None: + if family == -1: + family = AF_INET + if type == -1: + type = SOCK_STREAM + if proto == -1: + proto = 0 + self._sock = self._gevent_sock_class(family, type, proto, fileno) + self.timeout = None + self.__init_common() + + def __init_common(self): + _socket.socket.setblocking(self._sock, False) + fileno = _socket.socket.fileno(self._sock) + self.hub = get_hub() + io_class = self.hub.loop.io + self._read_event = io_class(fileno, 1) + self._write_event = io_class(fileno, 2) + self.timeout = _socket.getdefaulttimeout() + + def __getattr__(self, name): + return getattr(self._sock, name) + + if hasattr(_socket, 'SOCK_NONBLOCK'): + # Only defined under Linux + @property + def type(self): + # See https://github.com/gevent/gevent/pull/399 + if self.timeout != 0.0: + return self._sock.type & ~_socket.SOCK_NONBLOCK # pylint:disable=no-member + return self._sock.type + + def getblocking(self): + """ + Returns whether the socket will approximate blocking + behaviour. + + .. versionadded:: 1.3a2 + Added in Python 3.7. + """ + return self.timeout != 0.0 + + def __enter__(self): + return self + + def __exit__(self, *args): + if not self._closed: + self.close() + + def __repr__(self): + """Wrap __repr__() to reveal the real class name.""" + try: + s = _socket.socket.__repr__(self._sock) + except Exception as ex: # pylint:disable=broad-except + # Observed on Windows Py3.3, printing the repr of a socket + # that just suffered a ConnectionResetError [WinError 10054]: + # "OverflowError: no printf formatter to display the socket descriptor in decimal" + # Not sure what the actual cause is or if there's a better way to handle this + s = '' % ex + + if s.startswith(" socket object + + Return a new socket object connected to the same system resource. + """ + fd = dup(self.fileno()) + sock = self.__class__(self.family, self.type, self.proto, fileno=fd) + sock.settimeout(self.gettimeout()) + return sock + + def accept(self): + """accept() -> (socket object, address info) + + Wait for an incoming connection. Return a new socket + representing the connection, and the address of the client. + For IP sockets, the address info is a pair (hostaddr, port). + """ + while True: + try: + fd, addr = self._accept() + break + except BlockingIOError: + if self.timeout == 0.0: + raise + self._wait(self._read_event) + sock = socket(self.family, self.type, self.proto, fileno=fd) + # Python Issue #7995: if no default timeout is set and the listening + # socket had a (non-zero) timeout, force the new socket in blocking + # mode to override platform-specific socket flags inheritance. + # XXX do we need to do this? + if getdefaulttimeout() is None and self.gettimeout(): + sock.setblocking(True) + return sock, addr + + def makefile(self, mode="r", buffering=None, *, + encoding=None, errors=None, newline=None): + """Return an I/O stream connected to the socket + + The arguments are as for io.open() after the filename, + except the only mode characters supported are 'r', 'w' and 'b'. + The semantics are similar too. + """ + # (XXX refactor to share code?) + for c in mode: + if c not in {"r", "w", "b"}: + raise ValueError("invalid mode %r (only r, w, b allowed)") + writing = "w" in mode + reading = "r" in mode or not writing + assert reading or writing + binary = "b" in mode + rawmode = "" + if reading: + rawmode += "r" + if writing: + rawmode += "w" + raw = SocketIO(self, rawmode) + self._io_refs += 1 + if buffering is None: + buffering = -1 + if buffering < 0: + buffering = io.DEFAULT_BUFFER_SIZE + if buffering == 0: + if not binary: + raise ValueError("unbuffered streams must be binary") + return raw + if reading and writing: + buffer = io.BufferedRWPair(raw, raw, buffering) + elif reading: + buffer = io.BufferedReader(raw, buffering) + else: + assert writing + buffer = io.BufferedWriter(raw, buffering) + if binary: + return buffer + text = io.TextIOWrapper(buffer, encoding, errors, newline) + text.mode = mode + return text + + def _decref_socketios(self): + # Called by SocketIO when it is closed. + if self._io_refs > 0: + self._io_refs -= 1 + if self._closed: + self.close() + + def _drop_events(self): + if self._read_event is not None: + self.hub.cancel_wait(self._read_event, cancel_wait_ex, True) + self._read_event = None + if self._write_event is not None: + self.hub.cancel_wait(self._write_event, cancel_wait_ex, True) + self._write_event = None + + def _real_close(self, _ss=_socket.socket, cancel_wait_ex=cancel_wait_ex): + # This function should not reference any globals. See Python issue #808164. + + # Break any reference to the loop.io objects. Our fileno, + # which they were tied to, is now free to be reused, so these + # objects are no longer functional. + self._drop_events() + + _ss.close(self._sock) + + # Break any references to the underlying socket object. Tested + # by test__refcount. (Why does this matter?). Be sure to + # preserve our same family/type/proto if possible (if we + # don't, we can get TypeError instead of OSError; see + # test_socket.SendmsgUDP6Test.testSendmsgAfterClose)... but + # this isn't always possible (see test_socket.test_unknown_socket_family_repr) + # TODO: Can we use a simpler proxy, like _socket2 does? + try: + self._sock = self._gevent_sock_class(self.family, self.type, self.proto) + except OSError: + pass + else: + _ss.close(self._sock) + + + def close(self): + # This function should not reference any globals. See Python issue #808164. + self._closed = True + if self._io_refs <= 0: + self._real_close() + + @property + def closed(self): + return self._closed + + def detach(self): + """detach() -> file descriptor + + Close the socket object without closing the underlying file descriptor. + The object cannot be used after this call, but the file descriptor + can be reused for other purposes. The file descriptor is returned. + """ + self._closed = True + return self._sock.detach() + + def connect(self, address): + if self.timeout == 0.0: + return _socket.socket.connect(self._sock, address) + address = _socketcommon._resolve_addr(self._sock, address) + + with Timeout._start_new_or_dummy(self.timeout, timeout("timed out")): + while True: + err = self.getsockopt(SOL_SOCKET, SO_ERROR) + if err: + raise error(err, strerror(err)) + result = _socket.socket.connect_ex(self._sock, address) + + if not result or result == EISCONN: + break + elif (result in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (result == EINVAL and is_windows): + self._wait(self._write_event) + else: + if (isinstance(address, tuple) + and address[0] == 'fe80::1' + and result == EHOSTUNREACH): + # On Python 3.7 on mac, we see EHOSTUNREACH + # returned for this link-local address, but it really is + # supposed to be ECONNREFUSED according to the standard library + # tests (test_socket.NetworkConnectionNoServer.test_create_connection) + # (On previous versions, that code passed the '127.0.0.1' IPv4 address, so + # ipv6 link locals were never a factor; 3.7 passes 'localhost'.) + # It is something of a mystery how the stdlib socket code doesn't + # produce EHOSTUNREACH---I (JAM) can't see how socketmodule.c would avoid + # that. The normal connect just calls connect_ex much like we do. + result = ECONNREFUSED + raise error(result, strerror(result)) + + def connect_ex(self, address): + try: + return self.connect(address) or 0 + except timeout: + return EAGAIN + except gaierror: # pylint:disable=try-except-raise + # gaierror/overflowerror/typerror is not silenced by connect_ex; + # gaierror extends OSError (aka error) so catch it first + raise + except error as ex: + # error is now OSError and it has various subclasses. + # Only those that apply to actually connecting are silenced by + # connect_ex. + if ex.errno: + return ex.errno + raise # pragma: no cover + + def recv(self, *args): + while True: + try: + return _socket.socket.recv(self._sock, *args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + self._wait(self._read_event) + + if hasattr(_socket.socket, 'recvmsg'): + # Only on Unix; PyPy 3.5 5.10.0 provides sendmsg and recvmsg, but not + # recvmsg_into (at least on os x) + + def recvmsg(self, *args): + while True: + try: + return _socket.socket.recvmsg(self._sock, *args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + self._wait(self._read_event) + + if hasattr(_socket.socket, 'recvmsg_into'): + + def recvmsg_into(self, *args): + while True: + try: + return _socket.socket.recvmsg_into(self._sock, *args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + self._wait(self._read_event) + + def recvfrom(self, *args): + while True: + try: + return _socket.socket.recvfrom(self._sock, *args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + self._wait(self._read_event) + + def recvfrom_into(self, *args): + while True: + try: + return _socket.socket.recvfrom_into(self._sock, *args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + self._wait(self._read_event) + + def recv_into(self, *args): + while True: + try: + return _socket.socket.recv_into(self._sock, *args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + self._wait(self._read_event) + + def send(self, data, flags=0, timeout=timeout_default): + if timeout is timeout_default: + timeout = self.timeout + try: + return _socket.socket.send(self._sock, data, flags) + except error as ex: + if ex.args[0] not in _socketcommon.GSENDAGAIN or timeout == 0.0: + raise + self._wait(self._write_event) + try: + return _socket.socket.send(self._sock, data, flags) + except error as ex2: + if ex2.args[0] == EWOULDBLOCK: + return 0 + raise + + def sendall(self, data, flags=0): + # XXX Now that we run on PyPy3, see the notes in _socket2.py's sendall() + # and implement that here if needed. + # PyPy3 is not optimized for performance yet, and is known to be slower than + # PyPy2, so it's possibly premature to do this. However, there is a 3.5 test case that + # possibly exposes this in a severe way. + data_memory = _get_memory(data) + return _socketcommon._sendall(self, data_memory, flags) + + def sendto(self, *args): + try: + return _socket.socket.sendto(self._sock, *args) + except error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + self._wait(self._write_event) + try: + return _socket.socket.sendto(self._sock, *args) + except error as ex2: + if ex2.args[0] == EWOULDBLOCK: + return 0 + raise + + if hasattr(_socket.socket, 'sendmsg'): + # Only on Unix + def sendmsg(self, buffers, ancdata=(), flags=0, address=None): + try: + return _socket.socket.sendmsg(self._sock, buffers, ancdata, flags, address) + except error as ex: + if flags & getattr(_socket, 'MSG_DONTWAIT', 0): + # Enable non-blocking behaviour + # XXX: Do all platforms that have sendmsg have MSG_DONTWAIT? + raise + + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + self._wait(self._write_event) + try: + return _socket.socket.sendmsg(self._sock, buffers, ancdata, flags, address) + except error as ex2: + if ex2.args[0] == EWOULDBLOCK: + return 0 + raise + + def setblocking(self, flag): + # Beginning in 3.6.0b3 this is supposed to raise + # if the file descriptor is closed, but the test for it + # involves closing the fileno directly. Since we + # don't touch the fileno here, it doesn't make sense for + # us. + if flag: + self.timeout = None + else: + self.timeout = 0.0 + + def settimeout(self, howlong): + if howlong is not None: + try: + f = howlong.__float__ + except AttributeError: + raise TypeError('a float is required') + howlong = f() + if howlong < 0.0: + raise ValueError('Timeout value out of range') + self.__dict__['timeout'] = howlong + + def gettimeout(self): + return self.__dict__['timeout'] + + def shutdown(self, how): + if how == 0: # SHUT_RD + self.hub.cancel_wait(self._read_event, cancel_wait_ex) + elif how == 1: # SHUT_WR + self.hub.cancel_wait(self._write_event, cancel_wait_ex) + else: + self.hub.cancel_wait(self._read_event, cancel_wait_ex) + self.hub.cancel_wait(self._write_event, cancel_wait_ex) + self._sock.shutdown(how) + + # sendfile: new in 3.5. But there's no real reason to not + # support it everywhere. Note that we can't use os.sendfile() + # because it's not cooperative. + def _sendfile_use_sendfile(self, file, offset=0, count=None): + # This is called directly by tests + raise __socket__._GiveupOnSendfile() # pylint:disable=no-member + + def _sendfile_use_send(self, file, offset=0, count=None): + self._check_sendfile_params(file, offset, count) + if self.gettimeout() == 0: + raise ValueError("non-blocking sockets are not supported") + if offset: + file.seek(offset) + blocksize = min(count, 8192) if count else 8192 + total_sent = 0 + # localize variable access to minimize overhead + file_read = file.read + sock_send = self.send + try: + while True: + if count: + blocksize = min(count - total_sent, blocksize) + if blocksize <= 0: + break + data = memoryview(file_read(blocksize)) + if not data: + break # EOF + while True: + try: + sent = sock_send(data) + except BlockingIOError: + continue + else: + total_sent += sent + if sent < len(data): + data = data[sent:] + else: + break + return total_sent + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset + total_sent) + + def _check_sendfile_params(self, file, offset, count): + if 'b' not in getattr(file, 'mode', 'b'): + raise ValueError("file should be opened in binary mode") + if not self.type & SOCK_STREAM: + raise ValueError("only SOCK_STREAM type sockets are supported") + if count is not None: + if not isinstance(count, int): + raise TypeError( + "count must be a positive integer (got {!r})".format(count)) + if count <= 0: + raise ValueError( + "count must be a positive integer (got {!r})".format(count)) + + def sendfile(self, file, offset=0, count=None): + """sendfile(file[, offset[, count]]) -> sent + + Send a file until EOF is reached by using high-performance + os.sendfile() and return the total number of bytes which + were sent. + *file* must be a regular file object opened in binary mode. + If os.sendfile() is not available (e.g. Windows) or file is + not a regular file socket.send() will be used instead. + *offset* tells from where to start reading the file. + If specified, *count* is the total number of bytes to transmit + as opposed to sending the file until EOF is reached. + File position is updated on return or also in case of error in + which case file.tell() can be used to figure out the number of + bytes which were sent. + The socket must be of SOCK_STREAM type. + Non-blocking sockets are not supported. + + .. versionadded:: 1.1rc4 + Added in Python 3.5, but available under all Python 3 versions in + gevent. + """ + return self._sendfile_use_send(file, offset, count) + + # get/set_inheritable new in 3.4 + if hasattr(os, 'get_inheritable') or hasattr(os, 'get_handle_inheritable'): + # pylint:disable=no-member + if os.name == 'nt': + def get_inheritable(self): + return os.get_handle_inheritable(self.fileno()) + + def set_inheritable(self, inheritable): + os.set_handle_inheritable(self.fileno(), inheritable) + else: + def get_inheritable(self): + return os.get_inheritable(self.fileno()) + + def set_inheritable(self, inheritable): + os.set_inheritable(self.fileno(), inheritable) + _added = "\n\n.. versionadded:: 1.1rc4 Added in Python 3.4" + get_inheritable.__doc__ = "Get the inheritable flag of the socket" + _added + set_inheritable.__doc__ = "Set the inheritable flag of the socket" + _added + del _added + + +if sys.version_info[:2] == (3, 4) and sys.version_info[:3] <= (3, 4, 2): + # Python 3.4, up to and including 3.4.2, had a bug where the + # SocketType enumeration overwrote the SocketType class imported + # from _socket. This was fixed in 3.4.3 (http://bugs.python.org/issue20386 + # and https://github.com/python/cpython/commit/0d2f85f38a9691efdfd1e7285c4262cab7f17db7). + # Prior to that, if we replace SocketType with our own class, the implementation + # of socket.type breaks with "OSError: [Errno 97] Address family not supported by protocol". + # Therefore, on these old versions, we must preserve it as an enum; while this + # seems like it could lead to non-green behaviour, code on those versions + # cannot possibly be using SocketType as a class anyway. + SocketType = __socket__.SocketType # pylint:disable=no-member + # Fixup __all__; note that we get exec'd multiple times during unit tests + if 'SocketType' in __implements__: + __implements__.remove('SocketType') + if 'SocketType' not in __imports__: + __imports__.append('SocketType') +else: + SocketType = socket + + +def fromfd(fd, family, type, proto=0): + """ fromfd(fd, family, type[, proto]) -> socket object + + Create a socket object from a duplicate of the given file + descriptor. The remaining arguments are the same as for socket(). + """ + nfd = dup(fd) + return socket(family, type, proto, nfd) + + +if hasattr(_socket.socket, "share"): + def fromshare(info): + """ fromshare(info) -> socket object + + Create a socket object from a the bytes object returned by + socket.share(pid). + """ + return socket(0, 0, 0, info) + + __implements__.append('fromshare') + +if hasattr(_socket, "socketpair"): + + def socketpair(family=None, type=SOCK_STREAM, proto=0): + """socketpair([family[, type[, proto]]]) -> (socket object, socket object) + + Create a pair of socket objects from the sockets returned by the platform + socketpair() function. + The arguments are the same as for socket() except the default family is + AF_UNIX if defined on the platform; otherwise, the default is AF_INET. + + .. versionchanged:: 1.2 + All Python 3 versions on Windows supply this function (natively + supplied by Python 3.5 and above). + """ + if family is None: + try: + family = AF_UNIX + except NameError: + family = AF_INET + a, b = _socket.socketpair(family, type, proto) + a = socket(family, type, proto, a.detach()) + b = socket(family, type, proto, b.detach()) + return a, b + +else: # pragma: no cover + # Origin: https://gist.github.com/4325783, by Geert Jansen. Public domain. + + # gevent: taken from 3.6 release. Expected to be used only on Win. Added to Win/3.5 + # gevent: for < 3.5, pass the default value of 128 to lsock.listen() + # (3.5+ uses this as a default and the original code passed no value) + + _LOCALHOST = '127.0.0.1' + _LOCALHOST_V6 = '::1' + + def socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): + if family == AF_INET: + host = _LOCALHOST + elif family == AF_INET6: + host = _LOCALHOST_V6 + else: + raise ValueError("Only AF_INET and AF_INET6 socket address families " + "are supported") + if type != SOCK_STREAM: + raise ValueError("Only SOCK_STREAM socket type is supported") + if proto != 0: + raise ValueError("Only protocol zero is supported") + + # We create a connected TCP socket. Note the trick with + # setblocking(False) that prevents us from having to create a thread. + lsock = socket(family, type, proto) + try: + lsock.bind((host, 0)) + lsock.listen(128) + # On IPv6, ignore flow_info and scope_id + addr, port = lsock.getsockname()[:2] + csock = socket(family, type, proto) + try: + csock.setblocking(False) + try: + csock.connect((addr, port)) + except (BlockingIOError, InterruptedError): + pass + csock.setblocking(True) + ssock, _ = lsock.accept() + except: + csock.close() + raise + finally: + lsock.close() + return (ssock, csock) + + if sys.version_info[:2] < (3, 5): + # Not provided natively + if 'socketpair' in __implements__: + # Multiple imports can cause this to be missing if _socketcommon + # was successfully imported, leading to subsequent imports to cause + # ValueError + __implements__.remove('socketpair') + + +if hasattr(__socket__, 'close'): # Python 3.7b1+ + close = __socket__.close # pylint:disable=no-member + __imports__ += ['close'] + +__all__ = __implements__ + __extensions__ + __imports__ diff --git a/src/gevent/_socketcommon.py b/src/gevent/_socketcommon.py new file mode 100644 index 0000000..cd12ac1 --- /dev/null +++ b/src/gevent/_socketcommon.py @@ -0,0 +1,401 @@ +# Copyright (c) 2009-2014 Denis Bilenko and gevent contributors. See LICENSE for details. +from __future__ import absolute_import + +# standard functions and classes that this module re-implements in a gevent-aware way: +_implements = [ + 'create_connection', + 'socket', + 'SocketType', + 'fromfd', + 'socketpair', +] + +__dns__ = [ + 'getaddrinfo', + 'gethostbyname', + 'gethostbyname_ex', + 'gethostbyaddr', + 'getnameinfo', + 'getfqdn', +] + +_implements += __dns__ + +# non-standard functions that this module provides: +__extensions__ = [ + 'cancel_wait', + 'wait_read', + 'wait_write', + 'wait_readwrite', +] + +# standard functions and classes that this module re-imports +__imports__ = [ + 'error', + 'gaierror', + 'herror', + 'htonl', + 'htons', + 'ntohl', + 'ntohs', + 'inet_aton', + 'inet_ntoa', + 'inet_pton', + 'inet_ntop', + 'timeout', + 'gethostname', + 'getprotobyname', + 'getservbyname', + 'getservbyport', + 'getdefaulttimeout', + 'setdefaulttimeout', + # Windows: + 'errorTab', +] + +__py3_imports__ = [ + # Python 3 + 'AddressFamily', + 'SocketKind', + 'CMSG_LEN', + 'CMSG_SPACE', + 'dup', + 'if_indextoname', + 'if_nameindex', + 'if_nametoindex', + 'sethostname', +] + +__imports__.extend(__py3_imports__) + +import time +import sys +from gevent._hub_local import get_hub_noargs as get_hub +from gevent._compat import string_types, integer_types, PY3 +from gevent._util import copy_globals + +is_windows = sys.platform == 'win32' +is_macos = sys.platform == 'darwin' + +# pylint:disable=no-name-in-module,unused-import +if is_windows: + # no such thing as WSAEPERM or error code 10001 according to winsock.h or MSDN + from errno import WSAEINVAL as EINVAL + from errno import WSAEWOULDBLOCK as EWOULDBLOCK + from errno import WSAEINPROGRESS as EINPROGRESS + from errno import WSAEALREADY as EALREADY + from errno import WSAEISCONN as EISCONN + from gevent.win32util import formatError as strerror + EAGAIN = EWOULDBLOCK +else: + from errno import EINVAL + from errno import EWOULDBLOCK + from errno import EINPROGRESS + from errno import EALREADY + from errno import EAGAIN + from errno import EISCONN + from os import strerror + +try: + from errno import EBADF +except ImportError: + EBADF = 9 + +# macOS can return EPROTOTYPE when writing to a socket that is shutting +# Down. Retrying the write should return the expected EPIPE error. +# Downstream classes (like pywsgi) know how to handle/ignore EPIPE. +# This set is used by socket.send() to decide whether the write should +# be retried. The default is to retry only on EWOULDBLOCK. Here we add +# EPROTOTYPE on macOS to handle this platform-specific race condition. +GSENDAGAIN = (EWOULDBLOCK,) +if is_macos: + from errno import EPROTOTYPE + GSENDAGAIN += (EPROTOTYPE,) + +import _socket +_realsocket = _socket.socket +import socket as __socket__ + +_name = _value = None +__imports__ = copy_globals(__socket__, globals(), + only_names=__imports__, + ignore_missing_names=True) + +for _name in __socket__.__all__: + _value = getattr(__socket__, _name) + if isinstance(_value, (integer_types, string_types)): + globals()[_name] = _value + __imports__.append(_name) + +del _name, _value + +_timeout_error = timeout # pylint: disable=undefined-variable + +from gevent import _hub_primitives +_hub_primitives.set_default_timeout_error(_timeout_error) + +wait = _hub_primitives.wait_on_watcher +wait_read = _hub_primitives.wait_read +wait_write = _hub_primitives.wait_write +wait_readwrite = _hub_primitives.wait_readwrite + +#: The exception raised by default on a call to :func:`cancel_wait` +class cancel_wait_ex(error): # pylint: disable=undefined-variable + def __init__(self): + super(cancel_wait_ex, self).__init__( + EBADF, + 'File descriptor was closed in another greenlet') + + +def cancel_wait(watcher, error=cancel_wait_ex): + """See :meth:`gevent.hub.Hub.cancel_wait`""" + get_hub().cancel_wait(watcher, error) + + +def gethostbyname(hostname): + """ + gethostbyname(host) -> address + + Return the IP address (a string of the form '255.255.255.255') for a host. + + .. seealso:: :doc:`/dns` + """ + return get_hub().resolver.gethostbyname(hostname) + + +def gethostbyname_ex(hostname): + """ + gethostbyname_ex(host) -> (name, aliaslist, addresslist) + + Return the true host name, a list of aliases, and a list of IP addresses, + for a host. The host argument is a string giving a host name or IP number. + Resolve host and port into list of address info entries. + + .. seealso:: :doc:`/dns` + """ + return get_hub().resolver.gethostbyname_ex(hostname) + + +def getaddrinfo(host, port, family=0, socktype=0, proto=0, flags=0): + """ + Resolve host and port into list of address info entries. + + Translate the host/port argument into a sequence of 5-tuples that contain + all the necessary arguments for creating a socket connected to that service. + host is a domain name, a string representation of an IPv4/v6 address or + None. port is a string service name such as 'http', a numeric port number or + None. By passing None as the value of host and port, you can pass NULL to + the underlying C API. + + The family, type and proto arguments can be optionally specified in order to + narrow the list of addresses returned. Passing zero as a value for each of + these arguments selects the full range of results. + + .. seealso:: :doc:`/dns` + """ + return get_hub().resolver.getaddrinfo(host, port, family, socktype, proto, flags) + +if PY3: + # The name of the socktype param changed to type in Python 3. + # See https://github.com/gevent/gevent/issues/960 + # Using inspect here to directly detect the condition is painful because we have to + # wrap it with a try/except TypeError because not all Python 2 + # versions can get the args of a builtin; we also have to use a with to suppress + # the deprecation warning. + d = getaddrinfo.__doc__ + + def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): + # pylint:disable=function-redefined, undefined-variable + # Also, on Python 3, we need to translate into the special enums. + # Our lower-level resolvers, including the thread and blocking, which use _socket, + # function simply with integers. + addrlist = get_hub().resolver.getaddrinfo(host, port, family, type, proto, flags) + result = [ + (_intenum_converter(af, AddressFamily), + _intenum_converter(socktype, SocketKind), + proto, canonname, sa) + for af, socktype, proto, canonname, sa + in addrlist + ] + return result + + getaddrinfo.__doc__ = d + del d + + def _intenum_converter(value, enum_klass): + try: + return enum_klass(value) + except ValueError: # pragma: no cover + return value + + +def gethostbyaddr(ip_address): + """ + gethostbyaddr(ip_address) -> (name, aliaslist, addresslist) + + Return the true host name, a list of aliases, and a list of IP addresses, + for a host. The host argument is a string giving a host name or IP number. + + .. seealso:: :doc:`/dns` + """ + return get_hub().resolver.gethostbyaddr(ip_address) + + +def getnameinfo(sockaddr, flags): + """ + getnameinfo(sockaddr, flags) -> (host, port) + + Get host and port for a sockaddr. + + .. seealso:: :doc:`/dns` + """ + return get_hub().resolver.getnameinfo(sockaddr, flags) + + +def getfqdn(name=''): + """Get fully qualified domain name from name. + + An empty argument is interpreted as meaning the local host. + + First the hostname returned by gethostbyaddr() is checked, then + possibly existing aliases. In case no FQDN is available, hostname + from gethostname() is returned. + """ + # pylint: disable=undefined-variable + name = name.strip() + if not name or name == '0.0.0.0': + name = gethostname() + try: + hostname, aliases, _ = gethostbyaddr(name) + except error: + pass + else: + aliases.insert(0, hostname) + for name in aliases: # EWW! pylint:disable=redefined-argument-from-local + if isinstance(name, bytes): + if b'.' in name: + break + elif '.' in name: + break + else: + name = hostname + return name + +def __send_chunk(socket, data_memory, flags, timeleft, end, timeout=_timeout_error): + """ + Send the complete contents of ``data_memory`` before returning. + This is the core loop around :meth:`send`. + + :param timeleft: Either ``None`` if there is no timeout involved, + or a float indicating the timeout to use. + :param end: Either ``None`` if there is no timeout involved, or + a float giving the absolute end time. + :return: An updated value for ``timeleft`` (or None) + :raises timeout: If ``timeleft`` was given and elapsed while + sending this chunk. + """ + data_sent = 0 + len_data_memory = len(data_memory) + started_timer = 0 + while data_sent < len_data_memory: + chunk = data_memory[data_sent:] + if timeleft is None: + data_sent += socket.send(chunk, flags) + elif started_timer and timeleft <= 0: + # Check before sending to guarantee a check + # happens even if each chunk successfully sends its data + # (especially important for SSL sockets since they have large + # buffers). But only do this if we've actually tried to + # send something once to avoid spurious timeouts on non-blocking + # sockets. + raise timeout('timed out') + else: + started_timer = 1 + data_sent += socket.send(chunk, flags, timeout=timeleft) + timeleft = end - time.time() + + return timeleft + +def _sendall(socket, data_memory, flags, + SOL_SOCKET=__socket__.SOL_SOCKET, # pylint:disable=no-member + SO_SNDBUF=__socket__.SO_SNDBUF): # pylint:disable=no-member + """ + Send the *data_memory* (which should be a memoryview) + using the gevent *socket*, performing well on PyPy. + """ + + # On PyPy up through 5.10.0, both PyPy2 and PyPy3, subviews + # (slices) of a memoryview() object copy the underlying bytes the + # first time the builtin socket.send() method is called. On a + # non-blocking socket (that thus calls socket.send() many times) + # with a large input, this results in many repeated copies of an + # ever smaller string, depending on the networking buffering. For + # example, if each send() can process 1MB of a 50MB input, and we + # naively pass the entire remaining subview each time, we'd copy + # 49MB, 48MB, 47MB, etc, thus completely killing performance. To + # workaround this problem, we work in reasonable, fixed-size + # chunks. This results in a 10x improvement to bench_sendall.py, + # while having no measurable impact on CPython (since it doesn't + # copy at all the only extra overhead is a few python function + # calls, which is negligible for large inputs). + + # On one macOS machine, PyPy3 5.10.1 produced ~ 67.53 MB/s before this change, + # and ~ 616.01 MB/s after. + + # See https://bitbucket.org/pypy/pypy/issues/2091/non-blocking-socketsend-slow-gevent + + # Too small of a chunk (the socket's buf size is usually too + # small) results in reduced perf due to *too many* calls to send and too many + # small copies. With a buffer of 143K (the default on my system), for + # example, bench_sendall.py yields ~264MB/s, while using 1MB yields + # ~653MB/s (matching CPython). 1MB is arbitrary and might be better + # chosen, say, to match a page size? + + len_data_memory = len(data_memory) + if not len_data_memory: + # Don't try to send empty data at all, no point, and breaks ssl + # See issue 719 + return 0 + + + chunk_size = max(socket.getsockopt(SOL_SOCKET, SO_SNDBUF), 1024 * 1024) + + data_sent = 0 + end = None + timeleft = None + if socket.timeout is not None: + timeleft = socket.timeout + end = time.time() + timeleft + + while data_sent < len_data_memory: + chunk_end = min(data_sent + chunk_size, len_data_memory) + chunk = data_memory[data_sent:chunk_end] + + timeleft = __send_chunk(socket, chunk, flags, timeleft, end) + data_sent += len(chunk) # Guaranteed it sent the whole thing + +# pylint:disable=no-member +_RESOLVABLE_FAMILIES = (__socket__.AF_INET,) +if __socket__.has_ipv6: + _RESOLVABLE_FAMILIES += (__socket__.AF_INET6,) + +def _resolve_addr(sock, address): + # Internal method: resolve the AF_INET[6] address using + # getaddrinfo. + if sock.family not in _RESOLVABLE_FAMILIES or not isinstance(address, tuple): + return address + # address is (host, port) (ipv4) or (host, port, flowinfo, scopeid) (ipv6). + + # We don't pass the port to getaddrinfo because the C + # socket module doesn't either (on some systems its + # illegal to do that without also passing socket type and + # protocol). Instead we join the port back at the end. + # See https://github.com/gevent/gevent/issues/1252 + host, port = address[:2] + r = getaddrinfo(host, None, sock.family) + address = r[0][-1] + if len(address) == 2: + address = (address[0], port) + else: + address = (address[0], port, address[2], address[3]) + return address diff --git a/src/gevent/_ssl2.py b/src/gevent/_ssl2.py new file mode 100644 index 0000000..051e364 --- /dev/null +++ b/src/gevent/_ssl2.py @@ -0,0 +1,441 @@ +# Wrapper module for _ssl. Written by Bill Janssen. +# Ported to gevent by Denis Bilenko. +""" +SSL wrapper for socket objects on Python 2.7.8 and below. + +For the documentation, refer to :mod:`ssl` module manual. + +This module implements cooperative SSL socket wrappers. + +.. deprecated:: 1.3 + This module is not secure. Support for Python versions + with only this level of SSL will be dropped in gevent 1.4. +""" + +from __future__ import absolute_import +# Our import magic sadly makes this warning useless +# pylint: disable=undefined-variable,arguments-differ,no-member + +import ssl as __ssl__ + +_ssl = __ssl__._ssl + +import sys +import errno +from gevent._socket2 import socket +from gevent.socket import _fileobject, timeout_default +from gevent.socket import error as socket_error, EWOULDBLOCK +from gevent.socket import timeout as _socket_timeout +from gevent._compat import PYPY +from gevent._util import copy_globals + + +__implements__ = [ + 'SSLSocket', + 'wrap_socket', + 'get_server_certificate', + 'sslwrap_simple', +] + +# Import all symbols from Python's ssl.py, except those that we are implementing +# and "private" symbols. +__imports__ = copy_globals(__ssl__, globals(), + # SSLSocket *must* subclass gevent.socket.socket; see issue 597 + names_to_ignore=__implements__ + ['socket'], + dunder_names_to_keep=()) + + +# Py2.6 can get RAND_status added twice +__all__ = list(set(__implements__) | set(__imports__)) +if 'namedtuple' in __all__: + __all__.remove('namedtuple') + +class SSLSocket(socket): + """ + gevent `ssl.SSLSocket `_ + for Pythons < 2.7.9. + """ + + def __init__(self, sock, keyfile=None, certfile=None, + server_side=False, cert_reqs=CERT_NONE, + ssl_version=PROTOCOL_SSLv23, ca_certs=None, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + ciphers=None): + socket.__init__(self, _sock=sock) + + if PYPY: + sock._drop() + + if certfile and not keyfile: + keyfile = certfile + # see if it's connected + try: + socket.getpeername(self) + except socket_error as e: + if e.args[0] != errno.ENOTCONN: + raise + # no, no connection yet + self._sslobj = None + else: + # yes, create the SSL object + if ciphers is None: + self._sslobj = _ssl.sslwrap(self._sock, server_side, + keyfile, certfile, + cert_reqs, ssl_version, ca_certs) + else: + self._sslobj = _ssl.sslwrap(self._sock, server_side, + keyfile, certfile, + cert_reqs, ssl_version, ca_certs, + ciphers) + if do_handshake_on_connect: + self.do_handshake() + self.keyfile = keyfile + self.certfile = certfile + self.cert_reqs = cert_reqs + self.ssl_version = ssl_version + self.ca_certs = ca_certs + self.ciphers = ciphers + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs + self._makefile_refs = 0 + + def read(self, len=1024): + """Read up to LEN bytes and return them. + Return zero-length string on EOF.""" + while True: + try: + return self._sslobj.read(len) + except SSLError as ex: + if ex.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs: + return '' + if ex.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event, timeout_exc=_SSLErrorReadTimeout) + elif ex.args[0] == SSL_ERROR_WANT_WRITE: + if self.timeout == 0.0: + raise + sys.exc_clear() + # note: using _SSLErrorReadTimeout rather than _SSLErrorWriteTimeout below is intentional + self._wait(self._write_event, timeout_exc=_SSLErrorReadTimeout) + else: + raise + + def write(self, data): + """Write DATA to the underlying SSL channel. Returns + number of bytes of DATA actually transmitted.""" + while True: + try: + return self._sslobj.write(data) + except SSLError as ex: + if ex.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event, timeout_exc=_SSLErrorWriteTimeout) + elif ex.args[0] == SSL_ERROR_WANT_WRITE: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._write_event, timeout_exc=_SSLErrorWriteTimeout) + else: + raise + + def getpeercert(self, binary_form=False): + """Returns a formatted version of the data in the + certificate provided by the other end of the SSL channel. + Return None if no certificate was provided, {} if a + certificate was provided, but not validated.""" + return self._sslobj.peer_certificate(binary_form) + + def cipher(self): + if not self._sslobj: + return None + return self._sslobj.cipher() + + def send(self, data, flags=0, timeout=timeout_default): + if timeout is timeout_default: + timeout = self.timeout + if self._sslobj: + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to send() on %s" % + self.__class__) + while True: + try: + v = self._sslobj.write(data) + except SSLError as x: + if x.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + return 0 + sys.exc_clear() + self._wait(self._read_event) + elif x.args[0] == SSL_ERROR_WANT_WRITE: + if self.timeout == 0.0: + return 0 + sys.exc_clear() + self._wait(self._write_event) + else: + raise + else: + return v + else: + return socket.send(self, data, flags, timeout) + # is it possible for sendall() to send some data without encryption if another end shut down SSL? + + def sendall(self, data, flags=0): + try: + socket.sendall(self, data) + except _socket_timeout as ex: + if self.timeout == 0.0: + # Python 2 simply *hangs* in this case, which is bad, but + # Python 3 raises SSLWantWriteError. We do the same. + raise SSLError(SSL_ERROR_WANT_WRITE) + # Convert the socket.timeout back to the sslerror + raise SSLError(*ex.args) + + def sendto(self, *args): + if self._sslobj: + raise ValueError("sendto not allowed on instances of %s" % + self.__class__) + else: + return socket.sendto(self, *args) + + def recv(self, buflen=1024, flags=0): + if self._sslobj: + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to recv() on %s" % + self.__class__) + # QQQ Shouldn't we wrap the SSL_WANT_READ errors as socket.timeout errors to match socket.recv's behavior? + return self.read(buflen) + return socket.recv(self, buflen, flags) + + def recv_into(self, buffer, nbytes=None, flags=0): + if buffer and (nbytes is None): + nbytes = len(buffer) + elif nbytes is None: + nbytes = 1024 + if self._sslobj: + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to recv_into() on %s" % + self.__class__) + while True: + try: + tmp_buffer = self.read(nbytes) + v = len(tmp_buffer) + buffer[:v] = tmp_buffer + return v + except SSLError as x: + if x.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event) + continue + else: + raise + else: + return socket.recv_into(self, buffer, nbytes, flags) + + def recvfrom(self, *args): + if self._sslobj: + raise ValueError("recvfrom not allowed on instances of %s" % + self.__class__) + else: + return socket.recvfrom(self, *args) + + def recvfrom_into(self, *args): + if self._sslobj: + raise ValueError("recvfrom_into not allowed on instances of %s" % + self.__class__) + else: + return socket.recvfrom_into(self, *args) + + def pending(self): + if self._sslobj: + return self._sslobj.pending() + return 0 + + def _sslobj_shutdown(self): + while True: + try: + return self._sslobj.shutdown() + except SSLError as ex: + if ex.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs: + return '' + if ex.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event, timeout_exc=_SSLErrorReadTimeout) + elif ex.args[0] == SSL_ERROR_WANT_WRITE: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._write_event, timeout_exc=_SSLErrorWriteTimeout) + else: + raise + + def unwrap(self): + if not self._sslobj: + raise ValueError("No SSL wrapper around " + str(self)) + s = self._sslobj_shutdown() + self._sslobj = None + return socket(_sock=s) + + def shutdown(self, how): + self._sslobj = None + socket.shutdown(self, how) + + def close(self): + if self._makefile_refs < 1: + self._sslobj = None + socket.close(self) + else: + self._makefile_refs -= 1 + + if PYPY: + + def _reuse(self): + self._makefile_refs += 1 + + def _drop(self): + if self._makefile_refs < 1: + self.close() + else: + self._makefile_refs -= 1 + + def do_handshake(self): + """Perform a TLS/SSL handshake.""" + while True: + try: + return self._sslobj.do_handshake() + except SSLError as ex: + if ex.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event, timeout_exc=_SSLErrorHandshakeTimeout) + elif ex.args[0] == SSL_ERROR_WANT_WRITE: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._write_event, timeout_exc=_SSLErrorHandshakeTimeout) + else: + raise + + def connect(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + # Here we assume that the socket is client-side, and not + # connected at the time of the call. We connect it, then wrap it. + if self._sslobj: + raise ValueError("attempt to connect already-connected SSLSocket!") + socket.connect(self, addr) + if self.ciphers is None: + self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile, + self.cert_reqs, self.ssl_version, + self.ca_certs) + else: + self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile, + self.cert_reqs, self.ssl_version, + self.ca_certs, self.ciphers) + if self.do_handshake_on_connect: + self.do_handshake() + + def accept(self): + """Accepts a new connection from a remote client, and returns + a tuple containing that new connection wrapped with a server-side + SSL channel, and the address of the remote client.""" + sock = self._sock + while True: + try: + client_socket, address = sock.accept() + break + except socket_error as ex: + if ex.args[0] != EWOULDBLOCK or self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event) + + sslobj = SSLSocket(client_socket, + keyfile=self.keyfile, + certfile=self.certfile, + server_side=True, + cert_reqs=self.cert_reqs, + ssl_version=self.ssl_version, + ca_certs=self.ca_certs, + do_handshake_on_connect=self.do_handshake_on_connect, + suppress_ragged_eofs=self.suppress_ragged_eofs, + ciphers=self.ciphers) + + return sslobj, address + + def makefile(self, mode='r', bufsize=-1): + """Make and return a file-like object that + works with the SSL connection. Just use the code + from the socket module.""" + if not PYPY: + self._makefile_refs += 1 + # close=True so as to decrement the reference count when done with + # the file-like object. + return _fileobject(self, mode, bufsize, close=True) + +if PYPY or not hasattr(SSLSocket, 'timeout'): + # PyPy (and certain versions of CPython) doesn't have a direct + # 'timeout' property on raw sockets, because that's not part of + # the documented specification. We may wind up wrapping a raw + # socket (when ssl is used with PyWSGI) or a gevent socket, which + # does have a read/write timeout property as an alias for + # get/settimeout, so make sure that's always the case because + # pywsgi can depend on that. + SSLSocket.timeout = property(lambda self: self.gettimeout(), + lambda self, value: self.settimeout(value)) + + +_SSLErrorReadTimeout = SSLError('The read operation timed out') +_SSLErrorWriteTimeout = SSLError('The write operation timed out') +_SSLErrorHandshakeTimeout = SSLError('The handshake operation timed out') + + +def wrap_socket(sock, keyfile=None, certfile=None, + server_side=False, cert_reqs=CERT_NONE, + ssl_version=PROTOCOL_SSLv23, ca_certs=None, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, ciphers=None): + """Create a new :class:`SSLSocket` instance.""" + return SSLSocket(sock, keyfile=keyfile, certfile=certfile, + server_side=server_side, cert_reqs=cert_reqs, + ssl_version=ssl_version, ca_certs=ca_certs, + do_handshake_on_connect=do_handshake_on_connect, + suppress_ragged_eofs=suppress_ragged_eofs, + ciphers=ciphers) + + +def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv23, ca_certs=None): + """Retrieve the certificate from the server at the specified address, + and return it as a PEM-encoded string. + If 'ca_certs' is specified, validate the server cert against it. + If 'ssl_version' is specified, use it in the connection attempt.""" + + if ca_certs is not None: + cert_reqs = CERT_REQUIRED + else: + cert_reqs = CERT_NONE + s = wrap_socket(socket(), ssl_version=ssl_version, + cert_reqs=cert_reqs, ca_certs=ca_certs) + s.connect(addr) + dercert = s.getpeercert(True) + s.close() + return DER_cert_to_PEM_cert(dercert) + + +def sslwrap_simple(sock, keyfile=None, certfile=None): + """A replacement for the old socket.ssl function. Designed + for compatibility with Python 2.5 and earlier. Will disappear in + Python 3.0.""" + return SSLSocket(sock, keyfile, certfile) diff --git a/src/gevent/_ssl3.py b/src/gevent/_ssl3.py new file mode 100644 index 0000000..aefc823 --- /dev/null +++ b/src/gevent/_ssl3.py @@ -0,0 +1,712 @@ +# Wrapper module for _ssl. Written by Bill Janssen. +# Ported to gevent by Denis Bilenko. +"""SSL wrapper for socket objects on Python 3. + +For the documentation, refer to :mod:`ssl` module manual. + +This module implements cooperative SSL socket wrappers. +""" +# Our import magic sadly makes this warning useless +# pylint: disable=undefined-variable +# pylint:disable=no-member + +from __future__ import absolute_import +import ssl as __ssl__ + +_ssl = __ssl__._ssl + +import errno +from gevent.socket import socket, timeout_default +from gevent.socket import error as socket_error +from gevent.socket import timeout as _socket_timeout +from gevent._util import copy_globals + +from weakref import ref as _wref + +__implements__ = [ + 'SSLContext', + 'SSLSocket', + 'wrap_socket', + 'get_server_certificate', +] + +# Import all symbols from Python's ssl.py, except those that we are implementing +# and "private" symbols. +__imports__ = copy_globals(__ssl__, globals(), + # SSLSocket *must* subclass gevent.socket.socket; see issue 597 + names_to_ignore=__implements__ + ['socket'], + dunder_names_to_keep=()) + +__all__ = __implements__ + __imports__ +if 'namedtuple' in __all__: + __all__.remove('namedtuple') + +orig_SSLContext = __ssl__.SSLContext # pylint:disable=no-member + + +class SSLContext(orig_SSLContext): + + # Added in Python 3.7 + sslsocket_class = None # SSLSocket is assigned later + + def wrap_socket(self, sock, server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + session=None): + # pylint:disable=arguments-differ,not-callable + # (3.6 adds session) + # Sadly, using *args and **kwargs doesn't work + return self.sslsocket_class( + sock=sock, server_side=server_side, + do_handshake_on_connect=do_handshake_on_connect, + suppress_ragged_eofs=suppress_ragged_eofs, + server_hostname=server_hostname, + _context=self, + _session=session) + + if not hasattr(orig_SSLContext, 'check_hostname'): + # Python 3.3 lacks this + check_hostname = False + + if hasattr(orig_SSLContext.options, 'setter'): + # In 3.6, these became properties. They want to access the + # property __set__ method in the superclass, and they do so by using + # super(SSLContext, SSLContext). But we rebind SSLContext when we monkey + # patch, which causes infinite recursion. + # https://github.com/python/cpython/commit/328067c468f82e4ec1b5c510a4e84509e010f296 + # pylint:disable=no-member + @orig_SSLContext.options.setter + def options(self, value): + super(orig_SSLContext, orig_SSLContext).options.__set__(self, value) + + @orig_SSLContext.verify_flags.setter + def verify_flags(self, value): + super(orig_SSLContext, orig_SSLContext).verify_flags.__set__(self, value) + + @orig_SSLContext.verify_mode.setter + def verify_mode(self, value): + super(orig_SSLContext, orig_SSLContext).verify_mode.__set__(self, value) + + if hasattr(orig_SSLContext, 'minimum_version'): + # Like the above, added in 3.7 + @orig_SSLContext.minimum_version.setter + def minimum_version(self, value): + super(orig_SSLContext, orig_SSLContext).minimum_version.__set__(self, value) + + @orig_SSLContext.maximum_version.setter + def maximum_version(self, value): + super(orig_SSLContext, orig_SSLContext).maximum_version.__set__(self, value) + + +class _contextawaresock(socket._gevent_sock_class): + # We have to pass the raw stdlib socket to SSLContext.wrap_socket. + # That method in turn can pass that object on to things like SNI callbacks. + # It wouldn't have access to any of the attributes on the SSLSocket, like + # context, that it's supposed to (see test_ssl.test_sni_callback). Our + # solution is to keep a weak reference to the SSLSocket on the raw + # socket and delegate. + + # We keep it in a slot to avoid having the ability to set any attributes + # we're not prepared for (because we don't know what to delegate.) + + __slots__ = ('_sslsock',) + + @property + def context(self): + return self._sslsock().context + + @context.setter + def context(self, ctx): + self._sslsock().context = ctx + + @property + def session(self): + """The SSLSession for client socket.""" + return self._sslsock().session + + @session.setter + def session(self, session): + self._sslsock().session = session + + def __getattr__(self, name): + try: + return getattr(self._sslsock(), name) + except RuntimeError: + # XXX: If the attribute doesn't exist, + # we infinitely recurse + pass + raise AttributeError(name) + +try: + _SSLObject_factory = SSLObject +except NameError: + # 3.4 and below do not have SSLObject, something + # we magically import through copy_globals + pass +else: + if hasattr(SSLObject, '_create'): + # 3.7 is making thing difficult and won't let you + # actually construct an object + def _SSLObject_factory(sslobj, owner=None, session=None): + s = SSLObject.__new__(SSLObject) + s._sslobj = sslobj + s._sslobj.owner = owner or s + if session is not None: + s._sslobj.session = session + return s + +class SSLSocket(socket): + """ + gevent `ssl.SSLSocket `_ + for Python 3. + """ + + # pylint:disable=too-many-instance-attributes,too-many-public-methods + + _gevent_sock_class = _contextawaresock + + def __init__(self, sock=None, keyfile=None, certfile=None, + server_side=False, cert_reqs=CERT_NONE, + ssl_version=PROTOCOL_SSLv23, ca_certs=None, + do_handshake_on_connect=True, + family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None, + suppress_ragged_eofs=True, npn_protocols=None, ciphers=None, + server_hostname=None, + _session=None, # 3.6 + _context=None): + + # pylint:disable=too-many-locals,too-many-statements,too-many-branches + if _context: + self._context = _context + else: + if server_side and not certfile: + raise ValueError("certfile must be specified for server-side " + "operations") + if keyfile and not certfile: + raise ValueError("certfile must be specified") + if certfile and not keyfile: + keyfile = certfile + self._context = SSLContext(ssl_version) + self._context.verify_mode = cert_reqs + if ca_certs: + self._context.load_verify_locations(ca_certs) + if certfile: + self._context.load_cert_chain(certfile, keyfile) + if npn_protocols: + self._context.set_npn_protocols(npn_protocols) + if ciphers: + self._context.set_ciphers(ciphers) + self.keyfile = keyfile + self.certfile = certfile + self.cert_reqs = cert_reqs + self.ssl_version = ssl_version + self.ca_certs = ca_certs + self.ciphers = ciphers + # Can't use sock.type as other flags (such as SOCK_NONBLOCK) get + # mixed in. + if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM: + raise NotImplementedError("only stream sockets are supported") + if server_side: + if server_hostname: + raise ValueError("server_hostname can only be specified " + "in client mode") + if _session is not None: + raise ValueError("session can only be specified " + "in client mode") + if self._context.check_hostname and not server_hostname: + raise ValueError("check_hostname requires server_hostname") + self._session = _session + self.server_side = server_side + self.server_hostname = server_hostname + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs + connected = False + if sock is not None: + socket.__init__(self, + family=sock.family, + type=sock.type, + proto=sock.proto, + fileno=sock.fileno()) + self.settimeout(sock.gettimeout()) + # see if it's connected + try: + sock.getpeername() + except socket_error as e: + if e.errno != errno.ENOTCONN: + raise + else: + connected = True + sock.detach() + elif fileno is not None: + socket.__init__(self, fileno=fileno) + else: + socket.__init__(self, family=family, type=type, proto=proto) + + self._sock._sslsock = _wref(self) + self._closed = False + self._sslobj = None + self._connected = connected + if connected: + # create the SSL object + try: + self._sslobj = self._context._wrap_socket(self._sock, server_side, + server_hostname) + if _session is not None: # 3.6+ + self._sslobj = _SSLObject_factory(self._sslobj, owner=self, + session=self._session) + if do_handshake_on_connect: + timeout = self.gettimeout() + if timeout == 0.0: + # non-blocking + raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") + self.do_handshake() + + except socket_error as x: + self.close() + raise x + + @property + def context(self): + return self._context + + @context.setter + def context(self, ctx): + self._context = ctx + self._sslobj.context = ctx + + @property + def session(self): + """The SSLSession for client socket.""" + if self._sslobj is not None: + return self._sslobj.session + + @session.setter + def session(self, session): + self._session = session + if self._sslobj is not None: + self._sslobj.session = session + + @property + def session_reused(self): + """Was the client session reused during handshake""" + if self._sslobj is not None: + return self._sslobj.session_reused + + def dup(self): + raise NotImplementedError("Can't dup() %s instances" % + self.__class__.__name__) + + def _checkClosed(self, msg=None): + # raise an exception here if you wish to check for spurious closes + pass + + def _check_connected(self): + if not self._connected: + # getpeername() will raise ENOTCONN if the socket is really + # not connected; note that we can be connected even without + # _connected being set, e.g. if connect() first returned + # EAGAIN. + self.getpeername() + + def read(self, len=1024, buffer=None): + """Read up to LEN bytes and return them. + Return zero-length string on EOF.""" + # pylint:disable=too-many-branches + self._checkClosed() + + while True: + if not self._sslobj: + raise ValueError("Read on closed or unwrapped SSL socket.") + if len == 0: + return b'' if buffer is None else 0 + # Negative lengths are handled natively when the buffer is None + # to raise a ValueError + try: + if buffer is not None: + return self._sslobj.read(len, buffer) + return self._sslobj.read(len or 1024) + except SSLWantReadError: + if self.timeout == 0.0: + raise + self._wait(self._read_event, timeout_exc=_SSLErrorReadTimeout) + except SSLWantWriteError: + if self.timeout == 0.0: + raise + # note: using _SSLErrorReadTimeout rather than _SSLErrorWriteTimeout below is intentional + self._wait(self._write_event, timeout_exc=_SSLErrorReadTimeout) + except SSLError as ex: + if ex.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs: + if buffer is None: + return b'' + return 0 + raise + + def write(self, data): + """Write DATA to the underlying SSL channel. Returns + number of bytes of DATA actually transmitted.""" + self._checkClosed() + + while True: + if not self._sslobj: + raise ValueError("Write on closed or unwrapped SSL socket.") + + try: + return self._sslobj.write(data) + except SSLError as ex: + if ex.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + raise + self._wait(self._read_event, timeout_exc=_SSLErrorWriteTimeout) + elif ex.args[0] == SSL_ERROR_WANT_WRITE: + if self.timeout == 0.0: + raise + self._wait(self._write_event, timeout_exc=_SSLErrorWriteTimeout) + else: + raise + + def getpeercert(self, binary_form=False): + """Returns a formatted version of the data in the + certificate provided by the other end of the SSL channel. + Return None if no certificate was provided, {} if a + certificate was provided, but not validated.""" + + self._checkClosed() + self._check_connected() + try: + c = self._sslobj.peer_certificate + except AttributeError: + # 3.6 + c = self._sslobj.getpeercert + + return c(binary_form) + + def selected_npn_protocol(self): + self._checkClosed() + if not self._sslobj or not _ssl.HAS_NPN: + return None + return self._sslobj.selected_npn_protocol() + + if hasattr(_ssl, 'HAS_ALPN'): + # 3.5+ + def selected_alpn_protocol(self): + self._checkClosed() + if not self._sslobj or not _ssl.HAS_ALPN: # pylint:disable=no-member + return None + return self._sslobj.selected_alpn_protocol() + + def shared_ciphers(self): + """Return a list of ciphers shared by the client during the handshake or + None if this is not a valid server connection. + """ + return self._sslobj.shared_ciphers() + + def version(self): + """Return a string identifying the protocol version used by the + current SSL channel. """ + if not self._sslobj: + return None + return self._sslobj.version() + + # We inherit sendfile from super(); it always uses `send` + + def cipher(self): + self._checkClosed() + if not self._sslobj: + return None + return self._sslobj.cipher() + + def compression(self): + self._checkClosed() + if not self._sslobj: + return None + return self._sslobj.compression() + + def send(self, data, flags=0, timeout=timeout_default): + self._checkClosed() + if timeout is timeout_default: + timeout = self.timeout + if self._sslobj: + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to send() on %s" % + self.__class__) + while True: + try: + return self._sslobj.write(data) + except SSLWantReadError: + if self.timeout == 0.0: + return 0 + self._wait(self._read_event) + except SSLWantWriteError: + if self.timeout == 0.0: + return 0 + self._wait(self._write_event) + else: + return socket.send(self, data, flags, timeout) + + def sendto(self, data, flags_or_addr, addr=None): + self._checkClosed() + if self._sslobj: + raise ValueError("sendto not allowed on instances of %s" % + self.__class__) + elif addr is None: + return socket.sendto(self, data, flags_or_addr) + else: + return socket.sendto(self, data, flags_or_addr, addr) + + def sendmsg(self, *args, **kwargs): + # Ensure programs don't send data unencrypted if they try to + # use this method. + raise NotImplementedError("sendmsg not allowed on instances of %s" % + self.__class__) + + def sendall(self, data, flags=0): + self._checkClosed() + if self._sslobj: + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to sendall() on %s" % + self.__class__) + + try: + return socket.sendall(self, data, flags) + except _socket_timeout: + if self.timeout == 0.0: + # Raised by the stdlib on non-blocking sockets + raise SSLWantWriteError("The operation did not complete (write)") + raise + + def recv(self, buflen=1024, flags=0): + self._checkClosed() + if self._sslobj: + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to recv() on %s" % + self.__class__) + if buflen == 0: + # https://github.com/python/cpython/commit/00915577dd84ba75016400793bf547666e6b29b5 + # Python #23804 + return b'' + return self.read(buflen) + return socket.recv(self, buflen, flags) + + def recv_into(self, buffer, nbytes=None, flags=0): + self._checkClosed() + if buffer and (nbytes is None): + nbytes = len(buffer) + elif nbytes is None: + nbytes = 1024 + if self._sslobj: + if flags != 0: + raise ValueError("non-zero flags not allowed in calls to recv_into() on %s" % self.__class__) + return self.read(nbytes, buffer) + return socket.recv_into(self, buffer, nbytes, flags) + + def recvfrom(self, buflen=1024, flags=0): + self._checkClosed() + if self._sslobj: + raise ValueError("recvfrom not allowed on instances of %s" % + self.__class__) + else: + return socket.recvfrom(self, buflen, flags) + + def recvfrom_into(self, buffer, nbytes=None, flags=0): + self._checkClosed() + if self._sslobj: + raise ValueError("recvfrom_into not allowed on instances of %s" % + self.__class__) + else: + return socket.recvfrom_into(self, buffer, nbytes, flags) + + def recvmsg(self, *args, **kwargs): + raise NotImplementedError("recvmsg not allowed on instances of %s" % + self.__class__) + + def recvmsg_into(self, *args, **kwargs): + raise NotImplementedError("recvmsg_into not allowed on instances of " + "%s" % self.__class__) + + def pending(self): + self._checkClosed() + if self._sslobj: + return self._sslobj.pending() + return 0 + + def shutdown(self, how): + self._checkClosed() + self._sslobj = None + socket.shutdown(self, how) + + def unwrap(self): + if not self._sslobj: + raise ValueError("No SSL wrapper around " + str(self)) + + while True: + try: + s = self._sslobj.shutdown() + break + except SSLWantReadError: + # Callers of this method expect to get a socket + # back, so we can't simply return 0, we have + # to let these be raised + if self.timeout == 0.0: + raise + self._wait(self._read_event) + except SSLWantWriteError: + if self.timeout == 0.0: + raise + self._wait(self._write_event) + + self._sslobj = None + + # The return value of shutting down the SSLObject is the + # original wrapped socket passed to _wrap_socket, i.e., + # _contextawaresock. But that object doesn't have the + # gevent wrapper around it so it can't be used. We have to + # wrap it back up with a gevent wrapper. + assert s is self._sock + # In the stdlib, SSLSocket subclasses socket.socket and passes itself + # to _wrap_socket, so it gets itself back. We can't do that, we have to + # pass our subclass of _socket.socket, _contextawaresock. + # So ultimately we should return ourself. + + # See test_ftplib.py:TestTLS_FTPClass.test_ccc + return self + + def _real_close(self): + self._sslobj = None + # self._closed = True + socket._real_close(self) + + def do_handshake(self): + """Perform a TLS/SSL handshake.""" + self._check_connected() + while True: + try: + self._sslobj.do_handshake() + break + except SSLWantReadError: + if self.timeout == 0.0: + raise + self._wait(self._read_event, timeout_exc=_SSLErrorHandshakeTimeout) + except SSLWantWriteError: + if self.timeout == 0.0: + raise + self._wait(self._write_event, timeout_exc=_SSLErrorHandshakeTimeout) + + if sys.version_info[:2] < (3, 7) and self._context.check_hostname: + # In Python 3.7, the underlying OpenSSL name matching is used. + # The version implemented in Python doesn't understand IDNA encoding. + if not self.server_hostname: + raise ValueError("check_hostname needs server_hostname " + "argument") + match_hostname(self.getpeercert(), self.server_hostname) + + def _real_connect(self, addr, connect_ex): + if self.server_side: + raise ValueError("can't connect in server-side mode") + # Here we assume that the socket is client-side, and not + # connected at the time of the call. We connect it, then wrap it. + if self._connected: + raise ValueError("attempt to connect already-connected SSLSocket!") + self._sslobj = self._context._wrap_socket(self._sock, False, self.server_hostname) + if self._session is not None: # 3.6+ + self._sslobj = _SSLObject_factory(self._sslobj, owner=self, session=self._session) + try: + if connect_ex: + rc = socket.connect_ex(self, addr) + else: + rc = None + socket.connect(self, addr) + if not rc: + if self.do_handshake_on_connect: + self.do_handshake() + self._connected = True + return rc + except socket_error: + self._sslobj = None + raise + + def connect(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + self._real_connect(addr, False) + + def connect_ex(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + return self._real_connect(addr, True) + + def accept(self): + """Accepts a new connection from a remote client, and returns + a tuple containing that new connection wrapped with a server-side + SSL channel, and the address of the remote client.""" + + newsock, addr = socket.accept(self) + newsock._drop_events() + newsock = self._context.wrap_socket(newsock, + do_handshake_on_connect=self.do_handshake_on_connect, + suppress_ragged_eofs=self.suppress_ragged_eofs, + server_side=True) + return newsock, addr + + def get_channel_binding(self, cb_type="tls-unique"): + """Get channel binding data for current connection. Raise ValueError + if the requested `cb_type` is not supported. Return bytes of the data + or None if the data is not available (e.g. before the handshake). + """ + if hasattr(self._sslobj, 'get_channel_binding'): + # 3.7+, and sslobj is not None + return self._sslobj.get_channel_binding(cb_type) + if cb_type not in CHANNEL_BINDING_TYPES: + raise ValueError("Unsupported channel binding type") + if cb_type != "tls-unique": + raise NotImplementedError("{0} channel binding type not implemented".format(cb_type)) + if self._sslobj is None: + return None + return self._sslobj.tls_unique_cb() + + +# Python does not support forward declaration of types +SSLContext.sslsocket_class = SSLSocket + +# Python 3.2 onwards raise normal timeout errors, not SSLError. +# See https://bugs.python.org/issue10272 +_SSLErrorReadTimeout = _socket_timeout('The read operation timed out') +_SSLErrorWriteTimeout = _socket_timeout('The write operation timed out') +_SSLErrorHandshakeTimeout = _socket_timeout('The handshake operation timed out') + + +def wrap_socket(sock, keyfile=None, certfile=None, + server_side=False, cert_reqs=CERT_NONE, + ssl_version=PROTOCOL_SSLv23, ca_certs=None, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + ciphers=None): + + return SSLSocket(sock=sock, keyfile=keyfile, certfile=certfile, + server_side=server_side, cert_reqs=cert_reqs, + ssl_version=ssl_version, ca_certs=ca_certs, + do_handshake_on_connect=do_handshake_on_connect, + suppress_ragged_eofs=suppress_ragged_eofs, + ciphers=ciphers) + + +def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv23, ca_certs=None): + """Retrieve the certificate from the server at the specified address, + and return it as a PEM-encoded string. + If 'ca_certs' is specified, validate the server cert against it. + If 'ssl_version' is specified, use it in the connection attempt.""" + + _, _ = addr + if ca_certs is not None: + cert_reqs = CERT_REQUIRED + else: + cert_reqs = CERT_NONE + s = create_connection(addr) + s = wrap_socket(s, ssl_version=ssl_version, + cert_reqs=cert_reqs, ca_certs=ca_certs) + dercert = s.getpeercert(True) + s.close() + return DER_cert_to_PEM_cert(dercert) diff --git a/src/gevent/_sslgte279.py b/src/gevent/_sslgte279.py new file mode 100644 index 0000000..5580762 --- /dev/null +++ b/src/gevent/_sslgte279.py @@ -0,0 +1,714 @@ +# Wrapper module for _ssl. Written by Bill Janssen. +# Ported to gevent by Denis Bilenko. +"""SSL wrapper for socket objects on Python 2.7.9 and above. + +For the documentation, refer to :mod:`ssl` module manual. + +This module implements cooperative SSL socket wrappers. +""" + +from __future__ import absolute_import +# Our import magic sadly makes this warning useless +# pylint: disable=undefined-variable +# pylint: disable=too-many-instance-attributes,too-many-locals,too-many-statements,too-many-branches +# pylint: disable=arguments-differ,too-many-public-methods + +import ssl as __ssl__ + +_ssl = __ssl__._ssl # pylint:disable=no-member + +import errno +from gevent._socket2 import socket +from gevent.socket import timeout_default +from gevent.socket import create_connection +from gevent.socket import error as socket_error +from gevent.socket import timeout as _socket_timeout +from gevent._compat import PYPY +from gevent._util import copy_globals + +__implements__ = [ + 'SSLContext', + 'SSLSocket', + 'wrap_socket', + 'get_server_certificate', + 'create_default_context', + '_create_unverified_context', + '_create_default_https_context', + '_create_stdlib_context', +] + +# Import all symbols from Python's ssl.py, except those that we are implementing +# and "private" symbols. +__imports__ = copy_globals(__ssl__, globals(), + # SSLSocket *must* subclass gevent.socket.socket; see issue 597 and 801 + names_to_ignore=__implements__ + ['socket', 'create_connection'], + dunder_names_to_keep=()) + +try: + _delegate_methods +except NameError: # PyPy doesn't expose this detail + _delegate_methods = ('recv', 'recvfrom', 'recv_into', 'recvfrom_into', 'send', 'sendto') + +__all__ = __implements__ + __imports__ +if 'namedtuple' in __all__: + __all__.remove('namedtuple') + +orig_SSLContext = __ssl__.SSLContext # pylint: disable=no-member + + +class SSLContext(orig_SSLContext): + def wrap_socket(self, sock, server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None): + return SSLSocket(sock=sock, server_side=server_side, + do_handshake_on_connect=do_handshake_on_connect, + suppress_ragged_eofs=suppress_ragged_eofs, + server_hostname=server_hostname, + _context=self) + + +def create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, + capath=None, cadata=None): + """Create a SSLContext object with default settings. + + NOTE: The protocol and settings may change anytime without prior + deprecation. The values represent a fair balance between maximum + compatibility and security. + """ + if not isinstance(purpose, _ASN1Object): + raise TypeError(purpose) + + context = SSLContext(PROTOCOL_SSLv23) + + # SSLv2 considered harmful. + context.options |= OP_NO_SSLv2 + + # SSLv3 has problematic security and is only required for really old + # clients such as IE6 on Windows XP + context.options |= OP_NO_SSLv3 + + # disable compression to prevent CRIME attacks (OpenSSL 1.0+) + context.options |= getattr(_ssl, "OP_NO_COMPRESSION", 0) + + if purpose == Purpose.SERVER_AUTH: + # verify certs and host name in client mode + context.verify_mode = CERT_REQUIRED + context.check_hostname = True # pylint: disable=attribute-defined-outside-init + elif purpose == Purpose.CLIENT_AUTH: + # Prefer the server's ciphers by default so that we get stronger + # encryption + context.options |= getattr(_ssl, "OP_CIPHER_SERVER_PREFERENCE", 0) + + # Use single use keys in order to improve forward secrecy + context.options |= getattr(_ssl, "OP_SINGLE_DH_USE", 0) + context.options |= getattr(_ssl, "OP_SINGLE_ECDH_USE", 0) + + # disallow ciphers with known vulnerabilities + context.set_ciphers(_RESTRICTED_SERVER_CIPHERS) + + if cafile or capath or cadata: + context.load_verify_locations(cafile, capath, cadata) + elif context.verify_mode != CERT_NONE: + # no explicit cafile, capath or cadata but the verify mode is + # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system + # root CA certificates for the given purpose. This may fail silently. + context.load_default_certs(purpose) + return context + +def _create_unverified_context(protocol=PROTOCOL_SSLv23, cert_reqs=None, + check_hostname=False, purpose=Purpose.SERVER_AUTH, + certfile=None, keyfile=None, + cafile=None, capath=None, cadata=None): + """Create a SSLContext object for Python stdlib modules + + All Python stdlib modules shall use this function to create SSLContext + objects in order to keep common settings in one place. The configuration + is less restrict than create_default_context()'s to increase backward + compatibility. + """ + if not isinstance(purpose, _ASN1Object): + raise TypeError(purpose) + + context = SSLContext(protocol) + # SSLv2 considered harmful. + context.options |= OP_NO_SSLv2 + # SSLv3 has problematic security and is only required for really old + # clients such as IE6 on Windows XP + context.options |= OP_NO_SSLv3 + + if cert_reqs is not None: + context.verify_mode = cert_reqs + context.check_hostname = check_hostname # pylint: disable=attribute-defined-outside-init + + if keyfile and not certfile: + raise ValueError("certfile must be specified") + if certfile or keyfile: + context.load_cert_chain(certfile, keyfile) + + # load CA root certs + if cafile or capath or cadata: + context.load_verify_locations(cafile, capath, cadata) + elif context.verify_mode != CERT_NONE: + # no explicit cafile, capath or cadata but the verify mode is + # CERT_OPTIONAL or CERT_REQUIRED. Let's try to load default system + # root CA certificates for the given purpose. This may fail silently. + context.load_default_certs(purpose) + + return context + +# Used by http.client if no context is explicitly passed. +_create_default_https_context = create_default_context + + +# Backwards compatibility alias, even though it's not a public name. +_create_stdlib_context = _create_unverified_context + +class SSLSocket(socket): + """ + gevent `ssl.SSLSocket `_ + for Pythons >= 2.7.9 but less than 3. + """ + + def __init__(self, sock=None, keyfile=None, certfile=None, + server_side=False, cert_reqs=CERT_NONE, + ssl_version=PROTOCOL_SSLv23, ca_certs=None, + do_handshake_on_connect=True, + family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None, + suppress_ragged_eofs=True, npn_protocols=None, ciphers=None, + server_hostname=None, + _context=None): + # fileno is ignored + # pylint: disable=unused-argument + if _context: + self._context = _context + else: + if server_side and not certfile: + raise ValueError("certfile must be specified for server-side " + "operations") + if keyfile and not certfile: + raise ValueError("certfile must be specified") + if certfile and not keyfile: + keyfile = certfile + self._context = SSLContext(ssl_version) + self._context.verify_mode = cert_reqs + if ca_certs: + self._context.load_verify_locations(ca_certs) + if certfile: + self._context.load_cert_chain(certfile, keyfile) + if npn_protocols: + self._context.set_npn_protocols(npn_protocols) + if ciphers: + self._context.set_ciphers(ciphers) + self.keyfile = keyfile + self.certfile = certfile + self.cert_reqs = cert_reqs + self.ssl_version = ssl_version + self.ca_certs = ca_certs + self.ciphers = ciphers + # Can't use sock.type as other flags (such as SOCK_NONBLOCK) get + # mixed in. + if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM: + raise NotImplementedError("only stream sockets are supported") + + if PYPY: + socket.__init__(self, _sock=sock) + sock._drop() + else: + # CPython: XXX: Must pass the underlying socket, not our + # potential wrapper; test___example_servers fails the SSL test + # with a client-side EOF error. (Why?) + socket.__init__(self, _sock=sock._sock) + + # The initializer for socket overrides the methods send(), recv(), etc. + # in the instance, which we don't need -- but we want to provide the + # methods defined in SSLSocket. + for attr in _delegate_methods: + try: + delattr(self, attr) + except AttributeError: + pass + if server_side and server_hostname: + raise ValueError("server_hostname can only be specified " + "in client mode") + if self._context.check_hostname and not server_hostname: + raise ValueError("check_hostname requires server_hostname") + self.server_side = server_side + self.server_hostname = server_hostname + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs + self.settimeout(sock.gettimeout()) + + # See if we are connected + try: + self.getpeername() + except socket_error as e: + if e.errno != errno.ENOTCONN: + raise + connected = False + else: + connected = True + + self._makefile_refs = 0 + self._closed = False + self._sslobj = None + self._connected = connected + if connected: + # create the SSL object + try: + self._sslobj = self._context._wrap_socket(self._sock, server_side, + server_hostname, ssl_sock=self) + if do_handshake_on_connect: + timeout = self.gettimeout() + if timeout == 0.0: + # non-blocking + raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") + self.do_handshake() + + except socket_error as x: + self.close() + raise x + + + @property + def context(self): + return self._context + + @context.setter + def context(self, ctx): + self._context = ctx + self._sslobj.context = ctx + + def dup(self): + raise NotImplementedError("Can't dup() %s instances" % + self.__class__.__name__) + + def _checkClosed(self, msg=None): + # raise an exception here if you wish to check for spurious closes + pass + + def _check_connected(self): + if not self._connected: + # getpeername() will raise ENOTCONN if the socket is really + # not connected; note that we can be connected even without + # _connected being set, e.g. if connect() first returned + # EAGAIN. + self.getpeername() + + def read(self, len=1024, buffer=None): + """Read up to LEN bytes and return them. + Return zero-length string on EOF.""" + self._checkClosed() + + while 1: + if not self._sslobj: + raise ValueError("Read on closed or unwrapped SSL socket.") + if len == 0: + return b'' if buffer is None else 0 + if len < 0 and buffer is None: + # This is handled natively in python 2.7.12+ + raise ValueError("Negative read length") + try: + if buffer is not None: + return self._sslobj.read(len, buffer) + return self._sslobj.read(len or 1024) + except SSLWantReadError: + if self.timeout == 0.0: + raise + self._wait(self._read_event, timeout_exc=_SSLErrorReadTimeout) + except SSLWantWriteError: + if self.timeout == 0.0: + raise + # note: using _SSLErrorReadTimeout rather than _SSLErrorWriteTimeout below is intentional + self._wait(self._write_event, timeout_exc=_SSLErrorReadTimeout) + except SSLError as ex: + if ex.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs: + if buffer is not None: + return 0 + return b'' + raise + + def write(self, data): + """Write DATA to the underlying SSL channel. Returns + number of bytes of DATA actually transmitted.""" + self._checkClosed() + + while 1: + if not self._sslobj: + raise ValueError("Write on closed or unwrapped SSL socket.") + + try: + return self._sslobj.write(data) + except SSLError as ex: + if ex.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + raise + self._wait(self._read_event, timeout_exc=_SSLErrorWriteTimeout) + elif ex.args[0] == SSL_ERROR_WANT_WRITE: + if self.timeout == 0.0: + raise + self._wait(self._write_event, timeout_exc=_SSLErrorWriteTimeout) + else: + raise + + def getpeercert(self, binary_form=False): + """Returns a formatted version of the data in the + certificate provided by the other end of the SSL channel. + Return None if no certificate was provided, {} if a + certificate was provided, but not validated.""" + + self._checkClosed() + self._check_connected() + return self._sslobj.peer_certificate(binary_form) + + def selected_npn_protocol(self): + self._checkClosed() + if not self._sslobj or not _ssl.HAS_NPN: + return None + return self._sslobj.selected_npn_protocol() + + if hasattr(_ssl, 'HAS_ALPN'): + # 2.7.10+ + def selected_alpn_protocol(self): + self._checkClosed() + if not self._sslobj or not _ssl.HAS_ALPN: # pylint:disable=no-member + return None + return self._sslobj.selected_alpn_protocol() + + def cipher(self): + self._checkClosed() + if not self._sslobj: + return None + return self._sslobj.cipher() + + def compression(self): + self._checkClosed() + if not self._sslobj: + return None + return self._sslobj.compression() + + def __check_flags(self, meth, flags): + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to %s on %s" % + (meth, self.__class__)) + + def send(self, data, flags=0, timeout=timeout_default): + self._checkClosed() + self.__check_flags('send', flags) + + if timeout is timeout_default: + timeout = self.timeout + + if not self._sslobj: + return socket.send(self, data, flags, timeout) + + while True: + try: + return self._sslobj.write(data) + except SSLWantReadError: + if self.timeout == 0.0: + return 0 + self._wait(self._read_event) + except SSLWantWriteError: + if self.timeout == 0.0: + return 0 + self._wait(self._write_event) + + def sendto(self, data, flags_or_addr, addr=None): + self._checkClosed() + if self._sslobj: + raise ValueError("sendto not allowed on instances of %s" % + self.__class__) + elif addr is None: + return socket.sendto(self, data, flags_or_addr) + else: + return socket.sendto(self, data, flags_or_addr, addr) + + def sendmsg(self, *args, **kwargs): + # Ensure programs don't send data unencrypted if they try to + # use this method. + raise NotImplementedError("sendmsg not allowed on instances of %s" % + self.__class__) + + def sendall(self, data, flags=0): + self._checkClosed() + self.__check_flags('sendall', flags) + + try: + socket.sendall(self, data) + except _socket_timeout as ex: + if self.timeout == 0.0: + # Python 2 simply *hangs* in this case, which is bad, but + # Python 3 raises SSLWantWriteError. We do the same. + raise SSLWantWriteError("The operation did not complete (write)") + # Convert the socket.timeout back to the sslerror + raise SSLError(*ex.args) + + def recv(self, buflen=1024, flags=0): + self._checkClosed() + if self._sslobj: + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to recv() on %s" % + self.__class__) + if buflen == 0: + return b'' + return self.read(buflen) + return socket.recv(self, buflen, flags) + + def recv_into(self, buffer, nbytes=None, flags=0): + self._checkClosed() + if buffer is not None and (nbytes is None): + # Fix for python bug #23804: bool(bytearray()) is False, + # but we should read 0 bytes. + nbytes = len(buffer) + elif nbytes is None: + nbytes = 1024 + if self._sslobj: + if flags != 0: + raise ValueError( + "non-zero flags not allowed in calls to recv_into() on %s" % + self.__class__) + return self.read(nbytes, buffer) + return socket.recv_into(self, buffer, nbytes, flags) + + def recvfrom(self, buflen=1024, flags=0): + self._checkClosed() + if self._sslobj: + raise ValueError("recvfrom not allowed on instances of %s" % + self.__class__) + return socket.recvfrom(self, buflen, flags) + + def recvfrom_into(self, buffer, nbytes=None, flags=0): + self._checkClosed() + if self._sslobj: + raise ValueError("recvfrom_into not allowed on instances of %s" % + self.__class__) + else: + return socket.recvfrom_into(self, buffer, nbytes, flags) + + def recvmsg(self, *args, **kwargs): + raise NotImplementedError("recvmsg not allowed on instances of %s" % + self.__class__) + + def recvmsg_into(self, *args, **kwargs): + raise NotImplementedError("recvmsg_into not allowed on instances of " + "%s" % self.__class__) + + def pending(self): + self._checkClosed() + if self._sslobj: + return self._sslobj.pending() + return 0 + + def shutdown(self, how): + self._checkClosed() + self._sslobj = None + socket.shutdown(self, how) + + def close(self): + if self._makefile_refs < 1: + self._sslobj = None + socket.close(self) + else: + self._makefile_refs -= 1 + + if PYPY: + + def _reuse(self): + self._makefile_refs += 1 + + def _drop(self): + if self._makefile_refs < 1: + self.close() + else: + self._makefile_refs -= 1 + + def _sslobj_shutdown(self): + while True: + try: + return self._sslobj.shutdown() + except SSLError as ex: + if ex.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs: + return '' + if ex.args[0] == SSL_ERROR_WANT_READ: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._read_event, timeout_exc=_SSLErrorReadTimeout) + elif ex.args[0] == SSL_ERROR_WANT_WRITE: + if self.timeout == 0.0: + raise + sys.exc_clear() + self._wait(self._write_event, timeout_exc=_SSLErrorWriteTimeout) + else: + raise + + def unwrap(self): + if not self._sslobj: + raise ValueError("No SSL wrapper around " + str(self)) + + s = self._sslobj_shutdown() + self._sslobj = None + # match _ssl2; critical to drop/reuse here on PyPy + # XXX: _ssl3 returns an SSLSocket. Is that what the standard lib does on + # Python 2? Should we do that? + return socket(_sock=s) + + def _real_close(self): + self._sslobj = None + socket._real_close(self) # pylint: disable=no-member + + def do_handshake(self): + """Perform a TLS/SSL handshake.""" + self._check_connected() + while True: + try: + self._sslobj.do_handshake() + break + except SSLWantReadError: + if self.timeout == 0.0: + raise + self._wait(self._read_event, timeout_exc=_SSLErrorHandshakeTimeout) + except SSLWantWriteError: + if self.timeout == 0.0: + raise + self._wait(self._write_event, timeout_exc=_SSLErrorHandshakeTimeout) + + if self._context.check_hostname: + if not self.server_hostname: + raise ValueError("check_hostname needs server_hostname " + "argument") + match_hostname(self.getpeercert(), self.server_hostname) + + def _real_connect(self, addr, connect_ex): + if self.server_side: + raise ValueError("can't connect in server-side mode") + # Here we assume that the socket is client-side, and not + # connected at the time of the call. We connect it, then wrap it. + if self._connected: + raise ValueError("attempt to connect already-connected SSLSocket!") + self._sslobj = self._context._wrap_socket(self._sock, False, self.server_hostname, ssl_sock=self) + try: + if connect_ex: + rc = socket.connect_ex(self, addr) + else: + rc = None + socket.connect(self, addr) + if not rc: + self._connected = True + if self.do_handshake_on_connect: + self.do_handshake() + return rc + except socket_error: + self._sslobj = None + raise + + def connect(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + self._real_connect(addr, False) + + def connect_ex(self, addr): + """Connects to remote ADDR, and then wraps the connection in + an SSL channel.""" + return self._real_connect(addr, True) + + def accept(self): + """Accepts a new connection from a remote client, and returns + a tuple containing that new connection wrapped with a server-side + SSL channel, and the address of the remote client.""" + + newsock, addr = socket.accept(self) + newsock._drop_events() + newsock = self._context.wrap_socket(newsock, + do_handshake_on_connect=self.do_handshake_on_connect, + suppress_ragged_eofs=self.suppress_ragged_eofs, + server_side=True) + return newsock, addr + + def makefile(self, mode='r', bufsize=-1): + + """Make and return a file-like object that + works with the SSL connection. Just use the code + from the socket module.""" + if not PYPY: + self._makefile_refs += 1 + # close=True so as to decrement the reference count when done with + # the file-like object. + return _fileobject(self, mode, bufsize, close=True) + + def get_channel_binding(self, cb_type="tls-unique"): + """Get channel binding data for current connection. Raise ValueError + if the requested `cb_type` is not supported. Return bytes of the data + or None if the data is not available (e.g. before the handshake). + """ + if cb_type not in CHANNEL_BINDING_TYPES: + raise ValueError("Unsupported channel binding type") + if cb_type != "tls-unique": + raise NotImplementedError( + "{0} channel binding type not implemented" + .format(cb_type)) + if self._sslobj is None: + return None + return self._sslobj.tls_unique_cb() + + def version(self): + """ + Return a string identifying the protocol version used by the + current SSL channel, or None if there is no established channel. + """ + if self._sslobj is None: + return None + return self._sslobj.version() + +if PYPY or not hasattr(SSLSocket, 'timeout'): + # PyPy (and certain versions of CPython) doesn't have a direct + # 'timeout' property on raw sockets, because that's not part of + # the documented specification. We may wind up wrapping a raw + # socket (when ssl is used with PyWSGI) or a gevent socket, which + # does have a read/write timeout property as an alias for + # get/settimeout, so make sure that's always the case because + # pywsgi can depend on that. + SSLSocket.timeout = property(lambda self: self.gettimeout(), + lambda self, value: self.settimeout(value)) + + + +_SSLErrorReadTimeout = SSLError('The read operation timed out') +_SSLErrorWriteTimeout = SSLError('The write operation timed out') +_SSLErrorHandshakeTimeout = SSLError('The handshake operation timed out') + +def wrap_socket(sock, keyfile=None, certfile=None, + server_side=False, cert_reqs=CERT_NONE, + ssl_version=PROTOCOL_SSLv23, ca_certs=None, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + ciphers=None): + + return SSLSocket(sock=sock, keyfile=keyfile, certfile=certfile, + server_side=server_side, cert_reqs=cert_reqs, + ssl_version=ssl_version, ca_certs=ca_certs, + do_handshake_on_connect=do_handshake_on_connect, + suppress_ragged_eofs=suppress_ragged_eofs, + ciphers=ciphers) + +def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv23, ca_certs=None): + """Retrieve the certificate from the server at the specified address, + and return it as a PEM-encoded string. + If 'ca_certs' is specified, validate the server cert against it. + If 'ssl_version' is specified, use it in the connection attempt.""" + + _, _ = addr + if ca_certs is not None: + cert_reqs = CERT_REQUIRED + else: + cert_reqs = CERT_NONE + context = _create_stdlib_context(ssl_version, + cert_reqs=cert_reqs, + cafile=ca_certs) + with closing(create_connection(addr)) as sock: + with closing(context.wrap_socket(sock)) as sslsock: + dercert = sslsock.getpeercert(True) + return DER_cert_to_PEM_cert(dercert) diff --git a/src/gevent/_tblib.py b/src/gevent/_tblib.py new file mode 100644 index 0000000..1336401 --- /dev/null +++ b/src/gevent/_tblib.py @@ -0,0 +1,432 @@ +# -*- coding: utf-8 -*- +# A vendored version of part of https://github.com/ionelmc/python-tblib +# pylint:disable=redefined-outer-name,reimported,function-redefined,bare-except,no-else-return,broad-except +#### +# Copyright (c) 2013-2016, Ionel Cristian Mărieș +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +# following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +# disclaimer. + +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#### + +# cpython.py + +""" +Taken verbatim from Jinja2. + +https://github.com/mitsuhiko/jinja2/blob/master/jinja2/debug.py#L267 +""" +# pylint:disable=consider-using-dict-comprehension +#import platform # XXX: gevent cannot import platform at the top level; interferes with monkey patching +import sys + + +def _init_ugly_crap(): + """This function implements a few ugly things so that we can patch the + traceback objects. The function returned allows resetting `tb_next` on + any python traceback object. Do not attempt to use this on non cpython + interpreters + """ + import ctypes + from types import TracebackType + + # figure out side of _Py_ssize_t + if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'): + _Py_ssize_t = ctypes.c_int64 + else: + _Py_ssize_t = ctypes.c_int + + # regular python + class _PyObject(ctypes.Structure): + pass + + _PyObject._fields_ = [ + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + # python with trace + if hasattr(sys, 'getobjects'): + class _PyObject(ctypes.Structure): + pass + + _PyObject._fields_ = [ + ('_ob_next', ctypes.POINTER(_PyObject)), + ('_ob_prev', ctypes.POINTER(_PyObject)), + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + class _Traceback(_PyObject): + pass + + _Traceback._fields_ = [ + ('tb_next', ctypes.POINTER(_Traceback)), + ('tb_frame', ctypes.POINTER(_PyObject)), + ('tb_lasti', ctypes.c_int), + ('tb_lineno', ctypes.c_int) + ] + + def tb_set_next(tb, next): + """Set the tb_next attribute of a traceback object.""" + if not (isinstance(tb, TracebackType) and (next is None or isinstance(next, TracebackType))): + raise TypeError('tb_set_next arguments must be traceback objects') + obj = _Traceback.from_address(id(tb)) + if tb.tb_next is not None: + old = _Traceback.from_address(id(tb.tb_next)) + old.ob_refcnt -= 1 + if next is None: + obj.tb_next = ctypes.POINTER(_Traceback)() + else: + next = _Traceback.from_address(id(next)) + next.ob_refcnt += 1 + obj.tb_next = ctypes.pointer(next) + + return tb_set_next + + +tb_set_next = None +#try: +# if platform.python_implementation() == 'CPython': +# tb_set_next = _init_ugly_crap() +#except Exception as exc: +# sys.stderr.write("Failed to initialize cpython support: {!r}".format(exc)) +#del _init_ugly_crap + +# __init__.py +import re +from types import CodeType +from types import TracebackType + +try: + from __pypy__ import tproxy +except ImportError: + tproxy = None + +__version__ = '1.3.0' +__all__ = ('Traceback',) + +PY3 = sys.version_info[0] == 3 +FRAME_RE = re.compile(r'^\s*File "(?P.+)", line (?P\d+)(, in (?P.+))?$') + + +class _AttrDict(dict): + __slots__ = () + __getattr__ = dict.__getitem__ + + +# noinspection PyPep8Naming +class __traceback_maker(Exception): + pass + + +class TracebackParseError(Exception): + pass + + +class Code(object): + def __init__(self, code): + self.co_filename = code.co_filename + self.co_name = code.co_name + # gevent: copy more attributes + self.co_nlocals = code.co_nlocals + self.co_stacksize = code.co_stacksize + self.co_flags = code.co_flags + self.co_firstlineno = code.co_firstlineno + + +class Frame(object): + def __init__(self, frame): + self.f_globals = dict([ + (k, v) + for k, v in frame.f_globals.items() + if k in ("__file__", "__name__") + ]) + self.f_code = Code(frame.f_code) + + def clear(self): + # For compatibility with PyPy 3.5; + # clear was added to frame in Python 3.4 + # and is called by traceback.clear_frames(), which + # in turn is called by unittest.TestCase.assertRaises + pass + +class Traceback(object): + + tb_next = None + + def __init__(self, tb): + self.tb_frame = Frame(tb.tb_frame) + # noinspection SpellCheckingInspection + self.tb_lineno = int(tb.tb_lineno) + + # Build in place to avoid exceeding the recursion limit + tb = tb.tb_next + prev_traceback = self + cls = type(self) + while tb is not None: + traceback = object.__new__(cls) + traceback.tb_frame = Frame(tb.tb_frame) + traceback.tb_lineno = int(tb.tb_lineno) + prev_traceback.tb_next = traceback + prev_traceback = traceback + tb = tb.tb_next + + def as_traceback(self): + if tproxy: + return tproxy(TracebackType, self.__tproxy_handler) + if not tb_set_next: + raise RuntimeError("Cannot re-create traceback !") + + current = self + top_tb = None + tb = None + while current: + f_code = current.tb_frame.f_code + code = compile('\n' * (current.tb_lineno - 1) + 'raise __traceback_maker', current.tb_frame.f_code.co_filename, 'exec') + if PY3: + code = CodeType( + 0, code.co_kwonlyargcount, + code.co_nlocals, code.co_stacksize, code.co_flags, + code.co_code, code.co_consts, code.co_names, code.co_varnames, + f_code.co_filename, f_code.co_name, + code.co_firstlineno, code.co_lnotab, (), () + ) + else: + code = CodeType( + 0, + code.co_nlocals, code.co_stacksize, code.co_flags, + code.co_code, code.co_consts, code.co_names, code.co_varnames, + f_code.co_filename.encode(), f_code.co_name.encode(), + code.co_firstlineno, code.co_lnotab, (), () + ) + + # noinspection PyBroadException + try: + exec(code, current.tb_frame.f_globals, {}) + except: + next_tb = sys.exc_info()[2].tb_next + if top_tb is None: + top_tb = next_tb + if tb is not None: + tb_set_next(tb, next_tb) + tb = next_tb + del next_tb + + current = current.tb_next + try: + return top_tb + finally: + del top_tb + del tb + + + # noinspection SpellCheckingInspection + def __tproxy_handler(self, operation, *args, **kwargs): + if operation in ('__getattribute__', '__getattr__'): + if args[0] == 'tb_next': + return self.tb_next and self.tb_next.as_traceback() + else: + return getattr(self, args[0]) + else: + return getattr(self, operation)(*args, **kwargs) + + def to_dict(self): + """Convert a Traceback into a dictionary representation""" + if self.tb_next is None: + tb_next = None + else: + tb_next = self.tb_next.to_dict() + + code = { + 'co_filename': self.tb_frame.f_code.co_filename, + 'co_name': self.tb_frame.f_code.co_name, + } + frame = { + 'f_globals': self.tb_frame.f_globals, + 'f_code': code, + } + return { + 'tb_frame': frame, + 'tb_lineno': self.tb_lineno, + 'tb_next': tb_next, + } + + @classmethod + def from_dict(cls, dct): + if dct['tb_next']: + tb_next = cls.from_dict(dct['tb_next']) + else: + tb_next = None + + code = _AttrDict( + co_filename=dct['tb_frame']['f_code']['co_filename'], + co_name=dct['tb_frame']['f_code']['co_name'], + ) + frame = _AttrDict( + f_globals=dct['tb_frame']['f_globals'], + f_code=code, + ) + tb = _AttrDict( + tb_frame=frame, + tb_lineno=dct['tb_lineno'], + tb_next=tb_next, + ) + return cls(tb) + + @classmethod + def from_string(cls, string, strict=True): + frames = [] + header = strict + + for line in string.splitlines(): + line = line.rstrip() + if header: + if line == 'Traceback (most recent call last):': + header = False + continue + frame_match = FRAME_RE.match(line) + if frame_match: + frames.append(frame_match.groupdict()) + elif line.startswith(' '): + pass + elif strict: + break # traceback ended + + if frames: + previous = None + for frame in reversed(frames): + previous = _AttrDict( + frame, + tb_frame=_AttrDict( + frame, + f_globals=_AttrDict( + __file__=frame['co_filename'], + __name__='?', + ), + f_code=_AttrDict(frame), + ), + tb_next=previous, + ) + return cls(previous) + else: + raise TracebackParseError("Could not find any frames in %r." % string) + +# pickling_support.py + + +def unpickle_traceback(tb_frame, tb_lineno, tb_next): + ret = object.__new__(Traceback) + ret.tb_frame = tb_frame + ret.tb_lineno = tb_lineno + ret.tb_next = tb_next + return ret.as_traceback() + + +def pickle_traceback(tb): + return unpickle_traceback, (Frame(tb.tb_frame), tb.tb_lineno, tb.tb_next and Traceback(tb.tb_next)) + + +def install(): + try: + import copy_reg + except ImportError: + import copyreg as copy_reg + + copy_reg.pickle(TracebackType, pickle_traceback) + +# Added by gevent + +# We have to defer the initialization, and especially the import of platform, +# until runtime. If we're monkey patched, we need to be sure to use +# the original __import__ to avoid switching through the hub due to +# import locks on Python 2. See also builtins.py for details. + + +def _unlocked_imports(f): + def g(a): + if sys is None: # pragma: no cover + # interpreter shutdown on Py2 + return + + gb = None + if 'gevent.builtins' in sys.modules: + gb = sys.modules['gevent.builtins'] + gb._unlock_imports() + try: + return f(a) + finally: + if gb is not None: + gb._lock_imports() + g.__name__ = f.__name__ + g.__module__ = f.__module__ + return g + + +def _import_dump_load(): + global dumps + global loads + try: + import cPickle as pickle + except ImportError: + import pickle + dumps = pickle.dumps + loads = pickle.loads + +dumps = loads = None + +_installed = False + + +def _init(): + global _installed + global tb_set_next + if _installed: + return + + _installed = True + import platform + try: + if platform.python_implementation() == 'CPython': + tb_set_next = _init_ugly_crap() + except Exception as exc: + sys.stderr.write("Failed to initialize cpython support: {!r}".format(exc)) + + try: + from __pypy__ import tproxy + except ImportError: + tproxy = None + + if not tb_set_next and not tproxy: + raise ImportError("Cannot use tblib. Runtime not supported.") + _import_dump_load() + install() + + +@_unlocked_imports +def dump_traceback(tb): + # Both _init and dump/load have to be unlocked, because + # copy_reg and pickle can do imports to resolve class names; those + # class names are in this module and greenlet safe though + _init() + return dumps(tb) + + +@_unlocked_imports +def load_traceback(s): + _init() + return loads(s) diff --git a/src/gevent/_threading.py b/src/gevent/_threading.py new file mode 100644 index 0000000..9258dfb --- /dev/null +++ b/src/gevent/_threading.py @@ -0,0 +1,168 @@ +"""A clone of threading module (version 2.7.2) that always +targets real OS threads. (Unlike 'threading' which flips between +green and OS threads based on whether the monkey patching is in effect +or not). + +This module is missing 'Thread' class, but includes 'Queue'. +""" +from __future__ import absolute_import + +from collections import deque + +from gevent import monkey +from gevent._compat import thread_mod_name + + +__all__ = [ + 'Lock', + 'Queue', +] + + +start_new_thread, Lock, get_thread_ident, = monkey.get_original(thread_mod_name, [ + 'start_new_thread', 'allocate_lock', 'get_ident', +]) + + +# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return +# pylint:disable=assignment-from-no-return + + +class _Condition(object): + # pylint:disable=method-hidden + + def __init__(self, lock): + self.__lock = lock + self.__waiters = [] + + # If the lock defines _release_save() and/or _acquire_restore(), + # these override the default implementations (which just call + # release() and acquire() on the lock). Ditto for _is_owned(). + try: + self._release_save = lock._release_save + except AttributeError: + pass + try: + self._acquire_restore = lock._acquire_restore + except AttributeError: + pass + try: + self._is_owned = lock._is_owned + except AttributeError: + pass + + def __enter__(self): + return self.__lock.__enter__() + + def __exit__(self, t, v, tb): + return self.__lock.__exit__(t, v, tb) + + def __repr__(self): + return "" % (self.__lock, len(self.__waiters)) + + def _release_save(self): + self.__lock.release() # No state to save + + def _acquire_restore(self, x): # pylint:disable=unused-argument + self.__lock.acquire() # Ignore saved state + + def _is_owned(self): + # Return True if lock is owned by current_thread. + # This method is called only if __lock doesn't have _is_owned(). + if self.__lock.acquire(0): + self.__lock.release() + return False + return True + + def wait(self): + # The condition MUST be owned, but we don't check that. + waiter = Lock() + waiter.acquire() + self.__waiters.append(waiter) + saved_state = self._release_save() + try: # restore state no matter what (e.g., KeyboardInterrupt) + waiter.acquire() # Block on the native lock + finally: + self._acquire_restore(saved_state) + + def notify_one(self): + # The condition MUST be owned, but we don't check that. + try: + waiter = self.__waiters.pop() + except IndexError: + # Nobody around + pass + else: + waiter.release() + + +class Queue(object): + """Create a queue object. + + The queue is always infinite size. + """ + + __slots__ = ('_queue', '_mutex', '_not_empty', 'unfinished_tasks') + + def __init__(self): + self._queue = deque() + # mutex must be held whenever the queue is mutating. All methods + # that acquire mutex must release it before returning. mutex + # is shared between the three conditions, so acquiring and + # releasing the conditions also acquires and releases mutex. + self._mutex = Lock() + # Notify not_empty whenever an item is added to the queue; a + # thread waiting to get is notified then. + self._not_empty = _Condition(self._mutex) + + self.unfinished_tasks = 0 + + def task_done(self): + """Indicate that a formerly enqueued task is complete. + + Used by Queue consumer threads. For each get() used to fetch a task, + a subsequent call to task_done() tells the queue that the processing + on the task is complete. + + If a join() is currently blocking, it will resume when all items + have been processed (meaning that a task_done() call was received + for every item that had been put() into the queue). + + Raises a ValueError if called more times than there were items + placed in the queue. + """ + with self._mutex: + unfinished = self.unfinished_tasks - 1 + if unfinished <= 0: + if unfinished < 0: + raise ValueError('task_done() called too many times') + self.unfinished_tasks = unfinished + + def qsize(self, len=len): + """Return the approximate size of the queue (not reliable!).""" + return len(self._queue) + + def empty(self): + """Return True if the queue is empty, False otherwise (not reliable!).""" + return not self.qsize() + + def full(self): + """Return True if the queue is full, False otherwise (not reliable!).""" + return False + + def put(self, item): + """Put an item into the queue. + """ + with self._not_empty: + self._queue.append(item) + self.unfinished_tasks += 1 + self._not_empty.notify_one() + + def get(self): + """Remove and return an item from the queue. + """ + with self._not_empty: + while not self._queue: + self._not_empty.wait() + item = self._queue.popleft() + return item diff --git a/src/gevent/_tracer.py b/src/gevent/_tracer.py new file mode 100644 index 0000000..98307ab --- /dev/null +++ b/src/gevent/_tracer.py @@ -0,0 +1,179 @@ +# Copyright (c) 2018 gevent. See LICENSE for details. +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False +from __future__ import print_function, absolute_import, division + +import sys +import traceback + +from greenlet import settrace +from greenlet import getcurrent + +from gevent.util import format_run_info + +from gevent._compat import perf_counter +from gevent._util import gmctime + + +__all__ = [ + 'GreenletTracer', + 'HubSwitchTracer', + 'MaxSwitchTracer', +] + +# Recall these classes are cython compiled, so +# class variable declarations are bad. + + +class GreenletTracer(object): + def __init__(self): + # A counter, incremented by the greenlet trace function + # we install on every greenlet switch. This is reset when the + # periodic monitoring thread runs. + + self.greenlet_switch_counter = 0 + + # The greenlet last switched to. + self.active_greenlet = None + + # The trace function that was previously installed, + # if any. + # NOTE: Calling a class instance is cheaper than + # calling a bound method (at least when compiled with cython) + # even when it redirects to another function. + prev_trace = settrace(self) + + self.previous_trace_function = prev_trace + + self._killed = False + + def kill(self): + # Must be called in the monitored thread. + if not self._killed: + self._killed = True + settrace(self.previous_trace_function) + self.previous_trace_function = None + + def _trace(self, event, args): + # This function runs in the thread we are monitoring. + self.greenlet_switch_counter += 1 + if event in ('switch', 'throw'): + # args is (origin, target). This is the only defined + # case + self.active_greenlet = args[1] + else: + self.active_greenlet = None + if self.previous_trace_function is not None: + self.previous_trace_function(event, args) + + def __call__(self, event, args): + return self._trace(event, args) + + def did_block_hub(self, hub): + # Check to see if we have blocked since the last call to this + # method. Returns a true value if we blocked (not in the hub), + # a false value if everything is fine. + + # This may be called in the same thread being traced or a + # different thread; if a different thread, there is a race + # condition with this being incremented in the thread we're + # monitoring, but probably not often enough to lead to + # annoying false positives. + + active_greenlet = self.active_greenlet + did_switch = self.greenlet_switch_counter != 0 + self.greenlet_switch_counter = 0 + + if did_switch or active_greenlet is None or active_greenlet is hub: + # Either we switched, or nothing is running (we got a + # trace event we don't know about or were requested to + # ignore), or we spent the whole time in the hub, blocked + # for IO. Nothing to report. + return False + return True, active_greenlet + + def ignore_current_greenlet_blocking(self): + # Don't pay attention to the current greenlet. + self.active_greenlet = None + + def monitor_current_greenlet_blocking(self): + self.active_greenlet = getcurrent() + + def did_block_hub_report(self, hub, active_greenlet, format_kwargs): + report = ['=' * 80, + '\n%s : Greenlet %s appears to be blocked' % + (gmctime(), active_greenlet)] + report.append(" Reported by %s" % (self,)) + try: + frame = sys._current_frames()[hub.thread_ident] + except KeyError: + # The thread holding the hub has died. Perhaps we shouldn't + # even report this? + stack = ["Unknown: No thread found for hub %r\n" % (hub,)] + else: + stack = traceback.format_stack(frame) + report.append('Blocked Stack (for thread id %s):' % (hex(hub.thread_ident),)) + report.append(''.join(stack)) + report.append("Info:") + report.extend(format_run_info(**format_kwargs)) + + return report + + +class _HubTracer(GreenletTracer): + def __init__(self, hub, max_blocking_time): + GreenletTracer.__init__(self) + self.max_blocking_time = max_blocking_time + self.hub = hub + + def kill(self): + self.hub = None + GreenletTracer.kill(self) + + +class HubSwitchTracer(_HubTracer): + # A greenlet tracer that records the last time we switched *into* the hub. + + def __init__(self, hub, max_blocking_time): + _HubTracer.__init__(self, hub, max_blocking_time) + self.last_entered_hub = 0 + + def _trace(self, event, args): + GreenletTracer._trace(self, event, args) + if self.active_greenlet is self.hub: + self.last_entered_hub = perf_counter() + + def did_block_hub(self, hub): + if perf_counter() - self.last_entered_hub > self.max_blocking_time: + return True, self.active_greenlet + + +class MaxSwitchTracer(_HubTracer): + # A greenlet tracer that records the maximum time between switches, + # not including time spent in the hub. + + def __init__(self, hub, max_blocking_time): + _HubTracer.__init__(self, hub, max_blocking_time) + self.last_switch = perf_counter() + self.max_blocking = 0 + + def _trace(self, event, args): + old_active = self.active_greenlet + GreenletTracer._trace(self, event, args) + if old_active is not self.hub and old_active is not None: + # If we're switching out of the hub, the blocking + # time doesn't count. + switched_at = perf_counter() + self.max_blocking = max(self.max_blocking, + switched_at - self.last_switch) + + def did_block_hub(self, hub): + if self.max_blocking == 0: + # We never switched. Check the time now + self.max_blocking = perf_counter() - self.last_switch + + if self.max_blocking > self.max_blocking_time: + return True, self.active_greenlet + + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__tracer') diff --git a/src/gevent/_util.py b/src/gevent/_util.py new file mode 100644 index 0000000..4397aa3 --- /dev/null +++ b/src/gevent/_util.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- +""" +internal gevent utilities, not for external use. +""" + +from __future__ import print_function, absolute_import, division + +from functools import update_wrapper + +from gevent._compat import iteritems + + +class _NONE(object): + """ + A special object you must never pass to any gevent API. + Used as a marker object for keyword arguments that cannot have the + builtin None (because that might be a valid value). + """ + __slots__ = () + + def __repr__(self): + return '' + +_NONE = _NONE() + +def copy_globals(source, + globs, + only_names=None, + ignore_missing_names=False, + names_to_ignore=(), + dunder_names_to_keep=('__implements__', '__all__', '__imports__'), + cleanup_globs=True): + """ + Copy attributes defined in ``source.__dict__`` to the dictionary + in globs (which should be the caller's :func:`globals`). + + Names that start with ``__`` are ignored (unless they are in + *dunder_names_to_keep*). Anything found in *names_to_ignore* is + also ignored. + + If *only_names* is given, only those attributes will be + considered. In this case, *ignore_missing_names* says whether or + not to raise an :exc:`AttributeError` if one of those names can't + be found. + + If *cleanup_globs* has a true value, then common things imported but + not used at runtime are removed, including this function. + + Returns a list of the names copied; this should be assigned to ``__imports__``. + """ + if only_names: + if ignore_missing_names: + items = ((k, getattr(source, k, _NONE)) for k in only_names) + else: + items = ((k, getattr(source, k)) for k in only_names) + else: + items = iteritems(source.__dict__) + + copied = [] + for key, value in items: + if value is _NONE: + continue + if key in names_to_ignore: + continue + if key.startswith("__") and key not in dunder_names_to_keep: + continue + globs[key] = value + copied.append(key) + + if cleanup_globs: + if 'copy_globals' in globs: + del globs['copy_globals'] + + return copied + +def import_c_accel(globs, cname): + """ + Import the C-accelerator for the __name__ + and copy its globals. + """ + + name = globs.get('__name__') + + if not name or name == cname: + # Do nothing if we're being exec'd as a file (no name) + # or we're running from the C extension + return + + + from gevent._compat import PURE_PYTHON + if PURE_PYTHON: + return + + import importlib + import warnings + with warnings.catch_warnings(): + # Python 3.7 likes to produce + # "ImportWarning: can't resolve + # package from __spec__ or __package__, falling back on + # __name__ and __path__" + # when we load cython compiled files. This is probably a bug in + # Cython, but it doesn't seem to have any consequences, it's + # just annoying to see and can mess up our unittests. + warnings.simplefilter('ignore', ImportWarning) + mod = importlib.import_module(cname) + + # By adopting the entire __dict__, we get a more accurate + # __file__ and module repr, plus we don't leak any imported + # things we no longer need. + globs.clear() + globs.update(mod.__dict__) + + if 'import_c_accel' in globs: + del globs['import_c_accel'] + + +class Lazy(object): + """ + A non-data descriptor used just like @property. The + difference is the function value is assigned to the instance + dict the first time it is accessed and then the function is never + called agoin. + """ + def __init__(self, func): + self.data = (func, func.__name__) + update_wrapper(self, func) + + def __get__(self, inst, class_): + if inst is None: + return self + + func, name = self.data + value = func(inst) + inst.__dict__[name] = value + return value + +class readproperty(object): + """ + A non-data descriptor like @property. The difference is that + when the property is assigned to, it is cached in the instance + and the function is not called on that instance again. + """ + + def __init__(self, func): + self.func = func + update_wrapper(self, func) + + def __get__(self, inst, class_): + if inst is None: + return self + + return self.func(inst) + +def gmctime(): + """ + Returns the current time as a string in RFC3339 format. + """ + import time + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + +try: + from zope.interface import Interface + from zope.interface import implementer + from zope.interface import Attribute +except ImportError: + class Interface(object): + pass + def implementer(_iface): + def dec(c): + return c + return dec + + def Attribute(s): + return s + +Interface = Interface +implementer = implementer +Attribute = Attribute diff --git a/src/gevent/_util_py2.py b/src/gevent/_util_py2.py new file mode 100644 index 0000000..02332e3 --- /dev/null +++ b/src/gevent/_util_py2.py @@ -0,0 +1,23 @@ +import sys + +__all__ = ['reraise'] + + +def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + +exec_("""def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""") diff --git a/src/gevent/_waiter.py b/src/gevent/_waiter.py new file mode 100644 index 0000000..b9ba3e8 --- /dev/null +++ b/src/gevent/_waiter.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +# copyright 2018 gevent +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False +""" +Low-level waiting primitives. + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys + +from gevent._hub_local import get_hub_noargs as get_hub +from gevent.exceptions import ConcurrentObjectUseError + +__all__ = [ + 'Waiter', +] + +_NONE = object() + +locals()['getcurrent'] = __import__('greenlet').getcurrent +locals()['greenlet_init'] = lambda: None + + +class Waiter(object): + """ + A low level communication utility for greenlets. + + Waiter is a wrapper around greenlet's ``switch()`` and ``throw()`` calls that makes them somewhat safer: + + * switching will occur only if the waiting greenlet is executing :meth:`get` method currently; + * any error raised in the greenlet is handled inside :meth:`switch` and :meth:`throw` + * if :meth:`switch`/:meth:`throw` is called before the receiver calls :meth:`get`, then :class:`Waiter` + will store the value/exception. The following :meth:`get` will return the value/raise the exception. + + The :meth:`switch` and :meth:`throw` methods must only be called from the :class:`Hub` greenlet. + The :meth:`get` method must be called from a greenlet other than :class:`Hub`. + + >>> result = Waiter() + >>> timer = get_hub().loop.timer(0.1) + >>> timer.start(result.switch, 'hello from Waiter') + >>> result.get() # blocks for 0.1 seconds + 'hello from Waiter' + >>> timer.close() + + If switch is called before the greenlet gets a chance to call :meth:`get` then + :class:`Waiter` stores the value. + + >>> result = Waiter() + >>> timer = get_hub().loop.timer(0.1) + >>> timer.start(result.switch, 'hi from Waiter') + >>> sleep(0.2) + >>> result.get() # returns immediately without blocking + 'hi from Waiter' + >>> timer.close() + + .. warning:: + + This a limited and dangerous way to communicate between + greenlets. It can easily leave a greenlet unscheduled forever + if used incorrectly. Consider using safer classes such as + :class:`gevent.event.Event`, :class:`gevent.event.AsyncResult`, + or :class:`gevent.queue.Queue`. + """ + + __slots__ = ['hub', 'greenlet', 'value', '_exception'] + + def __init__(self, hub=None): + self.hub = get_hub() if hub is None else hub + self.greenlet = None + self.value = None + self._exception = _NONE + + def clear(self): + self.greenlet = None + self.value = None + self._exception = _NONE + + def __str__(self): + if self._exception is _NONE: + return '<%s greenlet=%s>' % (type(self).__name__, self.greenlet) + if self._exception is None: + return '<%s greenlet=%s value=%r>' % (type(self).__name__, self.greenlet, self.value) + return '<%s greenlet=%s exc_info=%r>' % (type(self).__name__, self.greenlet, self.exc_info) + + def ready(self): + """Return true if and only if it holds a value or an exception""" + return self._exception is not _NONE + + def successful(self): + """Return true if and only if it is ready and holds a value""" + return self._exception is None + + @property + def exc_info(self): + "Holds the exception info passed to :meth:`throw` if :meth:`throw` was called. Otherwise ``None``." + if self._exception is not _NONE: + return self._exception + + def switch(self, value): + """ + Switch to the greenlet if one's available. Otherwise store the + *value*. + + .. versionchanged:: 1.3b1 + The *value* is no longer optional. + """ + greenlet = self.greenlet + if greenlet is None: + self.value = value + self._exception = None + else: + if getcurrent() is not self.hub: # pylint:disable=undefined-variable + raise AssertionError("Can only use Waiter.switch method from the Hub greenlet") + switch = greenlet.switch + try: + switch(value) + except: # pylint:disable=bare-except + self.hub.handle_error(switch, *sys.exc_info()) + + def switch_args(self, *args): + return self.switch(args) + + def throw(self, *throw_args): + """Switch to the greenlet with the exception. If there's no greenlet, store the exception.""" + greenlet = self.greenlet + if greenlet is None: + self._exception = throw_args + else: + if getcurrent() is not self.hub: # pylint:disable=undefined-variable + raise AssertionError("Can only use Waiter.switch method from the Hub greenlet") + throw = greenlet.throw + try: + throw(*throw_args) + except: # pylint:disable=bare-except + self.hub.handle_error(throw, *sys.exc_info()) + + def get(self): + """If a value/an exception is stored, return/raise it. Otherwise until switch() or throw() is called.""" + if self._exception is not _NONE: + if self._exception is None: + return self.value + getcurrent().throw(*self._exception) # pylint:disable=undefined-variable + else: + if self.greenlet is not None: + raise ConcurrentObjectUseError('This Waiter is already used by %r' % (self.greenlet, )) + self.greenlet = getcurrent() # pylint:disable=undefined-variable + try: + return self.hub.switch() + finally: + self.greenlet = None + + def __call__(self, source): + if source.exception is None: + self.switch(source.value) + else: + self.throw(source.exception) + + # can also have a debugging version, that wraps the value in a tuple (self, value) in switch() + # and unwraps it in wait() thus checking that switch() was indeed called + + + +class MultipleWaiter(Waiter): + """ + An internal extension of Waiter that can be used if multiple objects + must be waited on, and there is a chance that in between waits greenlets + might be switched out. All greenlets that switch to this waiter + will have their value returned. + + This does not handle exceptions or throw methods. + """ + __slots__ = ['_values'] + + def __init__(self, hub=None): + Waiter.__init__(self, hub) + # we typically expect a relatively small number of these to be outstanding. + # since we pop from the left, a deque might be slightly + # more efficient, but since we're in the hub we avoid imports if + # we can help it to better support monkey-patching, and delaying the import + # here can be impractical (see https://github.com/gevent/gevent/issues/652) + self._values = list() + + def switch(self, value): + self._values.append(value) + Waiter.switch(self, True) + + def get(self): + if not self._values: + Waiter.get(self) + Waiter.clear(self) + + return self._values.pop(0) + +def _init(): + greenlet_init() # pylint:disable=undefined-variable + +_init() + + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent.__waiter') diff --git a/src/gevent/ares.py b/src/gevent/ares.py new file mode 100644 index 0000000..37980b3 --- /dev/null +++ b/src/gevent/ares.py @@ -0,0 +1,10 @@ +"""Backwards compatibility alias for :mod:`gevent.resolver.cares`. + +.. deprecated:: 1.3 + Use :mod:`gevent.resolver.cares` +""" + +from gevent.resolver.cares import * # pylint:disable=wildcard-import,unused-wildcard-import, +import gevent.resolver.cares as _cares +__all__ = _cares.__all__ # pylint:disable=c-extension-no-member +del _cares diff --git a/src/gevent/backdoor.py b/src/gevent/backdoor.py new file mode 100644 index 0000000..8489417 --- /dev/null +++ b/src/gevent/backdoor.py @@ -0,0 +1,210 @@ +# Copyright (c) 2009-2014, gevent contributors +# Based on eventlet.backdoor Copyright (c) 2005-2006, Bob Ippolito +""" +Interactive greenlet-based network console that can be used in any process. + +The :class:`BackdoorServer` provides a REPL inside a running process. As +long as the process is monkey-patched, the ``BackdoorServer`` can coexist +with other elements of the process. + +.. seealso:: :class:`code.InteractiveConsole` +""" +from __future__ import print_function, absolute_import +import sys +from code import InteractiveConsole + +from gevent.greenlet import Greenlet +from gevent.hub import getcurrent +from gevent.server import StreamServer +from gevent.pool import Pool + +__all__ = ['BackdoorServer'] + +try: + sys.ps1 +except AttributeError: + sys.ps1 = '>>> ' +try: + sys.ps2 +except AttributeError: + sys.ps2 = '... ' + +class _Greenlet_stdreplace(Greenlet): + # A greenlet that replaces sys.std[in/out/err] while running. + _fileobj = None + saved = None + + def switch(self, *args, **kw): + if self._fileobj is not None: + self.switch_in() + Greenlet.switch(self, *args, **kw) + + def switch_in(self): + self.saved = sys.stdin, sys.stderr, sys.stdout + sys.stdin = sys.stdout = sys.stderr = self._fileobj + + def switch_out(self): + sys.stdin, sys.stderr, sys.stdout = self.saved + self.saved = None + + def throw(self, *args, **kwargs): + # pylint:disable=arguments-differ + if self.saved is None and self._fileobj is not None: + self.switch_in() + Greenlet.throw(self, *args, **kwargs) + + def run(self): + try: + return Greenlet.run(self) + finally: + # Make sure to restore the originals. + self.switch_out() + + +class BackdoorServer(StreamServer): + """ + Provide a backdoor to a program for debugging purposes. + + .. warning:: This backdoor provides no authentication and makes no + attempt to limit what remote users can do. Anyone that + can access the server can take any action that the running + python process can. Thus, while you may bind to any interface, for + security purposes it is recommended that you bind to one + only accessible to the local machine, e.g., + 127.0.0.1/localhost. + + Basic usage:: + + from gevent.backdoor import BackdoorServer + server = BackdoorServer(('127.0.0.1', 5001), + banner="Hello from gevent backdoor!", + locals={'foo': "From defined scope!"}) + server.serve_forever() + + In a another terminal, connect with...:: + + $ telnet 127.0.0.1 5001 + Trying 127.0.0.1... + Connected to 127.0.0.1. + Escape character is '^]'. + Hello from gevent backdoor! + >> print(foo) + From defined scope! + + .. versionchanged:: 1.2a1 + Spawned greenlets are now tracked in a pool and killed when the server + is stopped. + """ + + def __init__(self, listener, locals=None, banner=None, **server_args): + """ + :keyword locals: If given, a dictionary of "builtin" values that will be available + at the top-level. + :keyword banner: If geven, a string that will be printed to each connecting user. + """ + group = Pool(greenlet_class=_Greenlet_stdreplace) # no limit on number + StreamServer.__init__(self, listener, spawn=group, **server_args) + _locals = {'__doc__': None, '__name__': '__console__'} + if locals: + _locals.update(locals) + self.locals = _locals + + self.banner = banner + self.stderr = sys.stderr + + def _create_interactive_locals(self): + # Create and return a *new* locals dictionary based on self.locals, + # and set any new entries in it. (InteractiveConsole does not + # copy its locals value) + _locals = self.locals.copy() + # __builtins__ may either be the __builtin__ module or + # __builtin__.__dict__; in the latter case typing + # locals() at the backdoor prompt spews out lots of + # useless stuff + try: + import __builtin__ + _locals["__builtins__"] = __builtin__ + except ImportError: + import builtins # pylint:disable=import-error + _locals["builtins"] = builtins + _locals['__builtins__'] = builtins + return _locals + + def handle(self, conn, _address): # pylint: disable=method-hidden + """ + Interact with one remote user. + + .. versionchanged:: 1.1b2 Each connection gets its own + ``locals`` dictionary. Previously they were shared in a + potentially unsafe manner. + """ + fobj = conn.makefile(mode="rw") + fobj = _fileobject(conn, fobj, self.stderr) + getcurrent()._fileobj = fobj + + getcurrent().switch_in() + try: + console = InteractiveConsole(self._create_interactive_locals()) + if sys.version_info[:3] >= (3, 6, 0): + # Beginning in 3.6, the console likes to print "now exiting " + # but probably our socket is already closed, so this just causes problems. + console.interact(banner=self.banner, exitmsg='') # pylint:disable=unexpected-keyword-arg + else: + console.interact(banner=self.banner) + except SystemExit: # raised by quit() + if hasattr(sys, 'exc_clear'): # py2 + sys.exc_clear() + finally: + conn.close() + fobj.close() + + +class _fileobject(object): + """ + A file-like object that wraps the result of socket.makefile (composition + instead of inheritance lets us work identically under CPython and PyPy). + + We write directly to the socket, avoiding the buffering that the text-oriented + makefile would want to do (otherwise we'd be at the mercy of waiting on a + flush() to get called for the remote user to see data); this beats putting + the file in binary mode and translating everywhere with a non-default + encoding. + """ + def __init__(self, sock, fobj, stderr): + self._sock = sock + self._fobj = fobj + self.stderr = stderr + + def __getattr__(self, name): + return getattr(self._fobj, name) + + def close(self): + self._fobj.close() + self._sock.close() + + def write(self, data): + if not isinstance(data, bytes): + data = data.encode('utf-8') + self._sock.sendall(data) + + def isatty(self): + return True + + def flush(self): + pass + + def readline(self, *a): + try: + return self._fobj.readline(*a).replace("\r\n", "\n") + except UnicodeError: + # Typically, under python 3, a ^C on the other end + return '' + + +if __name__ == '__main__': + if not sys.argv[1:]: + print('USAGE: %s PORT [banner]' % sys.argv[0]) + else: + BackdoorServer(('127.0.0.1', int(sys.argv[1])), + banner=(sys.argv[2] if len(sys.argv) > 2 else None), + locals={'hello': 'world'}).serve_forever() diff --git a/src/gevent/baseserver.py b/src/gevent/baseserver.py new file mode 100644 index 0000000..f987cc3 --- /dev/null +++ b/src/gevent/baseserver.py @@ -0,0 +1,409 @@ +"""Base class for implementing servers""" +# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details. +import sys +import _socket +import errno +from gevent.greenlet import Greenlet +from gevent.event import Event +from gevent.hub import get_hub +from gevent._compat import string_types, integer_types, xrange + + +__all__ = ['BaseServer'] + + +# We define a helper function to handle closing the socket in +# do_handle; We'd like to bind it to a kwarg to avoid *any* lookups at +# all, but that's incompatible with the calling convention of +# do_handle. On CPython, this is ~20% faster than creating and calling +# a closure and ~10% faster than using a @staticmethod. (In theory, we +# could create a closure only once in set_handle, to wrap self._handle, +# but this is safer from a backwards compat standpoint.) +# we also avoid unpacking the *args tuple when calling/spawning this object +# for a tiny improvement (benchmark shows a wash) +def _handle_and_close_when_done(handle, close, args_tuple): + try: + return handle(*args_tuple) + finally: + close(*args_tuple) + + +class BaseServer(object): + """ + An abstract base class that implements some common functionality for the servers in gevent. + + :param listener: Either be an address that the server should bind + on or a :class:`gevent.socket.socket` instance that is already + bound (and put into listening mode in case of TCP socket). + + :keyword handle: If given, the request handler. The request + handler can be defined in a few ways. Most commonly, + subclasses will implement a ``handle`` method as an + instance method. Alternatively, a function can be passed + as the ``handle`` argument to the constructor. In either + case, the handler can later be changed by calling + :meth:`set_handle`. + + When the request handler returns, the socket used for the + request will be closed. Therefore, the handler must not return if + the socket is still in use (for example, by manually spawned greenlets). + + :keyword spawn: If provided, is called to create a new + greenlet to run the handler. By default, + :func:`gevent.spawn` is used (meaning there is no + artificial limit on the number of concurrent requests). Possible values for *spawn*: + + - a :class:`gevent.pool.Pool` instance -- ``handle`` will be executed + using :meth:`gevent.pool.Pool.spawn` only if the pool is not full. + While it is full, no new connections are accepted; + - :func:`gevent.spawn_raw` -- ``handle`` will be executed in a raw + greenlet which has a little less overhead then :class:`gevent.Greenlet` instances spawned by default; + - ``None`` -- ``handle`` will be executed right away, in the :class:`Hub` greenlet. + ``handle`` cannot use any blocking functions as it would mean switching to the :class:`Hub`. + - an integer -- a shortcut for ``gevent.pool.Pool(integer)`` + + .. versionchanged:: 1.1a1 + When the *handle* function returns from processing a connection, + the client socket will be closed. This resolves the non-deterministic + closing of the socket, fixing ResourceWarnings under Python 3 and PyPy. + + """ + # pylint: disable=too-many-instance-attributes,bare-except,broad-except + + #: the number of seconds to sleep in case there was an error in accept() call + #: for consecutive errors the delay will double until it reaches max_delay + #: when accept() finally succeeds the delay will be reset to min_delay again + min_delay = 0.01 + max_delay = 1 + + #: Sets the maximum number of consecutive accepts that a process may perform on + #: a single wake up. High values give higher priority to high connection rates, + #: while lower values give higher priority to already established connections. + #: Default is 100. Note, that in case of multiple working processes on the same + #: listening value, it should be set to a lower value. (pywsgi.WSGIServer sets it + #: to 1 when environ["wsgi.multiprocess"] is true) + max_accept = 100 + + _spawn = Greenlet.spawn + + #: the default timeout that we wait for the client connections to close in stop() + stop_timeout = 1 + + fatal_errors = (errno.EBADF, errno.EINVAL, errno.ENOTSOCK) + + def __init__(self, listener, handle=None, spawn='default'): + self._stop_event = Event() + self._stop_event.set() + self._watcher = None + self._timer = None + self._handle = None + # XXX: FIXME: Subclasses rely on the presence or absence of the + # `socket` attribute to determine whether we are open/should be opened. + # Instead, have it be None. + self.pool = None + try: + self.set_listener(listener) + self.set_spawn(spawn) + self.set_handle(handle) + self.delay = self.min_delay + self.loop = get_hub().loop + if self.max_accept < 1: + raise ValueError('max_accept must be positive int: %r' % (self.max_accept, )) + except: + self.close() + raise + + def set_listener(self, listener): + if hasattr(listener, 'accept'): + if hasattr(listener, 'do_handshake'): + raise TypeError('Expected a regular socket, not SSLSocket: %r' % (listener, )) + self.family = listener.family + self.address = listener.getsockname() + self.socket = listener + else: + self.family, self.address = parse_address(listener) + + def set_spawn(self, spawn): + if spawn == 'default': + self.pool = None + self._spawn = self._spawn + elif hasattr(spawn, 'spawn'): + self.pool = spawn + self._spawn = spawn.spawn + elif isinstance(spawn, integer_types): + from gevent.pool import Pool + self.pool = Pool(spawn) + self._spawn = self.pool.spawn + else: + self.pool = None + self._spawn = spawn + if hasattr(self.pool, 'full'): + self.full = self.pool.full + if self.pool is not None: + self.pool._semaphore.rawlink(self._start_accepting_if_started) + + def set_handle(self, handle): + if handle is not None: + self.handle = handle + if hasattr(self, 'handle'): + self._handle = self.handle + else: + raise TypeError("'handle' must be provided") + + def _start_accepting_if_started(self, _event=None): + if self.started: + self.start_accepting() + + def start_accepting(self): + if self._watcher is None: + # just stop watcher without creating a new one? + self._watcher = self.loop.io(self.socket.fileno(), 1) + self._watcher.start(self._do_read) + + def stop_accepting(self): + if self._watcher is not None: + self._watcher.stop() + self._watcher.close() + self._watcher = None + if self._timer is not None: + self._timer.stop() + self._timer.close() + self._timer = None + + def do_handle(self, *args): + spawn = self._spawn + handle = self._handle + close = self.do_close + + try: + if spawn is None: + _handle_and_close_when_done(handle, close, args) + else: + spawn(_handle_and_close_when_done, handle, close, args) + except: + close(*args) + raise + + def do_close(self, *args): + pass + + def do_read(self): + raise NotImplementedError() + + def _do_read(self): + for _ in xrange(self.max_accept): + if self.full(): + self.stop_accepting() + return + try: + args = self.do_read() + self.delay = self.min_delay + if not args: + return + except: + self.loop.handle_error(self, *sys.exc_info()) + ex = sys.exc_info()[1] + if self.is_fatal_error(ex): + self.close() + sys.stderr.write('ERROR: %s failed with %s\n' % (self, str(ex) or repr(ex))) + return + if self.delay >= 0: + self.stop_accepting() + self._timer = self.loop.timer(self.delay) + self._timer.start(self._start_accepting_if_started) + self.delay = min(self.max_delay, self.delay * 2) + break + else: + try: + self.do_handle(*args) + except: + self.loop.handle_error((args[1:], self), *sys.exc_info()) + if self.delay >= 0: + self.stop_accepting() + self._timer = self.loop.timer(self.delay) + self._timer.start(self._start_accepting_if_started) + self.delay = min(self.max_delay, self.delay * 2) + break + + def full(self): + # copied from self.pool + # pylint: disable=method-hidden + return False + + def __repr__(self): + return '<%s at %s %s>' % (type(self).__name__, hex(id(self)), self._formatinfo()) + + def __str__(self): + return '<%s %s>' % (type(self).__name__, self._formatinfo()) + + def _formatinfo(self): + if hasattr(self, 'socket'): + try: + fileno = self.socket.fileno() + except Exception as ex: + fileno = str(ex) + result = 'fileno=%s ' % fileno + else: + result = '' + try: + if isinstance(self.address, tuple) and len(self.address) == 2: + result += 'address=%s:%s' % self.address + else: + result += 'address=%s' % (self.address, ) + except Exception as ex: + result += str(ex) or '' + + handle = self.__dict__.get('handle') + if handle is not None: + fself = getattr(handle, '__self__', None) + try: + if fself is self: + # Checks the __self__ of the handle in case it is a bound + # method of self to prevent recursivly defined reprs. + handle_repr = '' % ( + self.__class__.__name__, + handle.__name__, + ) + else: + handle_repr = repr(handle) + + result += ' handle=' + handle_repr + except Exception as ex: + result += str(ex) or '' + + return result + + @property + def server_host(self): + """IP address that the server is bound to (string).""" + if isinstance(self.address, tuple): + return self.address[0] + + @property + def server_port(self): + """Port that the server is bound to (an integer).""" + if isinstance(self.address, tuple): + return self.address[1] + + def init_socket(self): + """If the user initialized the server with an address rather than socket, + then this function will create a socket, bind it and put it into listening mode. + + It is not supposed to be called by the user, it is called by :meth:`start` before starting + the accept loop.""" + + @property + def started(self): + return not self._stop_event.is_set() + + def start(self): + """Start accepting the connections. + + If an address was provided in the constructor, then also create a socket, + bind it and put it into the listening mode. + """ + self.init_socket() + self._stop_event.clear() + try: + self.start_accepting() + except: + self.close() + raise + + def close(self): + """Close the listener socket and stop accepting.""" + self._stop_event.set() + try: + self.stop_accepting() + finally: + try: + self.socket.close() + except Exception: + pass + finally: + self.__dict__.pop('socket', None) + self.__dict__.pop('handle', None) + self.__dict__.pop('_handle', None) + self.__dict__.pop('_spawn', None) + self.__dict__.pop('full', None) + if self.pool is not None: + self.pool._semaphore.unlink(self._start_accepting_if_started) + # If the pool's semaphore had a notifier already started, + # there's a reference cycle we're a part of + # (self->pool->semaphere-hub callback->semaphore) + # But we can't destroy self.pool, because self.stop() + # calls this method, and then wants to join self.pool() + + @property + def closed(self): + return not hasattr(self, 'socket') + + def stop(self, timeout=None): + """ + Stop accepting the connections and close the listening socket. + + If the server uses a pool to spawn the requests, then + :meth:`stop` also waits for all the handlers to exit. If there + are still handlers executing after *timeout* has expired + (default 1 second, :attr:`stop_timeout`), then the currently + running handlers in the pool are killed. + + If the server does not use a pool, then this merely stops accepting connections; + any spawned greenlets that are handling requests continue running until + they naturally complete. + """ + self.close() + if timeout is None: + timeout = self.stop_timeout + if self.pool: + self.pool.join(timeout=timeout) + self.pool.kill(block=True, timeout=1) + + + def serve_forever(self, stop_timeout=None): + """Start the server if it hasn't been already started and wait until it's stopped.""" + # add test that serve_forever exists on stop() + if not self.started: + self.start() + try: + self._stop_event.wait() + finally: + Greenlet.spawn(self.stop, timeout=stop_timeout).join() + + def is_fatal_error(self, ex): + return isinstance(ex, _socket.error) and ex.args[0] in self.fatal_errors + + +def _extract_family(host): + if host.startswith('[') and host.endswith(']'): + host = host[1:-1] + return _socket.AF_INET6, host + return _socket.AF_INET, host + + +def _parse_address(address): + if isinstance(address, tuple): + if not address[0] or ':' in address[0]: + return _socket.AF_INET6, address + return _socket.AF_INET, address + + if ((isinstance(address, string_types) and ':' not in address) + or isinstance(address, integer_types)): # noqa (pep8 E129) + # Just a port + return _socket.AF_INET6, ('', int(address)) + + if not isinstance(address, string_types): + raise TypeError('Expected tuple or string, got %s' % type(address)) + + host, port = address.rsplit(':', 1) + family, host = _extract_family(host) + if host == '*': + host = '' + return family, (host, int(port)) + + +def parse_address(address): + try: + return _parse_address(address) + except ValueError as ex: # pylint:disable=try-except-raise + raise ValueError('Failed to parse address %r: %s' % (address, ex)) diff --git a/src/gevent/builtins.py b/src/gevent/builtins.py new file mode 100644 index 0000000..389a652 --- /dev/null +++ b/src/gevent/builtins.py @@ -0,0 +1,135 @@ +# Copyright (c) 2015 gevent contributors. See LICENSE for details. +"""gevent friendly implementations of builtin functions.""" +from __future__ import absolute_import + +import sys +import weakref + +from gevent.lock import RLock +from gevent._compat import imp_acquire_lock +from gevent._compat import imp_release_lock + + +# Normally we'd have the "expected" case inside the try +# (Python 3, because Python 3 is the way forward). But +# under Python 2, the popular `future` library *also* provides +# a `builtins` module---which lacks the __import__ attribute. +# So we test for the old, deprecated version first + +try: # Py2 + import __builtin__ as __gbuiltins__ + _allowed_module_name_types = (basestring,) # pylint:disable=undefined-variable + __target__ = '__builtin__' +except ImportError: + import builtins as __gbuiltins__ # pylint: disable=import-error + _allowed_module_name_types = (str,) + __target__ = 'builtins' + +_import = __gbuiltins__.__import__ + +# We need to protect imports both across threads and across greenlets. +# And the order matters. Note that under 3.4, the global import lock +# and imp module are deprecated. It seems that in all Py3 versions, a +# module lock is used such that this fix is not necessary. + +# We emulate the per-module locking system under Python 2 in order to +# avoid issues acquiring locks in multiple-level-deep imports +# that attempt to use the gevent blocking API at runtime; using one lock +# could lead to a LoopExit error as a greenlet attempts to block on it while +# it's already held by the main greenlet (issue #798). + +# We base this approach on a simplification of what `importlib._bootstrap` +# does; notably, we don't check for deadlocks + +_g_import_locks = {} # name -> wref of RLock + +__lock_imports = True + + +def __module_lock(name): + # Return the lock for the given module, creating it if necessary. + # It will be removed when no longer needed. + # Nothing in this function yields, so we're multi-greenlet safe + # (But not multi-threading safe.) + # XXX: What about on PyPy, where the GC is asynchronous (not ref-counting)? + # (Does it stop-the-world first?) + lock = None + try: + lock = _g_import_locks[name]() + except KeyError: + pass + + if lock is None: + lock = RLock() + + def cb(_): + # We've seen a KeyError on PyPy on RPi2 + _g_import_locks.pop(name, None) + _g_import_locks[name] = weakref.ref(lock, cb) + return lock + + +def __import__(*args, **kwargs): + """ + __import__(name, globals=None, locals=None, fromlist=(), level=0) -> object + + Normally python protects imports against concurrency by doing some locking + at the C level (at least, it does that in CPython). This function just + wraps the normal __import__ functionality in a recursive lock, ensuring that + we're protected against greenlet import concurrency as well. + """ + if args and not issubclass(type(args[0]), _allowed_module_name_types): + # if a builtin has been acquired as a bound instance method, + # python knows not to pass 'self' when the method is called. + # No such protection exists for monkey-patched builtins, + # however, so this is necessary. + args = args[1:] + + if not __lock_imports: + return _import(*args, **kwargs) + + module_lock = __module_lock(args[0]) # Get a lock for the module name + imp_acquire_lock() + try: + module_lock.acquire() + try: + result = _import(*args, **kwargs) + finally: + module_lock.release() + finally: + imp_release_lock() + return result + + +def _unlock_imports(): + """ + Internal function, called when gevent needs to perform imports + lazily, but does not know the state of the system. It may be impossible + to take the import lock because there are no other running greenlets, for + example. This causes a monkey-patched __import__ to avoid taking any locks. + until the corresponding call to lock_imports. This should only be done for limited + amounts of time and when the set of imports is statically known to be "safe". + """ + global __lock_imports + # This could easily become a list that we push/pop from or an integer + # we increment if we need to do this recursively, but we shouldn't get + # that complex. + __lock_imports = False + + +def _lock_imports(): + global __lock_imports + __lock_imports = True + +if sys.version_info[:2] >= (3, 3): + __implements__ = [] + __import__ = _import +else: + __implements__ = ['__import__'] +__all__ = __implements__ + + +from gevent._util import copy_globals + +__imports__ = copy_globals(__gbuiltins__, globals(), + names_to_ignore=__implements__) diff --git a/src/gevent/core.py b/src/gevent/core.py new file mode 100644 index 0000000..906e739 --- /dev/null +++ b/src/gevent/core.py @@ -0,0 +1,20 @@ +# Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details. +""" +Deprecated; this does not reflect all the possible options +and its interface varies. + +.. versionchanged:: 1.3a2 + Deprecated. +""" +from __future__ import absolute_import + +import sys + +from gevent._config import config +from gevent._util import copy_globals + +_core = sys.modules[config.loop.__module__] + +copy_globals(_core, globals()) + +__all__ = _core.__all__ # pylint:disable=no-member diff --git a/src/gevent/event.py b/src/gevent/event.py new file mode 100644 index 0000000..9a0d402 --- /dev/null +++ b/src/gevent/event.py @@ -0,0 +1,363 @@ +# Copyright (c) 2009-2016 Denis Bilenko, gevent contributors. See LICENSE for details. +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False,infer_types=True + +"""Basic synchronization primitives: Event and AsyncResult""" +from __future__ import print_function + +from gevent._util import _NONE +from gevent._compat import reraise +from gevent._tblib import dump_traceback, load_traceback + +from gevent.timeout import Timeout + + +__all__ = [ + 'Event', + 'AsyncResult', +] + +def _get_linkable(): + x = __import__('gevent._abstract_linkable') + return x._abstract_linkable.AbstractLinkable +locals()['AbstractLinkable'] = _get_linkable() +del _get_linkable + +# Sadly, something about the way we have to "import" AbstractLinkable +# breaks pylint's inference of slots, even though they're declared +# right here. +# pylint:disable=assigning-non-slot + +class Event(AbstractLinkable): # pylint:disable=undefined-variable + """A synchronization primitive that allows one greenlet to wake up one or more others. + It has the same interface as :class:`threading.Event` but works across greenlets. + + An event object manages an internal flag that can be set to true with the + :meth:`set` method and reset to false with the :meth:`clear` method. The :meth:`wait` method + blocks until the flag is true. + + .. note:: + The order and timing in which waiting greenlets are awakened is not determined. + As an implementation note, in gevent 1.1 and 1.0, waiting greenlets are awakened in a + undetermined order sometime *after* the current greenlet yields to the event loop. Other greenlets + (those not waiting to be awakened) may run between the current greenlet yielding and + the waiting greenlets being awakened. These details may change in the future. + """ + + __slots__ = ('_flag',) + + def __init__(self): + super(Event, self).__init__() + self._flag = False + + def __str__(self): + return '<%s %s _links[%s]>' % (self.__class__.__name__, (self._flag and 'set') or 'clear', + self.linkcount()) + + def is_set(self): + """Return true if and only if the internal flag is true.""" + return self._flag + + def isSet(self): + # makes it a better drop-in replacement for threading.Event + return self._flag + + def ready(self): + # makes it compatible with AsyncResult and Greenlet (for + # example in wait()) + return self._flag + + def set(self): + """ + Set the internal flag to true. + + All greenlets waiting for it to become true are awakened in + some order at some time in the future. Greenlets that call + :meth:`wait` once the flag is true will not block at all + (until :meth:`clear` is called). + """ + self._flag = True + self._check_and_notify() + + def clear(self): + """ + Reset the internal flag to false. + + Subsequently, threads calling :meth:`wait` will block until + :meth:`set` is called to set the internal flag to true again. + """ + self._flag = False + + def _wait_return_value(self, waited, wait_success): + # To avoid the race condition outlined in http://bugs.python.org/issue13502, + # if we had to wait, then we need to return whether or not + # the condition got changed. Otherwise we simply echo + # the current state of the flag (which should be true) + if not waited: + flag = self._flag + assert flag, "if we didn't wait we should already be set" + return flag + + return wait_success + + def wait(self, timeout=None): + """ + Block until the internal flag is true. + + If the internal flag is true on entry, return immediately. Otherwise, + block until another thread (greenlet) calls :meth:`set` to set the flag to true, + or until the optional timeout occurs. + + When the *timeout* argument is present and not ``None``, it should be a + floating point number specifying a timeout for the operation in seconds + (or fractions thereof). + + :return: This method returns true if and only if the internal flag has been set to + true, either before the wait call or after the wait starts, so it will + always return ``True`` except if a timeout is given and the operation + times out. + + .. versionchanged:: 1.1 + The return value represents the flag during the elapsed wait, not + just after it elapses. This solves a race condition if one greenlet + sets and then clears the flag without switching, while other greenlets + are waiting. When the waiters wake up, this will return True; previously, + they would still wake up, but the return value would be False. This is most + noticeable when the *timeout* is present. + """ + return self._wait(timeout) + + def _reset_internal_locks(self): # pragma: no cover + # for compatibility with threading.Event + # Exception AttributeError: AttributeError("'Event' object has no attribute '_reset_internal_locks'",) + # in ignored + pass + + +class AsyncResult(AbstractLinkable): # pylint:disable=undefined-variable + """A one-time event that stores a value or an exception. + + Like :class:`Event` it wakes up all the waiters when :meth:`set` or :meth:`set_exception` + is called. Waiters may receive the passed value or exception by calling :meth:`get` + instead of :meth:`wait`. An :class:`AsyncResult` instance cannot be reset. + + To pass a value call :meth:`set`. Calls to :meth:`get` (those that are currently blocking as well as + those made in the future) will return the value: + + >>> result = AsyncResult() + >>> result.set(100) + >>> result.get() + 100 + + To pass an exception call :meth:`set_exception`. This will cause :meth:`get` to raise that exception: + + >>> result = AsyncResult() + >>> result.set_exception(RuntimeError('failure')) + >>> result.get() + Traceback (most recent call last): + ... + RuntimeError: failure + + :class:`AsyncResult` implements :meth:`__call__` and thus can be used as :meth:`link` target: + + >>> import gevent + >>> result = AsyncResult() + >>> gevent.spawn(lambda : 1/0).link(result) + >>> try: + ... result.get() + ... except ZeroDivisionError: + ... print('ZeroDivisionError') + ZeroDivisionError + + .. note:: + The order and timing in which waiting greenlets are awakened is not determined. + As an implementation note, in gevent 1.1 and 1.0, waiting greenlets are awakened in a + undetermined order sometime *after* the current greenlet yields to the event loop. Other greenlets + (those not waiting to be awakened) may run between the current greenlet yielding and + the waiting greenlets being awakened. These details may change in the future. + + .. versionchanged:: 1.1 + The exact order in which waiting greenlets are awakened is not the same + as in 1.0. + .. versionchanged:: 1.1 + Callbacks :meth:`linked ` to this object are required to be hashable, and duplicates are + merged. + """ + + __slots__ = ('_value', '_exc_info', '_imap_task_index') + + def __init__(self): + super(AsyncResult, self).__init__() + self._value = _NONE + self._exc_info = () + + @property + def _exception(self): + return self._exc_info[1] if self._exc_info else _NONE + + @property + def value(self): + """ + Holds the value passed to :meth:`set` if :meth:`set` was called. Otherwise, + ``None`` + """ + return self._value if self._value is not _NONE else None + + @property + def exc_info(self): + """ + The three-tuple of exception information if :meth:`set_exception` was called. + """ + if self._exc_info: + return (self._exc_info[0], self._exc_info[1], load_traceback(self._exc_info[2])) + return () + + def __str__(self): + result = '<%s ' % (self.__class__.__name__, ) + if self.value is not None or self._exception is not _NONE: + result += 'value=%r ' % self.value + if self._exception is not None and self._exception is not _NONE: + result += 'exception=%r ' % self._exception + if self._exception is _NONE: + result += 'unset ' + return result + ' _links[%s]>' % self.linkcount() + + def ready(self): + """Return true if and only if it holds a value or an exception""" + return self._exc_info or self._value is not _NONE + + def successful(self): + """Return true if and only if it is ready and holds a value""" + return self._value is not _NONE + + @property + def exception(self): + """Holds the exception instance passed to :meth:`set_exception` if :meth:`set_exception` was called. + Otherwise ``None``.""" + if self._exc_info: + return self._exc_info[1] + + def set(self, value=None): + """Store the value and wake up any waiters. + + All greenlets blocking on :meth:`get` or :meth:`wait` are awakened. + Subsequent calls to :meth:`wait` and :meth:`get` will not block at all. + """ + self._value = value + self._check_and_notify() + + def set_exception(self, exception, exc_info=None): + """Store the exception and wake up any waiters. + + All greenlets blocking on :meth:`get` or :meth:`wait` are awakened. + Subsequent calls to :meth:`wait` and :meth:`get` will not block at all. + + :keyword tuple exc_info: If given, a standard three-tuple of type, value, :class:`traceback` + as returned by :func:`sys.exc_info`. This will be used when the exception + is re-raised to propagate the correct traceback. + """ + if exc_info: + self._exc_info = (exc_info[0], exc_info[1], dump_traceback(exc_info[2])) + else: + self._exc_info = (type(exception), exception, dump_traceback(None)) + + self._check_and_notify() + + def _raise_exception(self): + reraise(*self.exc_info) + + def get(self, block=True, timeout=None): + """Return the stored value or raise the exception. + + If this instance already holds a value or an exception, return or raise it immediately. + Otherwise, block until another greenlet calls :meth:`set` or :meth:`set_exception` or + until the optional timeout occurs. + + When the *timeout* argument is present and not ``None``, it should be a + floating point number specifying a timeout for the operation in seconds + (or fractions thereof). If the *timeout* elapses, the *Timeout* exception will + be raised. + + :keyword bool block: If set to ``False`` and this instance is not ready, + immediately raise a :class:`Timeout` exception. + """ + if self._value is not _NONE: + return self._value + if self._exc_info: + return self._raise_exception() + + if not block: + # Not ready and not blocking, so immediately timeout + raise Timeout() + + # Wait, raising a timeout that elapses + self._wait_core(timeout, ()) + + # by definition we are now ready + return self.get(block=False) + + def get_nowait(self): + """ + Return the value or raise the exception without blocking. + + If this object is not yet :meth:`ready `, raise + :class:`gevent.Timeout` immediately. + """ + return self.get(block=False) + + def _wait_return_value(self, waited, wait_success): + # pylint:disable=unused-argument + # Always return the value. Since this is a one-shot event, + # no race condition should reset it. + return self.value + + def wait(self, timeout=None): + """Block until the instance is ready. + + If this instance already holds a value, it is returned immediately. If this + instance already holds an exception, ``None`` is returned immediately. + + Otherwise, block until another greenlet calls :meth:`set` or :meth:`set_exception` + (at which point either the value or ``None`` will be returned, respectively), + or until the optional timeout expires (at which point ``None`` will also be + returned). + + When the *timeout* argument is present and not ``None``, it should be a + floating point number specifying a timeout for the operation in seconds + (or fractions thereof). + + .. note:: If a timeout is given and expires, ``None`` will be returned + (no timeout exception will be raised). + + """ + return self._wait(timeout) + + # link protocol + def __call__(self, source): + if source.successful(): + self.set(source.value) + else: + self.set_exception(source.exception, getattr(source, 'exc_info', None)) + + # Methods to make us more like concurrent.futures.Future + + def result(self, timeout=None): + return self.get(timeout=timeout) + + set_result = set + + def done(self): + return self.ready() + + # we don't support cancelling + + def cancel(self): + return False + + def cancelled(self): + return False + + # exception is a method, we use it as a property + + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent._event') diff --git a/src/gevent/events.py b/src/gevent/events.py new file mode 100644 index 0000000..12c02ec --- /dev/null +++ b/src/gevent/events.py @@ -0,0 +1,480 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 gevent. See LICENSE for details. +""" +Publish/subscribe event infrastructure. + +When certain "interesting" things happen during the lifetime of the +process, gevent will "publish" an event (an object). That event is +delivered to interested "subscribers" (functions that take one +parameter, the event object). + +Higher level frameworks may take this foundation and build richer +models on it. + +If :mod:`zope.event` is installed, then it will be used to provide the +functionality of `notify` and `subscribers`. See +:mod:`zope.event.classhandler` for a simple class-based approach to +subscribing to a filtered list of events, and see `zope.component +`_ for a +much higher-level, flexible system. If you are using one of these systems, +you generally will not want to directly modify `subscribers`. + +.. versionadded:: 1.3b1 +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +__all__ = [ + 'subscribers', + + # monitor thread + 'IEventLoopBlocked', + 'EventLoopBlocked', + 'IMemoryUsageThresholdExceeded', + 'MemoryUsageThresholdExceeded', + 'IMemoryUsageUnderThreshold', + 'MemoryUsageUnderThreshold', + + # Hub + 'IPeriodicMonitorThread', + 'IPeriodicMonitorThreadStartedEvent', + 'PeriodicMonitorThreadStartedEvent', + + # monkey + 'IGeventPatchEvent', + 'GeventPatchEvent', + 'IGeventWillPatchEvent', + 'DoNotPatch', + 'GeventWillPatchEvent', + 'IGeventDidPatchEvent', + 'IGeventWillPatchModuleEvent', + 'GeventWillPatchModuleEvent', + 'IGeventDidPatchModuleEvent', + 'GeventDidPatchModuleEvent', + 'IGeventWillPatchAllEvent', + 'GeventWillPatchAllEvent', + 'IGeventDidPatchBuiltinModulesEvent', + 'GeventDidPatchBuiltinModulesEvent', + 'IGeventDidPatchAllEvent', + 'GeventDidPatchAllEvent', +] + +# pylint:disable=no-self-argument + +try: + from zope.event import subscribers + from zope.event import notify +except ImportError: + #: Applications may register for notification of events by appending a + #: callable to the ``subscribers`` list. + #: + #: Each subscriber takes a single argument, which is the event object + #: being published. + #: + #: Exceptions raised by subscribers will be propagated *without* running + #: any remaining subscribers. + subscribers = [] + + def notify(event): + """ + Notify all subscribers of ``event``. + """ + for subscriber in subscribers: + subscriber(event) + +notify = notify # export +try: + # pkg_resources is technically optional, we don't + # list a hard dependency on it. + __import__('pkg_resources') +except ImportError: + notify_and_call_entry_points = notify +else: + from pkg_resources import iter_entry_points + import platform + try: + # Cache the platform info. pkg_resources uses + # platform.machine() for environment markers, and + # platform.machine() wants to call os.popen('uname'), which is + # broken on Py2 when the gevent child signal handler is + # installed. (see test__monkey_sigchild_2.py) + platform.uname() + except: # pylint:disable=bare-except + pass + finally: + del platform + + def notify_and_call_entry_points(event): + notify(event) + for plugin in iter_entry_points(event.ENTRY_POINT_NAME): + subscriber = plugin.load() + subscriber(event) + +from gevent._util import Interface +from gevent._util import implementer +from gevent._util import Attribute + + +class IPeriodicMonitorThread(Interface): + """ + The contract for the periodic monitoring thread that is started + by the hub. + """ + + def add_monitoring_function(function, period): + """ + Schedule the *function* to be called approximately every *period* fractional seconds. + + The *function* receives one argument, the hub being monitored. It is called + in the monitoring thread, *not* the hub thread. It **must not** attempt to + use the gevent asynchronous API. + + If the *function* is already a monitoring function, then its *period* + will be updated for future runs. + + If the *period* is ``None``, then the function will be removed. + + A *period* less than or equal to zero is not allowed. + """ + +class IPeriodicMonitorThreadStartedEvent(Interface): + """ + The event emitted when a hub starts a periodic monitoring thread. + + You can use this event to add additional monitoring functions. + """ + + monitor = Attribute("The instance of `IPeriodicMonitorThread` that was started.") + +class PeriodicMonitorThreadStartedEvent(object): + """ + The implementation of :class:`IPeriodicMonitorThreadStartedEvent`. + """ + + #: The name of the setuptools entry point that is called when this + #: event is emitted. + ENTRY_POINT_NAME = 'gevent.plugins.hub.periodic_monitor_thread_started' + + def __init__(self, monitor): + self.monitor = monitor + +class IEventLoopBlocked(Interface): + """ + The event emitted when the event loop is blocked. + + This event is emitted in the monitor thread. + """ + + greenlet = Attribute("The greenlet that appeared to be blocking the loop.") + blocking_time = Attribute("The approximate time in seconds the loop has been blocked.") + info = Attribute("A sequence of string lines providing extra info.") + +@implementer(IEventLoopBlocked) +class EventLoopBlocked(object): + """ + The event emitted when the event loop is blocked. + + Implements `IEventLoopBlocked`. + """ + + def __init__(self, greenlet, blocking_time, info): + self.greenlet = greenlet + self.blocking_time = blocking_time + self.info = info + +class IMemoryUsageThresholdExceeded(Interface): + """ + The event emitted when the memory usage threshold is exceeded. + + This event is emitted only while memory continues to grow + above the threshold. Only if the condition or stabilized is corrected (memory + usage drops) will the event be emitted in the future. + + This event is emitted in the monitor thread. + """ + + mem_usage = Attribute("The current process memory usage, in bytes.") + max_allowed = Attribute("The maximum allowed memory usage, in bytes.") + memory_info = Attribute("The tuple of memory usage stats return by psutil.") + +class _AbstractMemoryEvent(object): + + def __init__(self, mem_usage, max_allowed, memory_info): + self.mem_usage = mem_usage + self.max_allowed = max_allowed + self.memory_info = memory_info + + def __repr__(self): + return "<%s used=%d max=%d details=%r>" % ( + self.__class__.__name__, + self.mem_usage, + self.max_allowed, + self.memory_info, + ) + +@implementer(IMemoryUsageThresholdExceeded) +class MemoryUsageThresholdExceeded(_AbstractMemoryEvent): + """ + Implementation of `IMemoryUsageThresholdExceeded`. + """ + + +class IMemoryUsageUnderThreshold(Interface): + """ + The event emitted when the memory usage drops below the + threshold after having previously been above it. + + This event is emitted only the first time memory usage is detected + to be below the threshold after having previously been above it. + If memory usage climbs again, a `IMemoryUsageThresholdExceeded` + event will be broadcast, and then this event could be broadcast again. + + This event is emitted in the monitor thread. + """ + + mem_usage = Attribute("The current process memory usage, in bytes.") + max_allowed = Attribute("The maximum allowed memory usage, in bytes.") + max_memory_usage = Attribute("The memory usage that caused the previous " + "IMemoryUsageThresholdExceeded event.") + memory_info = Attribute("The tuple of memory usage stats return by psutil.") + + +@implementer(IMemoryUsageUnderThreshold) +class MemoryUsageUnderThreshold(_AbstractMemoryEvent): + """ + Implementation of `IMemoryUsageUnderThreshold`. + """ + + def __init__(self, mem_usage, max_allowed, memory_info, max_usage): + super(MemoryUsageUnderThreshold, self).__init__(mem_usage, max_allowed, memory_info) + self.max_memory_usage = max_usage + + +class IGeventPatchEvent(Interface): + """ + The root for all monkey-patch events gevent emits. + """ + + source = Attribute("The source object containing the patches.") + target = Attribute("The destination object to be patched.") + +@implementer(IGeventPatchEvent) +class GeventPatchEvent(object): + """ + Implementation of `IGeventPatchEvent`. + """ + + def __init__(self, source, target): + self.source = source + self.target = target + + def __repr__(self): + return '<%s source=%r target=%r at %x>' % (self.__class__.__name__, + self.source, + self.target, + id(self)) + +class IGeventWillPatchEvent(IGeventPatchEvent): + """ + An event emitted *before* gevent monkey-patches something. + + If a subscriber raises `DoNotPatch`, then patching this particular + item will not take place. + """ + + +class DoNotPatch(BaseException): + """ + Subscribers to will-patch events can raise instances + of this class to tell gevent not to patch that particular item. + """ + + +@implementer(IGeventWillPatchEvent) +class GeventWillPatchEvent(GeventPatchEvent): + """ + Implementation of `IGeventWillPatchEvent`. + """ + +class IGeventDidPatchEvent(IGeventPatchEvent): + """ + An event emitted *after* gevent has patched something. + """ + +@implementer(IGeventDidPatchEvent) +class GeventDidPatchEvent(GeventPatchEvent): + """ + Implementation of `IGeventDidPatchEvent`. + """ + +class IGeventWillPatchModuleEvent(IGeventWillPatchEvent): + """ + An event emitted *before* gevent begins patching a specific module. + + Both *source* and *target* attributes are module objects. + """ + + module_name = Attribute("The name of the module being patched. " + "This is the same as ``target.__name__``.") + + target_item_names = Attribute("The list of item names to patch. " + "This can be modified in place with caution.") + +@implementer(IGeventWillPatchModuleEvent) +class GeventWillPatchModuleEvent(GeventWillPatchEvent): + """ + Implementation of `IGeventWillPatchModuleEvent`. + """ + + #: The name of the setuptools entry point that is called when this + #: event is emitted. + ENTRY_POINT_NAME = 'gevent.plugins.monkey.will_patch_module' + + def __init__(self, module_name, source, target, items): + super(GeventWillPatchModuleEvent, self).__init__(source, target) + self.module_name = module_name + self.target_item_names = items + + +class IGeventDidPatchModuleEvent(IGeventDidPatchEvent): + """ + An event emitted *after* gevent has completed patching a specific + module. + """ + + module_name = Attribute("The name of the module being patched. " + "This is the same as ``target.__name__``.") + + +@implementer(IGeventDidPatchModuleEvent) +class GeventDidPatchModuleEvent(GeventDidPatchEvent): + """ + Implementation of `IGeventDidPatchModuleEvent`. + """ + + #: The name of the setuptools entry point that is called when this + #: event is emitted. + ENTRY_POINT_NAME = 'gevent.plugins.monkey.did_patch_module' + + def __init__(self, module_name, source, target): + super(GeventDidPatchModuleEvent, self).__init__(source, target) + self.module_name = module_name + +# TODO: Maybe it would be useful for the the module patch events +# to have an attribute telling if they're being done during patch_all? + +class IGeventWillPatchAllEvent(IGeventWillPatchEvent): + """ + An event emitted *before* gevent begins patching the system. + + Following this event will be a series of + `IGeventWillPatchModuleEvent` and `IGeventDidPatchModuleEvent` for + each patched module. + + Once the gevent builtin modules have been processed, + `IGeventDidPatchBuiltinModulesEvent` will be emitted. Processing + this event is an ideal time for third-party modules to be imported + and patched (which may trigger its own will/did patch module + events). + + Finally, a `IGeventDidPatchAllEvent` will be sent. + + If a subscriber to this event raises `DoNotPatch`, no patching + will be done. + + The *source* and *target* attributes have undefined values. + """ + + patch_all_arguments = Attribute( + "A dictionary of all the arguments to `gevent.monkey.patch_all`. " + "This dictionary should not be modified. " + ) + + patch_all_kwargs = Attribute( + "A dictionary of the extra arguments to `gevent.monkey.patch_all`. " + "This dictionary should not be modified. " + ) + + def will_patch_module(module_name): + """ + Return whether the module named *module_name* will be patched. + """ + +class _PatchAllMixin(object): + def __init__(self, patch_all_arguments, patch_all_kwargs): + super(_PatchAllMixin, self).__init__(None, None) + self._patch_all_arguments = patch_all_arguments + self._patch_all_kwargs = patch_all_kwargs + + @property + def patch_all_arguments(self): + return self._patch_all_arguments.copy() + + @property + def patch_all_kwargs(self): + return self._patch_all_kwargs.copy() + + def __repr__(self): + return '<%s %r at %x>' % (self.__class__.__name__, + self._patch_all_arguments, + id(self)) + +@implementer(IGeventWillPatchAllEvent) +class GeventWillPatchAllEvent(_PatchAllMixin, GeventWillPatchEvent): + """ + Implementation of `IGeventWillPatchAllEvent`. + """ + + #: The name of the setuptools entry point that is called when this + #: event is emitted. + ENTRY_POINT_NAME = 'gevent.plugins.monkey.will_patch_all' + + def will_patch_module(self, module_name): + return self.patch_all_arguments.get(module_name) + +class IGeventDidPatchBuiltinModulesEvent(IGeventDidPatchEvent): + """ + Event emitted *after* the builtin modules have been patched. + + The values of the *source* and *target* attributes are undefined. + """ + + patch_all_arguments = Attribute( + "A dictionary of all the arguments to `gevent.monkey.patch_all`. " + "This dictionary should not be modified. " + ) + + patch_all_kwargs = Attribute( + "A dictionary of the extra arguments to `gevent.monkey.patch_all`. " + "This dictionary should not be modified. " + ) + +@implementer(IGeventDidPatchBuiltinModulesEvent) +class GeventDidPatchBuiltinModulesEvent(_PatchAllMixin, GeventDidPatchEvent): + """ + Implementation of `IGeventDidPatchBuiltinModulesEvent`. + """ + + #: The name of the setuptools entry point that is called when this + #: event is emitted. + ENTRY_POINT_NAME = 'gevent.plugins.monkey.did_patch_builtins' + +class IGeventDidPatchAllEvent(IGeventDidPatchEvent): + """ + Event emitted after gevent has patched all modules, both builtin + and those provided by plugins/subscribers. + + The values of the *source* and *target* attributes are undefined. + """ + +@implementer(IGeventDidPatchAllEvent) +class GeventDidPatchAllEvent(_PatchAllMixin, GeventDidPatchEvent): + """ + Implementation of `IGeventDidPatchAllEvent`. + """ + + #: The name of the setuptools entry point that is called when this + #: event is emitted. + ENTRY_POINT_NAME = 'gevent.plugins.monkey.did_patch_all' diff --git a/src/gevent/exceptions.py b/src/gevent/exceptions.py new file mode 100644 index 0000000..2c779ce --- /dev/null +++ b/src/gevent/exceptions.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +# copyright 2018 gevent +""" +Exceptions. + +.. versionadded:: 1.3b1 + +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +__all__ = [ + 'LoopExit', +] + + +class LoopExit(Exception): + """ + Exception thrown when the hub finishes running (`gevent.hub.Hub.run` + would return). + + In a normal application, this is never thrown or caught + explicitly. The internal implementation of functions like + :meth:`gevent.hub.Hub.join` and :func:`gevent.joinall` may catch it, but user code + generally should not. + + .. caution:: + Errors in application programming can also lead to this exception being + raised. Some examples include (but are not limited too): + + - greenlets deadlocking on a lock; + - using a socket or other gevent object with native thread + affinity from a different thread + + """ + + def __repr__(self): + # pylint:disable=unsubscriptable-object + if len(self.args) == 3: # From the hub + import pprint + return "%s\n\tHub: %s\n\tHandles:\n%s" % ( + self.args[0], self.args[1], + pprint.pformat(self.args[2]) + ) + return Exception.__repr__(self) + + def __str__(self): + return repr(self) + +class BlockingSwitchOutError(AssertionError): + """ + Raised when a gevent synchronous function is called from a + low-level event loop callback. + + This is usually a programming error. + """ + + +class InvalidSwitchError(AssertionError): + """ + Raised when the event loop returns control to a greenlet in an + unexpected way. + + This is usually a bug in gevent, greenlet, or the event loop. + """ + +class ConcurrentObjectUseError(AssertionError): + """ + Raised when an object is used (waited on) by two greenlets + independently, meaning the object was entered into a blocking + state by one greenlet and then another while still blocking in the + first one. + + This is usually a programming error. + + .. seealso:: `gevent.socket.wait` + """ diff --git a/src/gevent/fileobject.py b/src/gevent/fileobject.py new file mode 100644 index 0000000..598f882 --- /dev/null +++ b/src/gevent/fileobject.py @@ -0,0 +1,61 @@ +""" +Wrappers to make file-like objects cooperative. + +.. class:: FileObject + + The main entry point to the file-like gevent-compatible behaviour. It will be defined + to be the best available implementation. + +There are two main implementations of ``FileObject``. On all systems, +there is :class:`FileObjectThread` which uses the built-in native +threadpool to avoid blocking the entire interpreter. On UNIX systems +(those that support the :mod:`fcntl` module), there is also +:class:`FileObjectPosix` which uses native non-blocking semantics. + +A third class, :class:`FileObjectBlock`, is simply a wrapper that executes everything +synchronously (and so is not gevent-compatible). It is provided for testing and debugging +purposes. + +Configuration +============= + +You may change the default value for ``FileObject`` using the +``GEVENT_FILE`` environment variable. Set it to ``posix``, ``thread``, +or ``block`` to choose from :class:`FileObjectPosix`, +:class:`FileObjectThread` and :class:`FileObjectBlock`, respectively. +You may also set it to the fully qualified class name of another +object that implements the file interface to use one of your own +objects. + +.. note:: The environment variable must be set at the time this module + is first imported. + +Classes +======= +""" +from __future__ import absolute_import + +from gevent._config import config + +__all__ = [ + 'FileObjectPosix', + 'FileObjectThread', + 'FileObjectBlock', + 'FileObject', +] + +try: + from fcntl import fcntl +except ImportError: + __all__.remove("FileObjectPosix") +else: + del fcntl + from gevent._fileobjectposix import FileObjectPosix + +from gevent._fileobjectcommon import FileObjectThread +from gevent._fileobjectcommon import FileObjectBlock + + +# None of the possible objects can live in this module because +# we would get an import cycle and the config couldn't be set from code. +FileObject = config.fileobject diff --git a/src/gevent/greenlet.py b/src/gevent/greenlet.py new file mode 100644 index 0000000..a226856 --- /dev/null +++ b/src/gevent/greenlet.py @@ -0,0 +1,1003 @@ +# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details. +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False + +from __future__ import absolute_import, print_function, division + +from sys import _getframe as sys_getframe +from sys import exc_info as sys_exc_info +from weakref import ref as wref + +# XXX: How to get cython to let us rename this as RawGreenlet +# like we prefer? +from greenlet import greenlet +from greenlet import GreenletExit + +from gevent._compat import reraise +from gevent._compat import PYPY as _PYPY +from gevent._tblib import dump_traceback +from gevent._tblib import load_traceback + +from gevent.exceptions import InvalidSwitchError + +from gevent._hub_primitives import iwait_on_objects as iwait +from gevent._hub_primitives import wait_on_objects as wait + +from gevent.timeout import Timeout + +from gevent._config import config as GEVENT_CONFIG +from gevent._util import Lazy +from gevent._util import readproperty +from gevent._hub_local import get_hub_noargs as get_hub +from gevent import _waiter + + +__all__ = [ + 'Greenlet', + 'joinall', + 'killall', +] + + +# In Cython, we define these as 'cdef inline' functions. The +# compilation unit cannot have a direct assignment to them (import +# is assignment) without generating a 'lvalue is not valid target' +# error. +locals()['getcurrent'] = __import__('greenlet').getcurrent +locals()['greenlet_init'] = lambda: None +locals()['Waiter'] = _waiter.Waiter + + +if _PYPY: + import _continuation # pylint:disable=import-error + _continulet = _continuation.continulet + + +class SpawnedLink(object): + """ + A wrapper around link that calls it in another greenlet. + + Can be called only from main loop. + """ + __slots__ = ['callback'] + + def __init__(self, callback): + if not callable(callback): + raise TypeError("Expected callable: %r" % (callback, )) + self.callback = callback + + def __call__(self, source): + g = greenlet(self.callback, get_hub()) + g.switch(source) + + def __hash__(self): + return hash(self.callback) + + def __eq__(self, other): + return self.callback == getattr(other, 'callback', other) + + def __str__(self): + return str(self.callback) + + def __repr__(self): + return repr(self.callback) + + def __getattr__(self, item): + assert item != 'callback' + return getattr(self.callback, item) + + +class SuccessSpawnedLink(SpawnedLink): + """A wrapper around link that calls it in another greenlet only if source succeed. + + Can be called only from main loop. + """ + __slots__ = [] + + def __call__(self, source): + if source.successful(): + return SpawnedLink.__call__(self, source) + + +class FailureSpawnedLink(SpawnedLink): + """A wrapper around link that calls it in another greenlet only if source failed. + + Can be called only from main loop. + """ + __slots__ = [] + + def __call__(self, source): + if not source.successful(): + return SpawnedLink.__call__(self, source) + +class _Frame(object): + + __slots__ = ('f_code', 'f_lineno', 'f_back') + + def __init__(self, f_code, f_lineno, f_back): + self.f_code = f_code + self.f_lineno = f_lineno + self.f_back = f_back + + @property + def f_globals(self): + return None + +def _Frame_from_list(frames): + previous = None + for frame in reversed(frames): + f = _Frame(frame[0], frame[1], previous) + previous = f + return previous + +def _extract_stack(limit): + try: + frame = sys_getframe() + except ValueError: + # In certain embedded cases that directly use the Python C api + # to call Greenlet.spawn (e.g., uwsgi) this can raise + # `ValueError: call stack is not deep enough`. This is because + # the Cython stack frames for Greenlet.spawn -> + # Greenlet.__init__ -> _extract_stack are all on the C level, + # not the Python level. + # See https://github.com/gevent/gevent/issues/1212 + frame = None + + frames = [] + + while limit and frame is not None: + limit -= 1 + frames.append((frame.f_code, frame.f_lineno)) + frame = frame.f_back + + return frames + + +_greenlet__init__ = greenlet.__init__ + +class Greenlet(greenlet): + """ + A light-weight cooperatively-scheduled execution unit. + """ + # pylint:disable=too-many-public-methods,too-many-instance-attributes + + spawning_stack_limit = 10 + + # pylint:disable=keyword-arg-before-vararg,super-init-not-called + def __init__(self, run=None, *args, **kwargs): + """ + :param args: The arguments passed to the ``run`` function. + :param kwargs: The keyword arguments passed to the ``run`` function. + :keyword callable run: The callable object to run. If not given, this object's + `_run` method will be invoked (typically defined by subclasses). + + .. versionchanged:: 1.1b1 + The ``run`` argument to the constructor is now verified to be a callable + object. Previously, passing a non-callable object would fail after the greenlet + was spawned. + + .. versionchanged:: 1.3b1 + The ``GEVENT_TRACK_GREENLET_TREE`` configuration value may be set to + a false value to disable ``spawn_tree_locals``, ``spawning_greenlet``, + and ``spawning_stack``. The first two will be None in that case, and the + latter will be empty. + """ + # The attributes are documented in the .rst file + + # greenlet.greenlet(run=None, parent=None) + # Calling it with both positional arguments instead of a keyword + # argument (parent=get_hub()) speeds up creation of this object ~30%: + # python -m timeit -s 'import gevent' 'gevent.Greenlet()' + # Python 3.5: 2.70usec with keywords vs 1.94usec with positional + # Python 3.4: 2.32usec with keywords vs 1.74usec with positional + # Python 3.3: 2.55usec with keywords vs 1.92usec with positional + # Python 2.7: 1.73usec with keywords vs 1.40usec with positional + + # Timings taken Feb 21 2018 prior to integration of #755 + # python -m perf timeit -s 'import gevent' 'gevent.Greenlet()' + # 3.6.4 : Mean +- std dev: 1.08 us +- 0.05 us + # 2.7.14 : Mean +- std dev: 1.44 us +- 0.06 us + # PyPy2 5.10.0: Mean +- std dev: 2.14 ns +- 0.08 ns + + # After the integration of spawning_stack, spawning_greenlet, + # and spawn_tree_locals on that same date: + # 3.6.4 : Mean +- std dev: 8.92 us +- 0.36 us -> 8.2x + # 2.7.14 : Mean +- std dev: 14.8 us +- 0.5 us -> 10.2x + # PyPy2 5.10.0: Mean +- std dev: 3.24 us +- 0.17 us -> 1.5x + + # Compiling with Cython gets us to these numbers: + # 3.6.4 : Mean +- std dev: 3.63 us +- 0.14 us + # 2.7.14 : Mean +- std dev: 3.37 us +- 0.20 us + # PyPy2 5.10.0 : Mean +- std dev: 4.44 us +- 0.28 us + + + _greenlet__init__(self, None, get_hub()) + + if run is not None: + self._run = run + + # If they didn't pass a callable at all, then they must + # already have one. Note that subclassing to override the run() method + # itself has never been documented or supported. + if not callable(self._run): + raise TypeError("The run argument or self._run must be callable") + + self.args = args + self.kwargs = kwargs + self.value = None + + #: An event, such as a timer or a callback that fires. It is established in + #: start() and start_later() as those two objects, respectively. + #: Once this becomes non-None, the Greenlet cannot be started again. Conversely, + #: kill() and throw() check for non-None to determine if this object has ever been + #: scheduled for starting. A placeholder _dummy_event is assigned by them to prevent + #: the greenlet from being started in the future, if necessary. + self._start_event = None + + self._notifier = None + self._formatted_info = None + self._links = [] + self._ident = None + + # Initial state: None. + # Completed successfully: (None, None, None) + # Failed with exception: (t, v, dump_traceback(tb))) + self._exc_info = None + + if GEVENT_CONFIG.track_greenlet_tree: + spawner = getcurrent() # pylint:disable=undefined-variable + self.spawning_greenlet = wref(spawner) + try: + self.spawn_tree_locals = spawner.spawn_tree_locals + except AttributeError: + self.spawn_tree_locals = {} + if spawner.parent is not None: + # The main greenlet has no parent. + # Its children get separate locals. + spawner.spawn_tree_locals = self.spawn_tree_locals + + self._spawning_stack_frames = _extract_stack(self.spawning_stack_limit) + self._spawning_stack_frames.extend(getattr(spawner, '_spawning_stack_frames', [])) + else: + # None is the default for all of these in Cython, but we + # need to declare them for pure-Python mode. + self.spawning_greenlet = None + self.spawn_tree_locals = None + self._spawning_stack_frames = None + + @Lazy + def spawning_stack(self): + # Store this in the __dict__. We don't use it from the C + # code. It's tempting to discard _spawning_stack_frames + # after this, but child greenlets may still be created + # that need it. + return _Frame_from_list(self._spawning_stack_frames or []) + + def _get_minimal_ident(self): + reg = self.parent.ident_registry + return reg.get_ident(self) + + @property + def minimal_ident(self): + """ + A small, unique integer that identifies this object. + + This is similar to :attr:`threading.Thread.ident` (and `id`) + in that as long as this object is alive, no other greenlet *in + this hub* will have the same id, but it makes a stronger + guarantee that the assigned values will be small and + sequential. Sometime after this object has died, the value + will be available for reuse. + + To get ids that are unique across all hubs, combine this with + the hub's ``minimal_ident``. + + .. versionadded:: 1.3a2 + """ + if self._ident is None: + self._ident = self._get_minimal_ident() + return self._ident + + @readproperty + def name(self): + """ + The greenlet name. By default, a unique name is constructed using + the :attr:`minimal_ident`. You can assign a string to this + value to change it. It is shown in the `repr` of this object if it + has been assigned to or if the `minimal_ident` has already been generated. + + .. versionadded:: 1.3a2 + .. versionchanged:: 1.4 + Stop showing generated names in the `repr` when the ``minimal_ident`` + hasn't been requested. This reduces overhead and may be less confusing, + since ``minimal_ident`` can get reused. + """ + return 'Greenlet-%d' % (self.minimal_ident,) + + def _raise_exception(self): + reraise(*self.exc_info) + + @property + def loop(self): + # needed by killall + return self.parent.loop + + def __nonzero__(self): + return self._start_event is not None and self._exc_info is None + try: + __bool__ = __nonzero__ # Python 3 + except NameError: # pragma: no cover + # When we're compiled with Cython, the __nonzero__ function + # goes directly into the slot and can't be accessed by name. + pass + + ### Lifecycle + + if _PYPY: + # oops - pypy's .dead relies on __nonzero__ which we overriden above + @property + def dead(self): + "Boolean indicating that the greenlet is dead and will not run again." + if self._greenlet__main: + return False + if self.__start_cancelled_by_kill() or self.__started_but_aborted(): + return True + + return self._greenlet__started and not _continulet.is_pending(self) + else: + @property + def dead(self): + "Boolean indicating that the greenlet is dead and will not run again." + return self.__start_cancelled_by_kill() or self.__started_but_aborted() or greenlet.dead.__get__(self) + + def __never_started_or_killed(self): + return self._start_event is None + + def __start_pending(self): + return (self._start_event is not None + and (self._start_event.pending or getattr(self._start_event, 'active', False))) + + def __start_cancelled_by_kill(self): + return self._start_event is _cancelled_start_event + + def __start_completed(self): + return self._start_event is _start_completed_event + + def __started_but_aborted(self): + return (not self.__never_started_or_killed() # we have been started or killed + and not self.__start_cancelled_by_kill() # we weren't killed, so we must have been started + and not self.__start_completed() # the start never completed + and not self.__start_pending()) # and we're not pending, so we must have been aborted + + def __cancel_start(self): + if self._start_event is None: + # prevent self from ever being started in the future + self._start_event = _cancelled_start_event + # cancel any pending start event + # NOTE: If this was a real pending start event, this will leave a + # "dangling" callback/timer object in the hub.loop.callbacks list; + # depending on where we are in the event loop, it may even be in a local + # variable copy of that list (in _run_callbacks). This isn't a problem, + # except for the leak-tests. + self._start_event.stop() + self._start_event.close() + + def __handle_death_before_start(self, args): + # args is (t, v, tb) or simply t or v + if self._exc_info is None and self.dead: + # the greenlet was never switched to before and it will never be, _report_error was not called + # the result was not set and the links weren't notified. let's do it here. + # checking that self.dead is true is essential, because throw() does not necessarily kill the greenlet + # (if the exception raised by throw() is caught somewhere inside the greenlet). + if len(args) == 1: + arg = args[0] + #if isinstance(arg, type): + if type(arg) is type(Exception): + args = (arg, arg(), None) + else: + args = (type(arg), arg, None) + elif not args: + args = (GreenletExit, GreenletExit(), None) + self._report_error(args) + + @property + def started(self): + # DEPRECATED + return bool(self) + + def ready(self): + """ + Return a true value if and only if the greenlet has finished + execution. + + .. versionchanged:: 1.1 + This function is only guaranteed to return true or false *values*, not + necessarily the literal constants ``True`` or ``False``. + """ + return self.dead or self._exc_info is not None + + def successful(self): + """ + Return a true value if and only if the greenlet has finished execution + successfully, that is, without raising an error. + + .. tip:: A greenlet that has been killed with the default + :class:`GreenletExit` exception is considered successful. + That is, ``GreenletExit`` is not considered an error. + + .. note:: This function is only guaranteed to return true or false *values*, + not necessarily the literal constants ``True`` or ``False``. + """ + return self._exc_info is not None and self._exc_info[1] is None + + def __repr__(self): + classname = self.__class__.__name__ + # If no name has been assigned, don't generate one, including a minimal_ident, + # if not necessary. This reduces the use of weak references and associated + # overhead. + if 'name' not in self.__dict__ and self._ident is None: + name = ' ' + else: + name = ' "%s" ' % (self.name,) + result = '<%s%sat %s' % (classname, name, hex(id(self))) + formatted = self._formatinfo() + if formatted: + result += ': ' + formatted + return result + '>' + + + def _formatinfo(self): + info = self._formatted_info + if info is not None: + return info + + # Are we running an arbitrary function provided to the constructor, + # or did a subclass override _run? + func = self._run + im_self = getattr(func, '__self__', None) + if im_self is self: + funcname = '_run' + elif im_self is not None: + funcname = repr(func) + else: + funcname = getattr(func, '__name__', '') or repr(func) + + result = funcname + args = [] + if self.args: + args = [repr(x)[:50] for x in self.args] + if self.kwargs: + args.extend(['%s=%s' % (key, repr(value)[:50]) for (key, value) in self.kwargs.items()]) + if args: + result += '(' + ', '.join(args) + ')' + # it is important to save the result here, because once the greenlet exits '_run' attribute will be removed + self._formatted_info = result + return result + + @property + def exception(self): + """ + Holds the exception instance raised by the function if the + greenlet has finished with an error. Otherwise ``None``. + """ + return self._exc_info[1] if self._exc_info is not None else None + + @property + def exc_info(self): + """ + Holds the exc_info three-tuple raised by the function if the + greenlet finished with an error. Otherwise a false value. + + .. note:: This is a provisional API and may change. + + .. versionadded:: 1.1 + """ + ei = self._exc_info + if ei is not None and ei[0] is not None: + return (ei[0], ei[1], load_traceback(ei[2])) + + def throw(self, *args): + """Immediately switch into the greenlet and raise an exception in it. + + Should only be called from the HUB, otherwise the current greenlet is left unscheduled forever. + To raise an exception in a safe manner from any greenlet, use :meth:`kill`. + + If a greenlet was started but never switched to yet, then also + a) cancel the event that will start it + b) fire the notifications as if an exception was raised in a greenlet + """ + self.__cancel_start() + + try: + if not self.dead: + # Prevent switching into a greenlet *at all* if we had never + # started it. Usually this is the same thing that happens by throwing, + # but if this is done from the hub with nothing else running, prevents a + # LoopExit. + greenlet.throw(self, *args) + finally: + self.__handle_death_before_start(args) + + def start(self): + """Schedule the greenlet to run in this loop iteration""" + if self._start_event is None: + _call_spawn_callbacks(self) + self._start_event = self.parent.loop.run_callback(self.switch) + + def start_later(self, seconds): + """ + start_later(seconds) -> None + + Schedule the greenlet to run in the future loop iteration + *seconds* later + """ + if self._start_event is None: + _call_spawn_callbacks(self) + self._start_event = self.parent.loop.timer(seconds) + self._start_event.start(self.switch) + + @staticmethod + def add_spawn_callback(callback): + """ + add_spawn_callback(callback) -> None + + Set up a *callback* to be invoked when :class:`Greenlet` objects + are started. + + The invocation order of spawn callbacks is unspecified. Adding the + same callback more than one time will not cause it to be called more + than once. + + .. versionadded:: 1.4.0 + """ + global _spawn_callbacks + if _spawn_callbacks is None: # pylint:disable=used-before-assignment + _spawn_callbacks = set() + _spawn_callbacks.add(callback) + + @staticmethod + def remove_spawn_callback(callback): + """ + remove_spawn_callback(callback) -> None + + Remove *callback* function added with :meth:`Greenlet.add_spawn_callback`. + This function will not fail if *callback* has been already removed or + if *callback* was never added. + + .. versionadded:: 1.4.0 + """ + global _spawn_callbacks + if _spawn_callbacks is not None: + _spawn_callbacks.discard(callback) + if not _spawn_callbacks: + _spawn_callbacks = None + + @classmethod + def spawn(cls, *args, **kwargs): + """ + spawn(function, *args, **kwargs) -> Greenlet + + Create a new :class:`Greenlet` object and schedule it to run ``function(*args, **kwargs)``. + This can be used as ``gevent.spawn`` or ``Greenlet.spawn``. + + The arguments are passed to :meth:`Greenlet.__init__`. + + .. versionchanged:: 1.1b1 + If a *function* is given that is not callable, immediately raise a :exc:`TypeError` + instead of spawning a greenlet that will raise an uncaught TypeError. + """ + g = cls(*args, **kwargs) + g.start() + return g + + @classmethod + def spawn_later(cls, seconds, *args, **kwargs): + """ + spawn_later(seconds, function, *args, **kwargs) -> Greenlet + + Create and return a new `Greenlet` object scheduled to run ``function(*args, **kwargs)`` + in a future loop iteration *seconds* later. This can be used as ``Greenlet.spawn_later`` + or ``gevent.spawn_later``. + + The arguments are passed to :meth:`Greenlet.__init__`. + + .. versionchanged:: 1.1b1 + If an argument that's meant to be a function (the first argument in *args*, or the ``run`` keyword ) + is given to this classmethod (and not a classmethod of a subclass), + it is verified to be callable. Previously, the spawned greenlet would have failed + when it started running. + """ + if cls is Greenlet and not args and 'run' not in kwargs: + raise TypeError("") + g = cls(*args, **kwargs) + g.start_later(seconds) + return g + + def kill(self, exception=GreenletExit, block=True, timeout=None): + """ + Raise the ``exception`` in the greenlet. + + If ``block`` is ``True`` (the default), wait until the greenlet dies or the optional timeout expires. + If block is ``False``, the current greenlet is not unscheduled. + + The function always returns ``None`` and never raises an error. + + .. note:: + + Depending on what this greenlet is executing and the state + of the event loop, the exception may or may not be raised + immediately when this greenlet resumes execution. It may + be raised on a subsequent green call, or, if this greenlet + exits before making such a call, it may not be raised at + all. As of 1.1, an example where the exception is raised + later is if this greenlet had called :func:`sleep(0) + `; an example where the exception is raised + immediately is if this greenlet had called + :func:`sleep(0.1) `. + + .. caution:: + + Use care when killing greenlets. If the code executing is not + exception safe (e.g., makes proper use of ``finally``) then an + unexpected exception could result in corrupted state. + + See also :func:`gevent.kill`. + + :keyword type exception: The type of exception to raise in the greenlet. The default + is :class:`GreenletExit`, which indicates a :meth:`successful` completion + of the greenlet. + + .. versionchanged:: 0.13.0 + *block* is now ``True`` by default. + .. versionchanged:: 1.1a2 + If this greenlet had never been switched to, killing it will prevent it from ever being switched to. + """ + self.__cancel_start() + + if self.dead: + self.__handle_death_before_start((exception,)) + else: + waiter = Waiter() if block else None # pylint:disable=undefined-variable + self.parent.loop.run_callback(_kill, self, exception, waiter) + if block: + waiter.get() + self.join(timeout) + # it should be OK to use kill() in finally or kill a greenlet from more than one place; + # thus it should not raise when the greenlet is already killed (= not started) + + def get(self, block=True, timeout=None): + """ + get(block=True, timeout=None) -> object + + Return the result the greenlet has returned or re-raise the + exception it has raised. + + If block is ``False``, raise :class:`gevent.Timeout` if the + greenlet is still alive. If block is ``True``, unschedule the + current greenlet until the result is available or the timeout + expires. In the latter case, :class:`gevent.Timeout` is + raised. + """ + if self.ready(): + if self.successful(): + return self.value + self._raise_exception() + if not block: + raise Timeout() + + switch = getcurrent().switch # pylint:disable=undefined-variable + self.rawlink(switch) + try: + t = Timeout._start_new_or_dummy(timeout) + try: + result = self.parent.switch() + if result is not self: + raise InvalidSwitchError('Invalid switch into Greenlet.get(): %r' % (result, )) + finally: + t.cancel() + except: + # unlinking in 'except' instead of finally is an optimization: + # if switch occurred normally then link was already removed in _notify_links + # and there's no need to touch the links set. + # Note, however, that if "Invalid switch" assert was removed and invalid switch + # did happen, the link would remain, causing another invalid switch later in this greenlet. + self.unlink(switch) + raise + + if self.ready(): + if self.successful(): + return self.value + self._raise_exception() + + def join(self, timeout=None): + """ + join(timeout=None) -> None + + Wait until the greenlet finishes or *timeout* expires. Return + ``None`` regardless. + """ + if self.ready(): + return + + switch = getcurrent().switch # pylint:disable=undefined-variable + self.rawlink(switch) + try: + t = Timeout._start_new_or_dummy(timeout) + try: + result = self.parent.switch() + if result is not self: + raise InvalidSwitchError('Invalid switch into Greenlet.join(): %r' % (result, )) + finally: + t.cancel() + except Timeout as ex: + self.unlink(switch) + if ex is not t: + raise + except: + self.unlink(switch) + raise + + def _report_result(self, result): + self._exc_info = (None, None, None) + self.value = result + if self._links and not self._notifier: + self._notifier = self.parent.loop.run_callback(self._notify_links) + + def _report_error(self, exc_info): + if isinstance(exc_info[1], GreenletExit): + self._report_result(exc_info[1]) + return + + self._exc_info = exc_info[0], exc_info[1], dump_traceback(exc_info[2]) + + if self._links and not self._notifier: + self._notifier = self.parent.loop.run_callback(self._notify_links) + + try: + self.parent.handle_error(self, *exc_info) + finally: + del exc_info + + def run(self): + try: + self.__cancel_start() + self._start_event = _start_completed_event + + try: + result = self._run(*self.args, **self.kwargs) + except: # pylint:disable=bare-except + self._report_error(sys_exc_info()) + return + self._report_result(result) + finally: + self.__dict__.pop('_run', None) + self.args = () + self.kwargs.clear() + + def _run(self): + """ + Subclasses may override this method to take any number of + arguments and keyword arguments. + + .. versionadded:: 1.1a3 + Previously, if no callable object was + passed to the constructor, the spawned greenlet would later + fail with an AttributeError. + """ + # We usually override this in __init__ + # pylint: disable=method-hidden + return + + def has_links(self): + return len(self._links) + + def rawlink(self, callback): + """ + Register a callable to be executed when the greenlet finishes + execution. + + The *callback* will be called with this instance as an + argument. + + .. caution:: The callable will be called in the HUB greenlet. + """ + if not callable(callback): + raise TypeError('Expected callable: %r' % (callback, )) + self._links.append(callback) # pylint:disable=no-member + if self.ready() and self._links and not self._notifier: + self._notifier = self.parent.loop.run_callback(self._notify_links) + + def link(self, callback, SpawnedLink=SpawnedLink): + """ + Link greenlet's completion to a callable. + + The *callback* will be called with this instance as an + argument once this greenlet is dead. A callable is called in + its own :class:`greenlet.greenlet` (*not* a + :class:`Greenlet`). + """ + # XXX: Is the redefinition of SpawnedLink supposed to just be an + # optimization, or do people use it? It's not documented + # pylint:disable=redefined-outer-name + self.rawlink(SpawnedLink(callback)) + + def unlink(self, callback): + """Remove the callback set by :meth:`link` or :meth:`rawlink`""" + try: + self._links.remove(callback) # pylint:disable=no-member + except ValueError: + pass + + def unlink_all(self): + """ + Remove all the callbacks. + + .. versionadded:: 1.3a2 + """ + del self._links[:] + + def link_value(self, callback, SpawnedLink=SuccessSpawnedLink): + """ + Like :meth:`link` but *callback* is only notified when the greenlet + has completed successfully. + """ + # pylint:disable=redefined-outer-name + self.link(callback, SpawnedLink=SpawnedLink) + + def link_exception(self, callback, SpawnedLink=FailureSpawnedLink): + """ + Like :meth:`link` but *callback* is only notified when the + greenlet dies because of an unhandled exception. + """ + # pylint:disable=redefined-outer-name + self.link(callback, SpawnedLink=SpawnedLink) + + def _notify_links(self): + while self._links: + # Early links are allowed to remove later links + # before we get to them, and they're also allowed to + # add new links, so we have to be careful about iterating. + + # We don't expect this list to be very large, so the time spent + # manipulating it should be small. a deque is probably not justified. + # Cython has optimizations to transform this into a memmove anyway. + link = self._links.pop(0) + try: + link(self) + except: # pylint:disable=bare-except + self.parent.handle_error((link, self), *sys_exc_info()) + + +class _dummy_event(object): + __slots__ = ('pending', 'active') + + def __init__(self): + self.pending = self.active = False + + def stop(self): + pass + + def start(self, cb): # pylint:disable=unused-argument + raise AssertionError("Cannot start the dummy event") + + def close(self): + pass + +_cancelled_start_event = _dummy_event() +_start_completed_event = _dummy_event() + + +def _kill(glet, exception, waiter): + try: + glet.throw(exception) + except: # pylint:disable=bare-except + # XXX do we need this here? + glet.parent.handle_error(glet, *sys_exc_info()) + if waiter is not None: + waiter.switch(None) + + +def joinall(greenlets, timeout=None, raise_error=False, count=None): + """ + Wait for the ``greenlets`` to finish. + + :param greenlets: A sequence (supporting :func:`len`) of greenlets to wait for. + :keyword float timeout: If given, the maximum number of seconds to wait. + :return: A sequence of the greenlets that finished before the timeout (if any) + expired. + """ + if not raise_error: + return wait(greenlets, timeout=timeout, count=count) + + done = [] + for obj in iwait(greenlets, timeout=timeout, count=count): + if getattr(obj, 'exception', None) is not None: + if hasattr(obj, '_raise_exception'): + obj._raise_exception() + else: + raise obj.exception + done.append(obj) + return done + + +def _killall3(greenlets, exception, waiter): + diehards = [] + for g in greenlets: + if not g.dead: + try: + g.throw(exception) + except: # pylint:disable=bare-except + g.parent.handle_error(g, *sys_exc_info()) + if not g.dead: + diehards.append(g) + waiter.switch(diehards) + + +def _killall(greenlets, exception): + for g in greenlets: + if not g.dead: + try: + g.throw(exception) + except: # pylint:disable=bare-except + g.parent.handle_error(g, *sys_exc_info()) + + +def _call_spawn_callbacks(gr): + if _spawn_callbacks is not None: + for cb in _spawn_callbacks: + cb(gr) + + +_spawn_callbacks = None + + +def killall(greenlets, exception=GreenletExit, block=True, timeout=None): + """ + Forceably terminate all the ``greenlets`` by causing them to raise ``exception``. + + .. caution:: Use care when killing greenlets. If they are not prepared for exceptions, + this could result in corrupted state. + + :param greenlets: A **bounded** iterable of the non-None greenlets to terminate. + *All* the items in this iterable must be greenlets that belong to the same thread. + :keyword exception: The exception to raise in the greenlets. By default this is + :class:`GreenletExit`. + :keyword bool block: If True (the default) then this function only returns when all the + greenlets are dead; the current greenlet is unscheduled during that process. + If greenlets ignore the initial exception raised in them, + then they will be joined (with :func:`gevent.joinall`) and allowed to die naturally. + If False, this function returns immediately and greenlets will raise + the exception asynchronously. + :keyword float timeout: A time in seconds to wait for greenlets to die. If given, it is + only honored when ``block`` is True. + :raise Timeout: If blocking and a timeout is given that elapses before + all the greenlets are dead. + + .. versionchanged:: 1.1a2 + *greenlets* can be any iterable of greenlets, like an iterator or a set. + Previously it had to be a list or tuple. + """ + # support non-indexable containers like iterators or set objects + greenlets = list(greenlets) + if not greenlets: + return + loop = greenlets[0].loop + if block: + waiter = Waiter() # pylint:disable=undefined-variable + loop.run_callback(_killall3, greenlets, exception, waiter) + t = Timeout._start_new_or_dummy(timeout) + try: + alive = waiter.get() + if alive: + joinall(alive, raise_error=False) + finally: + t.cancel() + else: + loop.run_callback(_killall, greenlets, exception) + +def _init(): + greenlet_init() # pylint:disable=undefined-variable + +_init() + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent._greenlet') diff --git a/src/gevent/hub.py b/src/gevent/hub.py new file mode 100644 index 0000000..2e26fa3 --- /dev/null +++ b/src/gevent/hub.py @@ -0,0 +1,750 @@ +# Copyright (c) 2009-2015 Denis Bilenko. See LICENSE for details. +""" +Event-loop hub. +""" +from __future__ import absolute_import, print_function +# XXX: FIXME: Refactor to make this smaller +# pylint:disable=too-many-lines +from functools import partial as _functools_partial + +import sys +import traceback + + +from greenlet import greenlet as RawGreenlet +from greenlet import getcurrent +from greenlet import GreenletExit + + + +__all__ = [ + 'getcurrent', + 'GreenletExit', + 'spawn_raw', + 'sleep', + 'kill', + 'signal', + 'reinit', + 'get_hub', + 'Hub', + 'Waiter', +] + +from gevent._config import config as GEVENT_CONFIG +from gevent._compat import thread_mod_name +from gevent._util import readproperty +from gevent._util import Lazy +from gevent._util import gmctime +from gevent._ident import IdentRegistry + +from gevent._hub_local import get_hub +from gevent._hub_local import get_loop +from gevent._hub_local import set_hub +from gevent._hub_local import set_loop +from gevent._hub_local import get_hub_if_exists as _get_hub +from gevent._hub_local import get_hub_noargs as _get_hub_noargs +from gevent._hub_local import set_default_hub_class + +from gevent._greenlet_primitives import TrackedRawGreenlet +from gevent._hub_primitives import WaitOperationsGreenlet + +# Export +from gevent import _hub_primitives +wait = _hub_primitives.wait_on_objects +iwait = _hub_primitives.iwait_on_objects + + +from gevent.exceptions import LoopExit + +from gevent._waiter import Waiter + +# Need the real get_ident. We're imported early enough (by gevent/__init__.py) +# that we can be sure nothing is monkey patched yet. +get_thread_ident = __import__(thread_mod_name).get_ident +MAIN_THREAD_IDENT = get_thread_ident() # XXX: Assuming import is done on the main thread. + + +def spawn_raw(function, *args, **kwargs): + """ + Create a new :class:`greenlet.greenlet` object and schedule it to + run ``function(*args, **kwargs)``. + + This returns a raw :class:`~greenlet.greenlet` which does not have all the useful + methods that :class:`gevent.Greenlet` has. Typically, applications + should prefer :func:`~gevent.spawn`, but this method may + occasionally be useful as an optimization if there are many + greenlets involved. + + .. versionchanged:: 1.1a3 + Verify that ``function`` is callable, raising a TypeError if not. Previously, + the spawned greenlet would have failed the first time it was switched to. + + .. versionchanged:: 1.1b1 + If *function* is not callable, immediately raise a :exc:`TypeError` + instead of spawning a greenlet that will raise an uncaught TypeError. + + .. versionchanged:: 1.1rc2 + Accept keyword arguments for ``function`` as previously (incorrectly) + documented. Note that this may incur an additional expense. + + .. versionchanged:: 1.3a2 + Populate the ``spawning_greenlet`` and ``spawn_tree_locals`` + attributes of the returned greenlet. + + .. versionchanged:: 1.3b1 + *Only* populate ``spawning_greenlet`` and ``spawn_tree_locals`` + if ``GEVENT_TRACK_GREENLET_TREE`` is enabled (the default). If not enabled, + those attributes will not be set. + + """ + if not callable(function): + raise TypeError("function must be callable") + + # The hub is always the parent. + hub = _get_hub_noargs() + + factory = TrackedRawGreenlet if GEVENT_CONFIG.track_greenlet_tree else RawGreenlet + + # The callback class object that we use to run this doesn't + # accept kwargs (and those objects are heavily used, as well as being + # implemented twice in core.ppyx and corecffi.py) so do it with a partial + if kwargs: + function = _functools_partial(function, *args, **kwargs) + g = factory(function, hub) + hub.loop.run_callback(g.switch) + else: + g = factory(function, hub) + hub.loop.run_callback(g.switch, *args) + + return g + + +def sleep(seconds=0, ref=True): + """ + Put the current greenlet to sleep for at least *seconds*. + + *seconds* may be specified as an integer, or a float if fractional + seconds are desired. + + .. tip:: In the current implementation, a value of 0 (the default) + means to yield execution to any other runnable greenlets, but + this greenlet may be scheduled again before the event loop + cycles (in an extreme case, a greenlet that repeatedly sleeps + with 0 can prevent greenlets that are ready to do I/O from + being scheduled for some (small) period of time); a value greater than + 0, on the other hand, will delay running this greenlet until + the next iteration of the loop. + + If *ref* is False, the greenlet running ``sleep()`` will not prevent :func:`gevent.wait` + from exiting. + + .. versionchanged:: 1.3a1 + Sleeping with a value of 0 will now be bounded to approximately block the + loop for no longer than :func:`gevent.getswitchinterval`. + + .. seealso:: :func:`idle` + """ + hub = _get_hub_noargs() + loop = hub.loop + if seconds <= 0: + waiter = Waiter(hub) + loop.run_callback(waiter.switch, None) + waiter.get() + else: + with loop.timer(seconds, ref=ref) as t: + # Sleeping is expected to be an "absolute" measure with + # respect to time.time(), not a relative measure, so it's + # important to update the loop's notion of now before we start + loop.update_now() + hub.wait(t) + + +def idle(priority=0): + """ + Cause the calling greenlet to wait until the event loop is idle. + + Idle is defined as having no other events of the same or higher + *priority* pending. That is, as long as sockets, timeouts or even + signals of the same or higher priority are being processed, the loop + is not idle. + + .. seealso:: :func:`sleep` + """ + hub = _get_hub_noargs() + watcher = hub.loop.idle() + if priority: + watcher.priority = priority + hub.wait(watcher) + + +def kill(greenlet, exception=GreenletExit): + """ + Kill greenlet asynchronously. The current greenlet is not unscheduled. + + .. note:: + + The method :meth:`Greenlet.kill` method does the same and + more (and the same caveats listed there apply here). However, the MAIN + greenlet - the one that exists initially - does not have a + ``kill()`` method, and neither do any created with :func:`spawn_raw`, + so you have to use this function. + + .. caution:: Use care when killing greenlets. If they are not prepared for + exceptions, this could result in corrupted state. + + .. versionchanged:: 1.1a2 + If the ``greenlet`` has a :meth:`kill ` method, calls it. This prevents a + greenlet from being switched to for the first time after it's been + killed but not yet executed. + """ + if not greenlet.dead: + if hasattr(greenlet, 'kill'): + # dealing with gevent.greenlet.Greenlet. Use it, especially + # to avoid allowing one to be switched to for the first time + # after it's been killed + greenlet.kill(exception=exception, block=False) + else: + _get_hub_noargs().loop.run_callback(greenlet.throw, exception) + + +class signal(object): + """ + Call the *handler* with the *args* and *kwargs* when the process + receives the signal *signalnum*. + + The *handler* will be run in a new greenlet when the signal is delivered. + + This returns an object with the useful method ``cancel``, which, when called, + will prevent future deliveries of *signalnum* from calling *handler*. + + .. note:: + + This may not operate correctly with SIGCHLD if libev child watchers + are used (as they are by default with os.fork). + + .. versionchanged:: 1.2a1 + The ``handler`` argument is required to be callable at construction time. + """ + + # XXX: This is manually documented in gevent.rst while it is aliased in + # the gevent module. + + greenlet_class = None + + def __init__(self, signalnum, handler, *args, **kwargs): + if not callable(handler): + raise TypeError("signal handler must be callable.") + + self.hub = _get_hub_noargs() + self.watcher = self.hub.loop.signal(signalnum, ref=False) + self.watcher.start(self._start) + self.handler = handler + self.args = args + self.kwargs = kwargs + if self.greenlet_class is None: + from gevent import Greenlet + self.greenlet_class = Greenlet + + def _get_ref(self): + return self.watcher.ref + + def _set_ref(self, value): + self.watcher.ref = value + + ref = property(_get_ref, _set_ref) + del _get_ref, _set_ref + + def cancel(self): + self.watcher.stop() + + def _start(self): + try: + greenlet = self.greenlet_class(self.handle) + greenlet.switch() + except: # pylint:disable=bare-except + self.hub.handle_error(None, *sys._exc_info()) # pylint:disable=no-member + + def handle(self): + try: + self.handler(*self.args, **self.kwargs) + except: # pylint:disable=bare-except + self.hub.handle_error(None, *sys.exc_info()) + + +def reinit(hub=None): + """ + reinit() -> None + + Prepare the gevent hub to run in a new (forked) process. + + This should be called *immediately* after :func:`os.fork` in the + child process. This is done automatically by + :func:`gevent.os.fork` or if the :mod:`os` module has been + monkey-patched. If this function is not called in a forked + process, symptoms may include hanging of functions like + :func:`socket.getaddrinfo`, and the hub's threadpool is unlikely + to work. + + .. note:: Registered fork watchers may or may not run before + this function (and thus ``gevent.os.fork``) return. If they have + not run, they will run "soon", after an iteration of the event loop. + You can force this by inserting a few small (but non-zero) calls to :func:`sleep` + after fork returns. (As of gevent 1.1 and before, fork watchers will + not have run, but this may change in the future.) + + .. note:: This function may be removed in a future major release + if the fork process can be more smoothly managed. + + .. warning:: See remarks in :func:`gevent.os.fork` about greenlets + and event loop watchers in the child process. + """ + # Note the signature line in the docstring: hub is not a public param. + + # The loop reinit function in turn calls libev's ev_loop_fork + # function. + hub = _get_hub() if hub is None else hub + if hub is None: + return + + # Note that we reinit the existing loop, not destroy it. + # See https://github.com/gevent/gevent/issues/200. + hub.loop.reinit() + # libev's fork watchers are slow to fire because the only fire + # at the beginning of a loop; due to our use of callbacks that + # run at the end of the loop, that may be too late. The + # threadpool and resolvers depend on the fork handlers being + # run (specifically, the threadpool will fail in the forked + # child if there were any threads in it, which there will be + # if the resolver_thread was in use (the default) before the + # fork.) + # + # If the forked process wants to use the threadpool or + # resolver immediately (in a queued callback), it would hang. + # + # The below is a workaround. Fortunately, all of these + # methods are idempotent and can be called multiple times + # following a fork if the suddenly started working, or were + # already working on some platforms. Other threadpools and fork handlers + # will be called at an arbitrary time later ('soon') + for obj in (hub._threadpool, hub._resolver, hub.periodic_monitoring_thread): + getattr(obj, '_on_fork', lambda: None)() + + # TODO: We'd like to sleep for a non-zero amount of time to force the loop to make a + # pass around before returning to this greenlet. That will allow any + # user-provided fork watchers to run. (Two calls are necessary.) HOWEVER, if + # we do this, certain tests that heavily mix threads and forking, + # like 2.7/test_threading:test_reinit_tls_after_fork, fail. It's not immediately clear + # why. + #sleep(0.00001) + #sleep(0.00001) + + +class Hub(WaitOperationsGreenlet): + """ + A greenlet that runs the event loop. + + It is created automatically by :func:`get_hub`. + + .. rubric:: Switching + + Every time this greenlet (i.e., the event loop) is switched *to*, + if the current greenlet has a ``switch_out`` method, it will be + called. This allows a greenlet to take some cleanup actions before + yielding control. This method should not call any gevent blocking + functions. + """ + + #: If instances of these classes are raised into the event loop, + #: they will be propagated out to the main greenlet (where they will + #: usually be caught by Python itself) + SYSTEM_ERROR = (KeyboardInterrupt, SystemExit, SystemError) + + #: Instances of these classes are not considered to be errors and + #: do not get logged/printed when raised by the event loop. + NOT_ERROR = (GreenletExit, SystemExit) + + #: The size we use for our threadpool. Either use a subclass + #: for this, or change it immediately after creating the hub. + threadpool_size = 10 + + # An instance of PeriodicMonitoringThread, if started. + periodic_monitoring_thread = None + + # The ident of the thread we were created in, which should be the + # thread that we run in. + thread_ident = None + + #: A string giving the name of this hub. Useful for associating hubs + #: with particular threads. Printed as part of the default repr. + #: + #: .. versionadded:: 1.3b1 + name = '' + + # NOTE: We cannot define a class-level 'loop' attribute + # because that conflicts with the slot we inherit from the + # Cythonized-bases. + + # This is the source for our 'minimal_ident' property. We don't use a + # IdentRegistry because we've seen some crashes having to do with + # clearing weak references on shutdown in Windows (see known_failures.py). + # This gives us slightly different semantics than a greenlet's minimal_ident + # (notably, there can be holes) but we never documented this object's minimal_ident, + # and there should be few enough hub's over the lifetime of a process so as not + # to matter much. + _hub_counter = 0 + + def __init__(self, loop=None, default=None): + WaitOperationsGreenlet.__init__(self, None, None) + self.thread_ident = get_thread_ident() + if hasattr(loop, 'run'): + if default is not None: + raise TypeError("Unexpected argument: default") + self.loop = loop + elif get_loop() is not None: + # Reuse a loop instance previously set by + # destroying a hub without destroying the associated + # loop. See #237 and #238. + self.loop = get_loop() + else: + if default is None and self.thread_ident != MAIN_THREAD_IDENT: + default = False + + if loop is None: + loop = self.backend + self.loop = self.loop_class(flags=loop, default=default) # pylint:disable=not-callable + self._resolver = None + self._threadpool = None + self.format_context = GEVENT_CONFIG.format_context + + Hub._hub_counter += 1 + self.minimal_ident = Hub._hub_counter + + @Lazy + def ident_registry(self): + return IdentRegistry() + + @property + def loop_class(self): + return GEVENT_CONFIG.loop + + @property + def backend(self): + return GEVENT_CONFIG.libev_backend + + @property + def main_hub(self): + """ + Is this the hub for the main thread? + + .. versionadded:: 1.3b1 + """ + return self.thread_ident == MAIN_THREAD_IDENT + + def __repr__(self): + if self.loop is None: + info = 'destroyed' + else: + try: + info = self.loop._format() + except Exception as ex: # pylint:disable=broad-except + info = str(ex) or repr(ex) or 'error' + result = '<%s %r at 0x%x %s' % ( + self.__class__.__name__, + self.name, + id(self), + info) + if self._resolver is not None: + result += ' resolver=%r' % self._resolver + if self._threadpool is not None: + result += ' threadpool=%r' % self._threadpool + result += ' thread_ident=%s' % (hex(self.thread_ident), ) + return result + '>' + + def handle_error(self, context, type, value, tb): + """ + Called by the event loop when an error occurs. The arguments + type, value, and tb are the standard tuple returned by :func:`sys.exc_info`. + + Applications can set a property on the hub with this same signature + to override the error handling provided by this class. + + Errors that are :attr:`system errors ` are passed + to :meth:`handle_system_error`. + + :param context: If this is ``None``, indicates a system error that + should generally result in exiting the loop and being thrown to the + parent greenlet. + """ + if isinstance(value, str): + # Cython can raise errors where the value is a plain string + # e.g., AttributeError, "_semaphore.Semaphore has no attr", + value = type(value) + if not issubclass(type, self.NOT_ERROR): + self.print_exception(context, type, value, tb) + if context is None or issubclass(type, self.SYSTEM_ERROR): + self.handle_system_error(type, value) + + def handle_system_error(self, type, value): + """ + Called from `handle_error` when the exception type is determined + to be a :attr:`system error `. + + System errors cause the exception to be raised in the main + greenlet (the parent of this hub). + """ + current = getcurrent() + if current is self or current is self.parent or self.loop is None: + self.parent.throw(type, value) + else: + # in case system error was handled and life goes on + # switch back to this greenlet as well + cb = None + try: + cb = self.loop.run_callback(current.switch) + except: # pylint:disable=bare-except + traceback.print_exc(file=self.exception_stream) + try: + self.parent.throw(type, value) + finally: + if cb is not None: + cb.stop() + + @readproperty + def exception_stream(self): + """ + The stream to which exceptions will be written. + Defaults to ``sys.stderr`` unless assigned to. + + .. versionadded:: 1.2a1 + """ + # Unwrap any FileObjectThread we have thrown around sys.stderr + # (because it can't be used in the hub). Tricky because we are + # called in error situations when it's not safe to import. + # Be careful not to access sys if we're in the process of interpreter + # shutdown. + stderr = sys.stderr if sys else None # pylint:disable=using-constant-test + if type(stderr).__name__ == 'FileObjectThread': + stderr = stderr.io # pylint:disable=no-member + return stderr + + def print_exception(self, context, type, value, tb): + # Python 3 does not gracefully handle None value or tb in + # traceback.print_exception() as previous versions did. + # pylint:disable=no-member + errstream = self.exception_stream + if not errstream: # pragma: no cover + # If the error stream is gone, such as when the sys dict + # gets cleared during interpreter shutdown, + # don't cause follow-on errors. + # See https://github.com/gevent/gevent/issues/1295 + return + + if value is None: + errstream.write('%s\n' % type.__name__) + else: + traceback.print_exception(type, value, tb, file=errstream) + del tb + + try: + errstream.write(gmctime()) + errstream.write(' ' if context is not None else '\n') + except: # pylint:disable=bare-except + # Possible not safe to import under certain + # error conditions in Python 2 + pass + + if context is not None: + if not isinstance(context, str): + try: + context = self.format_context(context) + except: # pylint:disable=bare-except + traceback.print_exc(file=self.exception_stream) + context = repr(context) + errstream.write('%s failed with %s\n\n' % (context, getattr(type, '__name__', 'exception'), )) + + + def run(self): + """ + Entry-point to running the loop. This method is called automatically + when the hub greenlet is scheduled; do not call it directly. + + :raises gevent.exceptions.LoopExit: If the loop finishes running. This means + that there are no other scheduled greenlets, and no active + watchers or servers. In some situations, this indicates a + programming error. + """ + assert self is getcurrent(), 'Do not call Hub.run() directly' + self.start_periodic_monitoring_thread() + while 1: + loop = self.loop + loop.error_handler = self + try: + loop.run() + finally: + loop.error_handler = None # break the refcount cycle + debug = [] + if hasattr(loop, 'debug'): + debug = loop.debug() + self.parent.throw(LoopExit('This operation would block forever', self, debug)) + # this function must never return, as it will cause switch() in the parent greenlet + # to return an unexpected value + # It is still possible to kill this greenlet with throw. However, in that case + # switching to it is no longer safe, as switch will return immediately + + def start_periodic_monitoring_thread(self): + if self.periodic_monitoring_thread is None and GEVENT_CONFIG.monitor_thread: + # Note that it is possible for one real thread to + # (temporarily) wind up with multiple monitoring threads, + # if hubs are started and stopped within the thread. This shows up + # in the threadpool tests. The monitoring threads will eventually notice their + # hub object is gone. + from gevent._monitor import PeriodicMonitoringThread + from gevent.events import PeriodicMonitorThreadStartedEvent + from gevent.events import notify_and_call_entry_points + self.periodic_monitoring_thread = PeriodicMonitoringThread(self) + + if self.main_hub: + self.periodic_monitoring_thread.install_monitor_memory_usage() + + notify_and_call_entry_points(PeriodicMonitorThreadStartedEvent( + self.periodic_monitoring_thread)) + + return self.periodic_monitoring_thread + + def join(self, timeout=None): + """Wait for the event loop to finish. Exits only when there are + no more spawned greenlets, started servers, active timeouts or watchers. + + If *timeout* is provided, wait no longer for the specified number of seconds. + + Returns True if exited because the loop finished execution. + Returns False if exited because of timeout expired. + """ + assert getcurrent() is self.parent, "only possible from the MAIN greenlet" + if self.dead: + return True + + waiter = Waiter(self) + + if timeout is not None: + timeout = self.loop.timer(timeout, ref=False) + timeout.start(waiter.switch, None) + + try: + try: + waiter.get() + except LoopExit: + return True + finally: + if timeout is not None: + timeout.stop() + timeout.close() + return False + + def destroy(self, destroy_loop=None): + """ + Destroy this hub and clean up its resources. + + If you manually create hubs, you *should* call this + method before disposing of the hub object reference. + """ + if self.periodic_monitoring_thread is not None: + self.periodic_monitoring_thread.kill() + self.periodic_monitoring_thread = None + if self._resolver is not None: + self._resolver.close() + del self._resolver + if self._threadpool is not None: + self._threadpool.kill() + del self._threadpool + if destroy_loop is None: + destroy_loop = not self.loop.default + if destroy_loop: + if get_loop() is self.loop: + # Don't let anyone try to reuse this + set_loop(None) + self.loop.destroy() + else: + # Store in case another hub is created for this + # thread. + set_loop(self.loop) + + + self.loop = None + if _get_hub() is self: + set_hub(None) + + + # XXX: We can probably simplify the resolver and threadpool properties. + + @property + def resolver_class(self): + return GEVENT_CONFIG.resolver + + def _get_resolver(self): + if self._resolver is None: + self._resolver = self.resolver_class(hub=self) # pylint:disable=not-callable + return self._resolver + + def _set_resolver(self, value): + self._resolver = value + + def _del_resolver(self): + self._resolver = None + + resolver = property(_get_resolver, _set_resolver, _del_resolver, + """ + The DNS resolver that the socket functions will use. + + .. seealso:: :doc:`/dns` + """) + + + @property + def threadpool_class(self): + return GEVENT_CONFIG.threadpool + + def _get_threadpool(self): + if self._threadpool is None: + # pylint:disable=not-callable + self._threadpool = self.threadpool_class(self.threadpool_size, hub=self) + return self._threadpool + + def _set_threadpool(self, value): + self._threadpool = value + + def _del_threadpool(self): + self._threadpool = None + + threadpool = property(_get_threadpool, _set_threadpool, _del_threadpool, + """ + The threadpool associated with this hub. + + Usually this is a + :class:`gevent.threadpool.ThreadPool`, but + you :attr:`can customize that + `. + + Use this object to schedule blocking + (non-cooperative) operations in a different + thread to prevent them from halting the event loop. + """) + + +set_default_hub_class(Hub) + + + +class linkproxy(object): + __slots__ = ['callback', 'obj'] + + def __init__(self, callback, obj): + self.callback = callback + self.obj = obj + + def __call__(self, *args): + callback = self.callback + obj = self.obj + self.callback = None + self.obj = None + callback(obj) diff --git a/src/gevent/libev/__init__.py b/src/gevent/libev/__init__.py new file mode 100644 index 0000000..412d64c --- /dev/null +++ b/src/gevent/libev/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# Nothing public here +__all__ = [] diff --git a/src/gevent/libev/_corecffi_build.py b/src/gevent/libev/_corecffi_build.py new file mode 100644 index 0000000..a6025fc --- /dev/null +++ b/src/gevent/libev/_corecffi_build.py @@ -0,0 +1,75 @@ +# pylint: disable=no-member + +# This module is only used to create and compile the gevent._corecffi module; +# nothing should be directly imported from it except `ffi`, which should only be +# used for `ffi.compile()`; programs should import gevent._corecfffi. +# However, because we are using "out-of-line" mode, it is necessary to examine +# this file to know what functions are created and available on the generated +# module. +from __future__ import absolute_import, print_function +import sys +import os +import os.path # pylint:disable=no-name-in-module +import struct + +__all__ = [] + + +def system_bits(): + return struct.calcsize('P') * 8 + + +def st_nlink_type(): + if sys.platform == "darwin" or sys.platform.startswith("freebsd"): + return "short" + if system_bits() == 32: + return "unsigned long" + return "long long" + + +from cffi import FFI +ffi = FFI() + +thisdir = os.path.dirname(os.path.abspath(__file__)) +def read_source(name): + with open(os.path.join(thisdir, name), 'r') as f: + return f.read() + +_cdef = read_source('_corecffi_cdef.c') +_source = read_source('_corecffi_source.c') + +_cdef = _cdef.replace('#define GEVENT_ST_NLINK_T int', '') +_cdef = _cdef.replace('#define GEVENT_STRUCT_DONE int', '') +_cdef = _cdef.replace('GEVENT_ST_NLINK_T', st_nlink_type()) +_cdef = _cdef.replace("GEVENT_STRUCT_DONE _;", '...;') + + +if sys.platform.startswith('win'): + # We must have the vfd_open, etc, functions on + # Windows. But on other platforms, going through + # CFFI to just return the file-descriptor is slower + # than just doing it in Python, so we check for and + # workaround their absence in corecffi.py + _cdef += """ +typedef int... vfd_socket_t; +int vfd_open(vfd_socket_t); +vfd_socket_t vfd_get(int); +void vfd_free(int); +""" + + + +include_dirs = [ + thisdir, # libev_vfd.h + os.path.abspath(os.path.join(thisdir, '..', '..', '..', 'deps', 'libev')), +] +ffi.cdef(_cdef) +ffi.set_source('gevent.libev._corecffi', _source, include_dirs=include_dirs) + +if __name__ == '__main__': + # XXX: Note, on Windows, we would need to specify the external libraries + # that should be linked in, such as ws2_32 and (because libev_vfd.h makes + # Python.h calls) the proper Python library---at least for PyPy. I never got + # that to work though, and calling python functions is strongly discouraged + # from CFFI code. + ffi.compile() diff --git a/src/gevent/libev/_corecffi_cdef.c b/src/gevent/libev/_corecffi_cdef.c new file mode 100644 index 0000000..3280e99 --- /dev/null +++ b/src/gevent/libev/_corecffi_cdef.c @@ -0,0 +1,243 @@ +/* libev interface */ + +#define EV_MINPRI ... +#define EV_MAXPRI ... + +#define EV_VERSION_MAJOR ... +#define EV_VERSION_MINOR ... + +#define EV_UNDEF ... +#define EV_NONE ... +#define EV_READ ... +#define EV_WRITE ... +#define EV__IOFDSET ... +#define EV_TIMER ... +#define EV_PERIODIC ... +#define EV_SIGNAL ... +#define EV_CHILD ... +#define EV_STAT ... +#define EV_IDLE ... +#define EV_PREPARE ... +#define EV_CHECK ... +#define EV_EMBED ... +#define EV_FORK ... +#define EV_CLEANUP ... +#define EV_ASYNC ... +#define EV_CUSTOM ... +#define EV_ERROR ... + +#define EVFLAG_AUTO ... +#define EVFLAG_NOENV ... +#define EVFLAG_FORKCHECK ... +#define EVFLAG_NOINOTIFY ... +#define EVFLAG_SIGNALFD ... +#define EVFLAG_NOSIGMASK ... + +#define EVBACKEND_SELECT ... +#define EVBACKEND_POLL ... +#define EVBACKEND_EPOLL ... +#define EVBACKEND_KQUEUE ... +#define EVBACKEND_DEVPOLL ... +#define EVBACKEND_PORT ... +/* #define EVBACKEND_IOCP ... */ + +#define EVBACKEND_ALL ... +#define EVBACKEND_MASK ... + +#define EVRUN_NOWAIT ... +#define EVRUN_ONCE ... + +#define EVBREAK_CANCEL ... +#define EVBREAK_ONE ... +#define EVBREAK_ALL ... + +/* markers for the CFFI parser. Replaced when the string is read. */ +#define GEVENT_STRUCT_DONE int +#define GEVENT_ST_NLINK_T int + +struct ev_loop { + int backend_fd; + int activecnt; + GEVENT_STRUCT_DONE _; +}; + +// Watcher types +// base for all watchers +struct ev_watcher{ + void* data; + GEVENT_STRUCT_DONE _; +}; + +struct ev_io { + int fd; + int events; + void* data; + GEVENT_STRUCT_DONE _; +}; +struct ev_timer { + double at; + void* data; + GEVENT_STRUCT_DONE _; +}; +struct ev_signal { + void* data; + GEVENT_STRUCT_DONE _; +}; +struct ev_idle { + void* data; + GEVENT_STRUCT_DONE _; +}; +struct ev_prepare { + void* data; + GEVENT_STRUCT_DONE _; +}; +struct ev_check { + void* data; + GEVENT_STRUCT_DONE _; +}; +struct ev_fork { + void* data; + GEVENT_STRUCT_DONE _; +}; +struct ev_async { + void* data; + GEVENT_STRUCT_DONE _; +}; + +struct ev_child { + int pid; + int rpid; + int rstatus; + void* data; + GEVENT_STRUCT_DONE _; +}; + +struct stat { + GEVENT_ST_NLINK_T st_nlink; + GEVENT_STRUCT_DONE _; +}; + +struct ev_stat { + struct stat attr; + const char* path; + struct stat prev; + double interval; + void* data; + GEVENT_STRUCT_DONE _; +}; + +typedef double ev_tstamp; + +int ev_version_major(); +int ev_version_minor(); + +unsigned int ev_supported_backends (void); +unsigned int ev_recommended_backends (void); +unsigned int ev_embeddable_backends (void); + +ev_tstamp ev_time (void); +void ev_set_syserr_cb(void *); + +void ev_set_userdata(struct ev_loop*, void*); +void* ev_userdata(struct ev_loop*); + +int ev_priority(void*); +void ev_set_priority(void*, int); + +int ev_is_pending(void*); +int ev_is_active(void*); +void ev_io_init(struct ev_io*, void* callback, int fd, int events); +void ev_io_start(struct ev_loop*, struct ev_io*); +void ev_io_stop(struct ev_loop*, struct ev_io*); +void ev_feed_event(struct ev_loop*, void*, int); + +void ev_timer_init(struct ev_timer*, void *callback, double, double); +void ev_timer_start(struct ev_loop*, struct ev_timer*); +void ev_timer_stop(struct ev_loop*, struct ev_timer*); +void ev_timer_again(struct ev_loop*, struct ev_timer*); + +void ev_signal_init(struct ev_signal*, void* callback, int); +void ev_signal_start(struct ev_loop*, struct ev_signal*); +void ev_signal_stop(struct ev_loop*, struct ev_signal*); + +void ev_idle_init(struct ev_idle*, void* callback); +void ev_idle_start(struct ev_loop*, struct ev_idle*); +void ev_idle_stop(struct ev_loop*, struct ev_idle*); + +void ev_prepare_init(struct ev_prepare*, void* callback); +void ev_prepare_start(struct ev_loop*, struct ev_prepare*); +void ev_prepare_stop(struct ev_loop*, struct ev_prepare*); + +void ev_check_init(struct ev_check*, void* callback); +void ev_check_start(struct ev_loop*, struct ev_check*); +void ev_check_stop(struct ev_loop*, struct ev_check*); + +void ev_fork_init(struct ev_fork*, void* callback); +void ev_fork_start(struct ev_loop*, struct ev_fork*); +void ev_fork_stop(struct ev_loop*, struct ev_fork*); + +void ev_async_init(struct ev_async*, void* callback); +void ev_async_start(struct ev_loop*, struct ev_async*); +void ev_async_stop(struct ev_loop*, struct ev_async*); +void ev_async_send(struct ev_loop*, struct ev_async*); +int ev_async_pending(struct ev_async*); + +void ev_child_init(struct ev_child*, void* callback, int, int); +void ev_child_start(struct ev_loop*, struct ev_child*); +void ev_child_stop(struct ev_loop*, struct ev_child*); + +void ev_stat_init(struct ev_stat*, void* callback, char*, double); +void ev_stat_start(struct ev_loop*, struct ev_stat*); +void ev_stat_stop(struct ev_loop*, struct ev_stat*); + +struct ev_loop *ev_default_loop (unsigned int flags); +struct ev_loop* ev_loop_new(unsigned int flags); +void ev_loop_destroy(struct ev_loop*); +void ev_loop_fork(struct ev_loop*); +int ev_is_default_loop (struct ev_loop *); +unsigned int ev_iteration(struct ev_loop*); +unsigned int ev_depth(struct ev_loop*); +unsigned int ev_backend(struct ev_loop*); +void ev_verify(struct ev_loop*); +void ev_run(struct ev_loop*, int flags); + +ev_tstamp ev_now (struct ev_loop *); +void ev_now_update (struct ev_loop *); /* update event loop time */ +void ev_ref(struct ev_loop*); +void ev_unref(struct ev_loop*); +void ev_break(struct ev_loop*, int); +unsigned int ev_pending_count(struct ev_loop*); + +struct ev_loop* gevent_ev_default_loop(unsigned int flags); +void gevent_install_sigchld_handler(); +void gevent_reset_sigchld_handler(); + +void (*gevent_noop)(struct ev_loop *_loop, struct ev_timer *w, int revents); +void ev_sleep (ev_tstamp delay); /* sleep for a while */ + +/* gevent callbacks */ +/* These will be created as static functions at the end of the + * _source.c and must be declared there too. + */ +extern "Python" { + int python_callback(void* handle, int revents); + void python_handle_error(void* handle, int revents); + void python_stop(void* handle); + void python_check_callback(struct ev_loop*, void*, int); + void python_prepare_callback(struct ev_loop*, void*, int); + + // libev specific + void _syserr_cb(char*); +} +/* + * We use a single C callback for every watcher type, which in turn calls the + * Python callbacks. The ev_watcher pointer type can be used for every watcher type + * because they all start with the same members---libev itself relies on this. Each + * watcher types has a 'void* data' that stores the CFFI handle to the Python watcher + * object. + */ +static void _gevent_generic_callback(struct ev_loop* loop, struct ev_watcher* watcher, int revents); + +static void gevent_zero_check(struct ev_check* handle); +static void gevent_zero_timer(struct ev_timer* handle); +static void gevent_zero_prepare(struct ev_prepare* handle); diff --git a/src/gevent/libev/_corecffi_source.c b/src/gevent/libev/_corecffi_source.c new file mode 100644 index 0000000..63b216e --- /dev/null +++ b/src/gevent/libev/_corecffi_source.c @@ -0,0 +1,69 @@ +// passed to the real C compiler +#define LIBEV_EMBED 1 + +#ifdef _WIN32 +#define EV_STANDALONE 1 +#include "libev_vfd.h" +#endif + + +#include "libev.h" + +static void +_gevent_noop(struct ev_loop *_loop, struct ev_timer *w, int revents) { } + +void (*gevent_noop)(struct ev_loop *, struct ev_timer *, int) = &_gevent_noop; + +static int python_callback(void* handle, int revents); +static void python_handle_error(void* handle, int revents); +static void python_stop(void* handle); + +static void _gevent_generic_callback(struct ev_loop* loop, + struct ev_watcher* watcher, + int revents) +{ + void* handle = watcher->data; + int cb_result = python_callback(handle, revents); + switch(cb_result) { + case -1: + // in case of exception, call self.loop.handle_error; + // this function is also responsible for stopping the watcher + // and allowing memory to be freed + python_handle_error(handle, revents); + break; + case 1: + // Code to stop the event. Note that if python_callback + // has disposed of the last reference to the handle, + // `watcher` could now be invalid/disposed memory! + if (!ev_is_active(watcher)) { + python_stop(handle); + } + break; + case 2: + // watcher is already stopped and dead, nothing to do. + break; + default: + fprintf(stderr, + "WARNING: gevent: Unexpected return value %d from Python callback " + "for watcher %p and handle %d\n", + cb_result, + watcher, handle); + // XXX: Possible leaking of resources here? Should we be + // closing the watcher? + } +} + +static void gevent_zero_timer(struct ev_timer* handle) +{ + memset(handle, 0, sizeof(struct ev_timer)); +} + +static void gevent_zero_check(struct ev_check* handle) +{ + memset(handle, 0, sizeof(struct ev_check)); +} + +static void gevent_zero_prepare(struct ev_prepare* handle) +{ + memset(handle, 0, sizeof(struct ev_prepare)); +} diff --git a/src/gevent/libev/callbacks.c b/src/gevent/libev/callbacks.c new file mode 100644 index 0000000..751b425 --- /dev/null +++ b/src/gevent/libev/callbacks.c @@ -0,0 +1,216 @@ +/* Copyright (c) 2011-2012 Denis Bilenko. See LICENSE for details. */ +#include +#include "Python.h" +#include "ev.h" +#include "corecext.h" +#include "callbacks.h" +#ifdef Py_PYTHON_H + +#if PY_MAJOR_VERSION >= 3 + #define PyInt_FromLong PyLong_FromLong +#endif + + +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #elif defined(__GNUC__) + #define CYTHON_INLINE __inline__ + #elif defined(_MSC_VER) + #define CYTHON_INLINE __inline + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_INLINE inline + #else + #define CYTHON_INLINE + #endif +#endif + + +static CYTHON_INLINE void gevent_check_signals(struct PyGeventLoopObject* loop) { + if (!ev_is_default_loop(loop->_ptr)) { + /* only reporting signals on the default loop */ + return; + } + PyErr_CheckSignals(); + if (PyErr_Occurred()) gevent_handle_error(loop, Py_None); +} + +#define GET_OBJECT(PY_TYPE, EV_PTR, MEMBER) \ + ((struct PY_TYPE *)(((char *)EV_PTR) - offsetof(struct PY_TYPE, MEMBER))) + + +#ifdef WITH_THREAD +#define GIL_DECLARE PyGILState_STATE ___save +#define GIL_ENSURE ___save = PyGILState_Ensure(); +#define GIL_RELEASE PyGILState_Release(___save); +#else +#define GIL_DECLARE +#define GIL_ENSURE +#define GIL_RELEASE +#endif + + +static void gevent_stop(PyObject* watcher, struct PyGeventLoopObject* loop) { + PyObject *result, *method; + int error; + error = 1; + method = PyObject_GetAttrString(watcher, "stop"); + if (method) { + result = PyObject_Call(method, _empty_tuple, NULL); + if (result) { + Py_DECREF(result); + error = 0; + } + Py_DECREF(method); + } + if (error) { + gevent_handle_error(loop, watcher); + } +} + + +static void gevent_callback(struct PyGeventLoopObject* loop, PyObject* callback, PyObject* args, PyObject* watcher, void *c_watcher, int revents) { + GIL_DECLARE; + PyObject *result, *py_events; + long length; + py_events = 0; + GIL_ENSURE; + Py_INCREF(loop); + Py_INCREF(callback); + Py_INCREF(args); + Py_INCREF(watcher); + gevent_check_signals(loop); + if (args == Py_None) { + args = _empty_tuple; + } + length = PyTuple_Size(args); + if (length < 0) { + gevent_handle_error(loop, watcher); + goto end; + } + if (length > 0 && PyTuple_GET_ITEM(args, 0) == GEVENT_CORE_EVENTS) { + py_events = PyInt_FromLong(revents); + if (!py_events) { + gevent_handle_error(loop, watcher); + goto end; + } + PyTuple_SET_ITEM(args, 0, py_events); + } + else { + py_events = NULL; + } + result = PyObject_Call(callback, args, NULL); + if (result) { + Py_DECREF(result); + } + else { + gevent_handle_error(loop, watcher); + if (revents & (EV_READ|EV_WRITE)) { + /* io watcher: not stopping it may cause the failing callback to be called repeatedly */ + gevent_stop(watcher, loop); + goto end; + } + } + if (!ev_is_active(c_watcher)) { + /* Watcher was stopped, maybe by libev. Let's call stop() to clean up + * 'callback' and 'args' properties, do Py_DECREF() and ev_ref() if necessary. + * BTW, we don't need to check for EV_ERROR, because libev stops the watcher in that case. */ + gevent_stop(watcher, loop); + } +end: + if (py_events) { + Py_DECREF(py_events); + PyTuple_SET_ITEM(args, 0, GEVENT_CORE_EVENTS); + } + Py_DECREF(watcher); + Py_DECREF(args); + Py_DECREF(callback); + Py_DECREF(loop); + GIL_RELEASE; +} + + +void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb) { + /* no need for GIL here because it is only called from run_callbacks which already has GIL */ + PyObject *result, *callback, *args; + if (!loop || !cb) + return; + callback = cb->callback; + args = cb->args; + if (!callback || !args) + return; + if (callback == Py_None || args == Py_None) + return; + Py_INCREF(loop); + Py_INCREF(callback); + Py_INCREF(args); + + Py_INCREF(Py_None); + Py_DECREF(cb->callback); + cb->callback = Py_None; + + result = PyObject_Call(callback, args, NULL); + if (result) { + Py_DECREF(result); + } + else { + gevent_handle_error(loop, (PyObject*)cb); + } + + Py_INCREF(Py_None); + Py_DECREF(cb->args); + cb->args = Py_None; + + Py_DECREF(callback); + Py_DECREF(args); + Py_DECREF(loop); +} + +/* + * PyGeventWatcherObject is the first member of all the structs, so + * it is the same in all of them and they can all safely be cast to + * it. We could also use the *data member of the libev watcher objects. + */ + +#undef DEFINE_CALLBACK +#define DEFINE_CALLBACK(WATCHER_LC, WATCHER_TYPE) \ + void gevent_callback_##WATCHER_LC(struct ev_loop *_loop, void *c_watcher, int revents) { \ + struct PyGeventWatcherObject* watcher = (struct PyGeventWatcherObject*)GET_OBJECT(PyGevent##WATCHER_TYPE##Object, c_watcher, _watcher); \ + gevent_callback(watcher->loop, watcher->_callback, watcher->args, (PyObject*)watcher, c_watcher, revents); \ + } + + +DEFINE_CALLBACKS + + +void gevent_run_callbacks(struct ev_loop *_loop, void *watcher, int revents) { + struct PyGeventLoopObject* loop; + PyObject *result; + GIL_DECLARE; + GIL_ENSURE; + loop = GET_OBJECT(PyGeventLoopObject, watcher, _prepare); + Py_INCREF(loop); + gevent_check_signals(loop); + result = gevent_loop_run_callbacks(loop); + if (result) { + Py_DECREF(result); + } + else { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(loop); + GIL_RELEASE; +} + +/* This is only used on Win32 */ + +void gevent_periodic_signal_check(struct ev_loop *_loop, void *watcher, int revents) { + GIL_DECLARE; + GIL_ENSURE; + gevent_check_signals(GET_OBJECT(PyGeventLoopObject, watcher, _periodic_signal_checker)); + GIL_RELEASE; +} + + +#endif /* Py_PYTHON_H */ diff --git a/src/gevent/libev/callbacks.h b/src/gevent/libev/callbacks.h new file mode 100644 index 0000000..ed87224 --- /dev/null +++ b/src/gevent/libev/callbacks.h @@ -0,0 +1,38 @@ +struct ev_loop; +struct PyGeventLoopObject; +struct PyGeventCallbackObject; + +#define DEFINE_CALLBACK(WATCHER_LC, WATCHER_TYPE) \ + void gevent_callback_##WATCHER_LC(struct ev_loop *, void *, int); + + +#define DEFINE_CALLBACKS0 \ + DEFINE_CALLBACK(io, IO); \ + DEFINE_CALLBACK(timer, Timer); \ + DEFINE_CALLBACK(signal, Signal); \ + DEFINE_CALLBACK(idle, Idle); \ + DEFINE_CALLBACK(prepare, Prepare); \ + DEFINE_CALLBACK(check, Check); \ + DEFINE_CALLBACK(fork, Fork); \ + DEFINE_CALLBACK(async, Async); \ + DEFINE_CALLBACK(stat, Stat); \ + DEFINE_CALLBACK(child, Child); + + +#define DEFINE_CALLBACKS DEFINE_CALLBACKS0 + + +DEFINE_CALLBACKS + + +void gevent_run_callbacks(struct ev_loop *, void *, int); + + + +void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb); + +static void gevent_noop(struct ev_loop *_loop, void *watcher, int revents) { +} + +/* Only used on Win32 */ +void gevent_periodic_signal_check(struct ev_loop *, void *, int); diff --git a/src/gevent/libev/corecext.pyx b/src/gevent/libev/corecext.pyx new file mode 100644 index 0000000..c7ec4c1 --- /dev/null +++ b/src/gevent/libev/corecext.pyx @@ -0,0 +1,1352 @@ +# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details. + +# This first directive, supported in Cython 0.24+, causes sources +# files to be *much* smaller when it's false (139,027 LOC vs 35,000 +# LOC) and thus cythonpp.py (and probably the compiler; also Visual C +# has limits on source file sizes) to be faster (73s vs 46s). But it does +# make debugging more difficult. Auto-pickling was added in 0.26, and +# that's a new feature that we don't need or want to allow in a gevent +# point release. + +# cython: emit_code_comments=False, auto_pickle=False + +# NOTE: We generally cannot use the Cython IF directive as documented +# at +# http://cython.readthedocs.io/en/latest/src/userguide/language_basics.html#conditional-compilation +# (e.g., IF UNAME_SYSNAME == "Windows") because when Cython says +# "compilation", it means when *Cython* compiles, not when the C +# compiler compiles. We distribute an sdist with a single pre-compiled +# C file for all platforms so that end users that don't use a binary +# wheel don't have to sit through cythonpp and other steps the Makefile does. +# See https://github.com/gevent/gevent/issues/1076 + +cimport cython +cimport libev + +from cpython.ref cimport Py_INCREF +from cpython.ref cimport Py_DECREF +from cpython.mem cimport PyMem_Malloc +from cpython.mem cimport PyMem_Free +from libc.errno cimport errno + +cdef extern from "Python.h": + int Py_ReprEnter(object) + void Py_ReprLeave(object) + +# Work around lack of absolute_import in Cython +# Note for PY3: not doing so will leave reference to locals() on import +# (reproducible under Python 3.3, not under Python 3.4; see test__refcount_core.py) +sys = __import__('sys', level=0) +os = __import__('os', level=0) +traceback = __import__('traceback', level=0) +signalmodule = __import__('signal', level=0) +getswitchinterval = __import__('gevent', level=0).getswitchinterval + + +__all__ = ['get_version', + 'get_header_version', + 'supported_backends', + 'recommended_backends', + 'embeddable_backends', + 'time', + 'loop'] + +cdef tuple integer_types + +if sys.version_info[0] >= 3: + integer_types = int, +else: + integer_types = (int, long) + + +cdef extern from "callbacks.h": + void gevent_callback_io(libev.ev_loop, void*, int) + void gevent_callback_timer(libev.ev_loop, void*, int) + void gevent_callback_signal(libev.ev_loop, void*, int) + void gevent_callback_idle(libev.ev_loop, void*, int) + void gevent_callback_prepare(libev.ev_loop, void*, int) + void gevent_callback_check(libev.ev_loop, void*, int) + void gevent_callback_fork(libev.ev_loop, void*, int) + void gevent_callback_async(libev.ev_loop, void*, int) + void gevent_callback_child(libev.ev_loop, void*, int) + void gevent_callback_stat(libev.ev_loop, void*, int) + void gevent_run_callbacks(libev.ev_loop, void*, int) + void gevent_periodic_signal_check(libev.ev_loop, void*, int) + void gevent_call(loop, callback) + void gevent_noop(libev.ev_loop, void*, int) + +cdef extern from "stathelper.c": + object _pystat_fromstructstat(void*) + + +UNDEF = libev.EV_UNDEF +NONE = libev.EV_NONE +READ = libev.EV_READ +WRITE = libev.EV_WRITE +TIMER = libev.EV_TIMER +PERIODIC = libev.EV_PERIODIC +SIGNAL = libev.EV_SIGNAL +CHILD = libev.EV_CHILD +STAT = libev.EV_STAT +IDLE = libev.EV_IDLE +PREPARE = libev.EV_PREPARE +CHECK = libev.EV_CHECK +EMBED = libev.EV_EMBED +FORK = libev.EV_FORK +CLEANUP = libev.EV_CLEANUP +ASYNC = libev.EV_ASYNC +CUSTOM = libev.EV_CUSTOM +ERROR = libev.EV_ERROR + +READWRITE = libev.EV_READ | libev.EV_WRITE + +MINPRI = libev.EV_MINPRI +MAXPRI = libev.EV_MAXPRI + +BACKEND_PORT = libev.EVBACKEND_PORT +BACKEND_KQUEUE = libev.EVBACKEND_KQUEUE +BACKEND_EPOLL = libev.EVBACKEND_EPOLL +BACKEND_POLL = libev.EVBACKEND_POLL +BACKEND_SELECT = libev.EVBACKEND_SELECT +FORKCHECK = libev.EVFLAG_FORKCHECK +NOINOTIFY = libev.EVFLAG_NOINOTIFY +SIGNALFD = libev.EVFLAG_SIGNALFD +NOSIGMASK = libev.EVFLAG_NOSIGMASK + + +@cython.internal +cdef class _EVENTSType: + + def __repr__(self): + return 'gevent.core.EVENTS' + + +cdef public object GEVENT_CORE_EVENTS = _EVENTSType() +EVENTS = GEVENT_CORE_EVENTS + + +def get_version(): + return 'libev-%d.%02d' % (libev.ev_version_major(), libev.ev_version_minor()) + + +def get_header_version(): + return 'libev-%d.%02d' % (libev.EV_VERSION_MAJOR, libev.EV_VERSION_MINOR) + + +# This list backends in the order they are actually tried by libev +_flags = [(libev.EVBACKEND_PORT, 'port'), + (libev.EVBACKEND_KQUEUE, 'kqueue'), + (libev.EVBACKEND_EPOLL, 'epoll'), + (libev.EVBACKEND_POLL, 'poll'), + (libev.EVBACKEND_SELECT, 'select'), + (libev.EVFLAG_NOENV, 'noenv'), + (libev.EVFLAG_FORKCHECK, 'forkcheck'), + (libev.EVFLAG_NOINOTIFY, 'noinotify'), + (libev.EVFLAG_SIGNALFD, 'signalfd'), + (libev.EVFLAG_NOSIGMASK, 'nosigmask')] + + +_flags_str2int = dict((string, flag) for (flag, string) in _flags) + + +_events = [(libev.EV_READ, 'READ'), + (libev.EV_WRITE, 'WRITE'), + (libev.EV__IOFDSET, '_IOFDSET'), + (libev.EV_PERIODIC, 'PERIODIC'), + (libev.EV_SIGNAL, 'SIGNAL'), + (libev.EV_CHILD, 'CHILD'), + (libev.EV_STAT, 'STAT'), + (libev.EV_IDLE, 'IDLE'), + (libev.EV_PREPARE, 'PREPARE'), + (libev.EV_CHECK, 'CHECK'), + (libev.EV_EMBED, 'EMBED'), + (libev.EV_FORK, 'FORK'), + (libev.EV_CLEANUP, 'CLEANUP'), + (libev.EV_ASYNC, 'ASYNC'), + (libev.EV_CUSTOM, 'CUSTOM'), + (libev.EV_ERROR, 'ERROR')] + + +cpdef _flags_to_list(unsigned int flags): + cdef list result = [] + for code, value in _flags: + if flags & code: + result.append(value) + flags &= ~code + if not flags: + break + if flags: + result.append(flags) + return result + + +if sys.version_info[0] >= 3: + basestring = (bytes, str) +else: + basestring = __builtins__.basestring + + +cpdef unsigned int _flags_to_int(object flags) except? -1: + # Note, that order does not matter, libev has its own predefined order + if not flags: + return 0 + if isinstance(flags, integer_types): + return flags + cdef unsigned int result = 0 + try: + if isinstance(flags, basestring): + flags = flags.split(',') + for value in flags: + value = value.strip().lower() + if value: + result |= _flags_str2int[value] + except KeyError as ex: + raise ValueError('Invalid backend or flag: %s\nPossible values: %s' % (ex, ', '.join(sorted(_flags_str2int.keys())))) + return result + + +cdef str _str_hex(object flag): + if isinstance(flag, integer_types): + return hex(flag) + return str(flag) + + +cpdef _check_flags(unsigned int flags): + cdef list as_list + flags &= libev.EVBACKEND_MASK + if not flags: + return + if not (flags & libev.EVBACKEND_ALL): + raise ValueError('Invalid value for backend: 0x%x' % flags) + if not (flags & libev.ev_supported_backends()): + as_list = [_str_hex(x) for x in _flags_to_list(flags)] + raise ValueError('Unsupported backend: %s' % '|'.join(as_list)) + + +cpdef _events_to_str(int events): + cdef list result = [] + cdef int c_flag + for (flag, string) in _events: + c_flag = flag + if events & c_flag: + result.append(string) + events = events & (~c_flag) + if not events: + break + if events: + result.append(hex(events)) + return '|'.join(result) + + +def supported_backends(): + return _flags_to_list(libev.ev_supported_backends()) + + +def recommended_backends(): + return _flags_to_list(libev.ev_recommended_backends()) + + +def embeddable_backends(): + return _flags_to_list(libev.ev_embeddable_backends()) + + +def time(): + return libev.ev_time() + +cdef bint _check_loop(loop loop) except -1: + if not loop._ptr: + raise ValueError('operation on destroyed loop') + return 1 + + + +cdef public class callback [object PyGeventCallbackObject, type PyGeventCallback_Type]: + cdef public object callback + cdef public tuple args + cdef callback next + + def __init__(self, callback, args): + self.callback = callback + self.args = args + + def stop(self): + self.callback = None + self.args = None + + close = stop + + # Note, that __nonzero__ and pending are different + # nonzero is used in contexts where we need to know whether to schedule another callback, + # so it's true if it's pending or currently running + # 'pending' has the same meaning as libev watchers: it is cleared before entering callback + + def __nonzero__(self): + # it's nonzero if it's pending or currently executing + return self.args is not None + + @property + def pending(self): + return self.callback is not None + + def __repr__(self): + if Py_ReprEnter(self) != 0: + return "<...>" + try: + format = self._format() + result = "<%s at 0x%x%s" % (self.__class__.__name__, id(self), format) + if self.pending: + result += " pending" + if self.callback is not None: + result += " callback=%r" % (self.callback, ) + if self.args is not None: + result += " args=%r" % (self.args, ) + if self.callback is None and self.args is None: + result += " stopped" + return result + ">" + finally: + Py_ReprLeave(self) + + def _format(self): + return '' + +DEF CALLBACK_CHECK_COUNT = 50 + +@cython.final +@cython.internal +cdef class CallbackFIFO(object): + cdef callback head + cdef callback tail + + def __init__(self): + self.head = None + self.tail = None + + cdef inline callback popleft(self): + cdef callback head = self.head + self.head = head.next + if self.head is self.tail or self.head is None: + self.tail = None + head.next = None + return head + + + cdef inline append(self, callback new_tail): + assert not new_tail.next + if self.tail is None: + if self.head is None: + # Completely empty, so this + # is now our head + self.head = new_tail + return + self.tail = self.head + + + assert self.head is not None + old_tail = self.tail + old_tail.next = new_tail + self.tail = new_tail + + def __nonzero__(self): + return self.head is not None + + def __len__(self): + cdef Py_ssize_t count = 0 + head = self.head + while head is not None: + count += 1 + head = head.next + return count + + def __iter__(self): + cdef list objects = [] + head = self.head + while head is not None: + objects.append(head) + head = head.next + return iter(objects) + + cdef bint has_callbacks(self): + return self.head + + def __repr__(self): + return "" % (id(self), len(self), self.head, self.tail) + + +cdef public class loop [object PyGeventLoopObject, type PyGeventLoop_Type]: + ## embedded struct members + cdef libev.ev_prepare _prepare + cdef libev.ev_timer _timer0 + # We'll only actually start this timer if we're on Windows, + # but it doesn't hurt to compile it in on all platforms. + cdef libev.ev_timer _periodic_signal_checker + + ## pointer members + cdef public object error_handler + cdef libev.ev_loop* _ptr + cdef public CallbackFIFO _callbacks + + ## data members + cdef bint starting_timer_may_update_loop_time + # We must capture the 'default' state at initialiaztion + # time. Destroying the default loop in libev sets + # the libev internal pointer to 0, and ev_is_default_loop will + # no longer work. + cdef bint _default + cdef readonly double approx_timer_resolution + + def __cinit__(self, object flags=None, object default=None, libev.intptr_t ptr=0): + self.starting_timer_may_update_loop_time = 0 + self._default = 0 + libev.ev_prepare_init(&self._prepare, + gevent_run_callbacks) + libev.ev_timer_init(&self._periodic_signal_checker, + gevent_periodic_signal_check, + 0.3, 0.3) + libev.ev_timer_init(&self._timer0, + gevent_noop, + 0.0, 0.0) + + cdef unsigned int c_flags + cdef object old_handler = None + if ptr: + self._ptr = ptr + self._default = libev.ev_is_default_loop(self._ptr) + else: + c_flags = _flags_to_int(flags) + _check_flags(c_flags) + c_flags |= libev.EVFLAG_NOENV + c_flags |= libev.EVFLAG_FORKCHECK + if default is None: + default = True + if default: + self._default = 1 + self._ptr = libev.gevent_ev_default_loop(c_flags) + if not self._ptr: + raise SystemError("ev_default_loop(%s) failed" % (c_flags, )) + if sys.platform == "win32": + libev.ev_timer_start(self._ptr, &self._periodic_signal_checker) + libev.ev_unref(self._ptr) + else: + self._ptr = libev.ev_loop_new(c_flags) + if not self._ptr: + raise SystemError("ev_loop_new(%s) failed" % (c_flags, )) + if default or __SYSERR_CALLBACK is None: + set_syserr_cb(self._handle_syserr) + + # Mark as not destroyed + libev.ev_set_userdata(self._ptr, self._ptr) + + libev.ev_prepare_start(self._ptr, &self._prepare) + libev.ev_unref(self._ptr) + + def __init__(self, object flags=None, object default=None, libev.intptr_t ptr=0): + self._callbacks = CallbackFIFO() + # See libev.corecffi for this attribute. + self.approx_timer_resolution = 0.00001 + + cdef _run_callbacks(self): + cdef callback cb + cdef object callbacks + cdef int count = CALLBACK_CHECK_COUNT + self.starting_timer_may_update_loop_time = True + cdef libev.ev_tstamp now = libev.ev_now(self._ptr) + cdef libev.ev_tstamp expiration = now + getswitchinterval() + + try: + libev.ev_timer_stop(self._ptr, &self._timer0) + while self._callbacks.head is not None: + cb = self._callbacks.popleft() + + libev.ev_unref(self._ptr) + gevent_call(self, cb) # XXX: Why is this a C callback, not cython? + count -= 1 + + if count == 0 and self._callbacks.head is not None: + # We still have more to run but we've reached + # the end of one check group + count = CALLBACK_CHECK_COUNT + + libev.ev_now_update(self._ptr) + if libev.ev_now(self._ptr) >= expiration: + now = 0 + break + + if now != 0: + libev.ev_now_update(self._ptr) + if self._callbacks.head is not None: + libev.ev_timer_start(self._ptr, &self._timer0) + finally: + self.starting_timer_may_update_loop_time = False + + cdef _stop_watchers(self, libev.ev_loop* ptr): + if not ptr: + return + + if libev.ev_is_active(&self._prepare): + libev.ev_ref(ptr) + libev.ev_prepare_stop(ptr, &self._prepare) + if libev.ev_is_active(&self._periodic_signal_checker): + libev.ev_ref(ptr) + libev.ev_timer_stop(ptr, &self._periodic_signal_checker) + + def destroy(self): + cdef libev.ev_loop* ptr = self._ptr + self._ptr = NULL + + if ptr: + if not libev.ev_userdata(ptr): + # Whoops! Program error. They destroyed the loop, + # using a different loop object. Our _ptr is still + # valid, but the libev loop is gone. Doing anything + # else with it will likely cause a crash. + return + # Mark as destroyed + libev.ev_set_userdata(ptr, NULL) + self._stop_watchers(ptr) + if __SYSERR_CALLBACK == self._handle_syserr: + set_syserr_cb(None) + libev.ev_loop_destroy(ptr) + + def __dealloc__(self): + cdef libev.ev_loop* ptr = self._ptr + self._ptr = NULL + if ptr != NULL: + if not libev.ev_userdata(ptr): + # See destroy(). This is a bug in the caller. + return + self._stop_watchers(ptr) + if not self._default: + libev.ev_loop_destroy(ptr) + # Mark as destroyed + libev.ev_set_userdata(ptr, NULL) + + @property + def ptr(self): + return self._ptr + + @property + def WatcherType(self): + return watcher + + @property + def MAXPRI(self): + return libev.EV_MAXPRI + + @property + def MINPRI(self): + return libev.EV_MINPRI + + def _handle_syserr(self, message, errno): + if sys.version_info[0] >= 3: + message = message.decode() + self.handle_error(None, SystemError, SystemError(message + ': ' + os.strerror(errno)), None) + + cpdef handle_error(self, context, type, value, tb): + cdef object handle_error + cdef object error_handler = self.error_handler + if error_handler is not None: + # we do want to do getattr every time so that setting Hub.handle_error property just works + handle_error = getattr(error_handler, 'handle_error', error_handler) + handle_error(context, type, value, tb) + else: + self._default_handle_error(context, type, value, tb) + + cpdef _default_handle_error(self, context, type, value, tb): + # note: Hub sets its own error handler so this is not used by gevent + # this is here to make core.loop usable without the rest of gevent + traceback.print_exception(type, value, tb) + if self._ptr: + libev.ev_break(self._ptr, libev.EVBREAK_ONE) + + def run(self, nowait=False, once=False): + _check_loop(self) + cdef unsigned int flags = 0 + if nowait: + flags |= libev.EVRUN_NOWAIT + if once: + flags |= libev.EVRUN_ONCE + with nogil: + libev.ev_run(self._ptr, flags) + + def reinit(self): + if self._ptr: + libev.ev_loop_fork(self._ptr) + + def ref(self): + _check_loop(self) + libev.ev_ref(self._ptr) + + def unref(self): + _check_loop(self) + libev.ev_unref(self._ptr) + + def break_(self, int how=libev.EVBREAK_ONE): + _check_loop(self) + libev.ev_break(self._ptr, how) + + def verify(self): + _check_loop(self) + libev.ev_verify(self._ptr) + + cpdef libev.ev_tstamp now(self) except *: + _check_loop(self) + return libev.ev_now(self._ptr) + + cpdef void update_now(self) except *: + _check_loop(self) + libev.ev_now_update(self._ptr) + + update = update_now # Old name, deprecated. + + def __repr__(self): + return '<%s at 0x%x %s>' % (self.__class__.__name__, id(self), self._format()) + + @property + def default(self): + # If we're destroyed, we are not the default loop anymore, + # as far as Python is concerned. + return self._default if self._ptr else False + + @property + def iteration(self): + _check_loop(self) + return libev.ev_iteration(self._ptr) + + @property + def depth(self): + _check_loop(self) + return libev.ev_depth(self._ptr) + + @property + def backend_int(self): + _check_loop(self) + return libev.ev_backend(self._ptr) + + @property + def backend(self): + _check_loop(self) + cdef unsigned int backend = libev.ev_backend(self._ptr) + for key, value in _flags: + if key == backend: + return value + return backend + + @property + def pendingcnt(self): + _check_loop(self) + return libev.ev_pending_count(self._ptr) + + def io(self, libev.vfd_socket_t fd, int events, ref=True, priority=None): + return io(self, fd, events, ref, priority) + + def timer(self, double after, double repeat=0.0, ref=True, priority=None): + return timer(self, after, repeat, ref, priority) + + def signal(self, int signum, ref=True, priority=None): + return signal(self, signum, ref, priority) + + def idle(self, ref=True, priority=None): + return idle(self, ref, priority) + + def prepare(self, ref=True, priority=None): + return prepare(self, ref, priority) + + def check(self, ref=True, priority=None): + return check(self, ref, priority) + + def fork(self, ref=True, priority=None): + return fork(self, ref, priority) + + def async_(self, ref=True, priority=None): + return async_(self, ref, priority) + + # cython doesn't enforce async as a keyword + async = async_ + + def child(self, int pid, bint trace=0, ref=True): + if sys.platform == 'win32': + raise AttributeError("Child watchers are not supported on Windows") + return child(self, pid, trace, ref) + + def install_sigchld(self): + libev.gevent_install_sigchld_handler() + + def reset_sigchld(self): + libev.gevent_reset_sigchld_handler() + + def stat(self, str path, float interval=0.0, ref=True, priority=None): + return stat(self, path, interval, ref, priority) + + def run_callback(self, func, *args): + _check_loop(self) + cdef callback cb = callback(func, args) + self._callbacks.append(cb) + libev.ev_ref(self._ptr) + return cb + + def _format(self): + if not self._ptr: + return 'destroyed' + cdef object msg = self.backend + if self._default: + msg += ' default' + msg += ' pending=%s' % self.pendingcnt + msg += self._format_details() + return msg + + def _format_details(self): + cdef str msg = '' + cdef object fileno = self.fileno() + cdef object activecnt = None + try: + sigfd = self.sigfd + except AttributeError: + sigfd = None + try: + activecnt = self.activecnt + except AttributeError: + pass + if activecnt is not None: + msg += ' ref=' + repr(activecnt) + if fileno is not None: + msg += ' fileno=' + repr(fileno) + return msg + + def fileno(self): + cdef int fd + if self._ptr: + fd = libev.gevent_ev_loop_backend_fd(self._ptr) + if fd >= 0: + return fd + + @property + def activecnt(self): + _check_loop(self) + return libev.gevent_ev_loop_activecnt(self._ptr) + + @property + def sig_pending(self): + _check_loop(self) + return libev.gevent_ev_loop_sig_pending(self._ptr) + + @property + def origflags(self): + return _flags_to_list(self.origflags_int) + + @property + def origflags_int(self): + _check_loop(self) + return libev.gevent_ev_loop_origflags(self._ptr) + + @property + def sigfd(self): + _check_loop(self) + fd = libev.gevent_ev_loop_sigfd(self._ptr) + if fd >= 0: + return fd + + # Explicitly not EV_USE_SIGNALFD + raise AttributeError("sigfd") + +try: + from zope.interface import classImplements +except ImportError: + pass +else: + # XXX: This invokes the side-table lookup, we would + # prefer to have it stored directly on the class. + from gevent._interfaces import ILoop + classImplements(loop, ILoop) + +# about readonly _flags attribute: +# bit #1 set if object owns Python reference to itself (Py_INCREF was +# called and we must call Py_DECREF later) +DEF FLAG_WATCHER_OWNS_PYREF = 1 << 0 # 0x1 +# bit #2 set if ev_unref() was called and we must call ev_ref() later +DEF FLAG_WATCHER_NEEDS_EVREF = 1 << 1 # 0x2 +# bit #3 set if user wants to call ev_unref() before start() +DEF FLAG_WATCHER_UNREF_BEFORE_START = 1 << 2 # 0x4 +# bits 2 and 3 are *both* set when we are active, but the user +# request us not to be ref'd anymore. We unref us (because going active will +# ref us) and then make a note of this in the future +DEF FLAG_WATCHER_MASK_UNREF_NEEDS_REF = 0x6 + + +cdef void _python_incref(watcher self): + if not self._flags & FLAG_WATCHER_OWNS_PYREF: + Py_INCREF(self) + self._flags |= FLAG_WATCHER_OWNS_PYREF + +cdef void _python_decref(watcher self): + if self._flags & FLAG_WATCHER_OWNS_PYREF: + Py_DECREF(self) + self._flags &= ~FLAG_WATCHER_OWNS_PYREF + +cdef void _libev_ref(watcher self): + if self._flags & FLAG_WATCHER_NEEDS_EVREF: + libev.ev_ref(self.loop._ptr) + self._flags &= ~FLAG_WATCHER_NEEDS_EVREF + +cdef void _libev_unref(watcher self): + if self._flags & FLAG_WATCHER_MASK_UNREF_NEEDS_REF == FLAG_WATCHER_UNREF_BEFORE_START: + libev.ev_unref(self.loop._ptr) + self._flags |= FLAG_WATCHER_NEEDS_EVREF + + +ctypedef void (*start_stop_func)(libev.ev_loop*, void*) nogil + +cdef struct start_and_stop: + start_stop_func start + start_stop_func stop + +cdef start_and_stop make_ss(void* start, void* stop): + cdef start_and_stop result = start_and_stop(start, stop) + return result + +cdef bint _watcher_start(watcher self, object callback, tuple args) except -1: + # This method should be called by subclasses of watcher, if they + # override the python-level `start` function: they've already paid + # for argument unpacking, and `start` cannot be cpdef since it + # uses varargs. + + # We keep this as a function, not a cdef method of watcher. + # If it's a cdef method, it could potentially be overridden + # by a subclass, which means that the watcher gains a pointer to a + # function table (vtable), making each object 8 bytes larger. + + _check_loop(self.loop) + if callback is None or not callable(callback): + raise TypeError("Expected callable, not %r" % (callback, )) + self._callback = callback + self.args = args + _libev_unref(self) + _python_incref(self) + self.__ss.start(self.loop._ptr, self.__watcher) + return 1 + +cdef public class watcher [object PyGeventWatcherObject, type PyGeventWatcher_Type]: + """Abstract base class for all the watchers""" + ## pointer members + cdef public loop loop + cdef object _callback + cdef public tuple args + + # By keeping a __watcher cached, the size of the io and timer + # structs becomes 152 bytes and child is 160 and stat is 512 (when + # the start_and_stop is inlined). On 64-bit macOS CPython 2.7. I + # hoped that using libev's data pointer and allocating the + # watchers directly and not as inline members would result in + # overall savings thanks to better padding, but it didn't. And it + # added lots of casts, making the code ugly. + + # Table: + # gevent ver | 1.2 | This | +data + # Watcher Kind | | | + # Timer | 120 | 152 | 160 + # IO | 120 | 152 | 160 + # Child | 128 | 160 | 168 + # Stat | 480 | 512 | 512 + cdef libev.ev_watcher* __watcher + + # By inlining the start_and_stop struct, instead of taking the address + # of a static struct or using the watcher's data pointer, we + # use an additional pointer of memory and incur an additional pointer copy + # on creation. + # But we use fewer pointer accesses for start/stop, and they have + # better cache locality. (Then again, we're bigger). + # Right now we're going for size, so we use the pointer. IO/Timer objects + # are then 144 bytes. + cdef start_and_stop* __ss + + ## Int members + + # Our subclasses will declare the ev_X struct + # as an inline member. This is good for locality, but + # probably bad for alignment, as it will get tacked on + # immediately after our data. + + # But all ev_watchers start with some ints, so maybe we can help that + # out by putting our ints here. + cdef readonly unsigned int _flags + + def __init__(self, loop loop, ref=True, priority=None): + if not self.__watcher or not self.__ss.start or not self.__ss.stop: + raise ValueError("Cannot construct a bare watcher") + self.loop = loop + self._flags = 0 if ref else FLAG_WATCHER_UNREF_BEFORE_START + if priority is not None: + libev.ev_set_priority(self.__watcher, priority) + + @property + def ref(self): + return False if self._flags & 4 else True + + @ref.setter + def ref(self, object value): + _check_loop(self.loop) + if value: + # self.ref should be true after this. + if self.ref: + return # ref is already True + + if self._flags & FLAG_WATCHER_NEEDS_EVREF: # ev_unref was called, undo + libev.ev_ref(self.loop._ptr) + # do not want unref, no outstanding unref + self._flags &= ~FLAG_WATCHER_MASK_UNREF_NEEDS_REF + else: + # self.ref must be false after this + if not self.ref: + return # ref is already False + self._flags |= FLAG_WATCHER_UNREF_BEFORE_START + if not self._flags & FLAG_WATCHER_NEEDS_EVREF and libev.ev_is_active(self.__watcher): + libev.ev_unref(self.loop._ptr) + self._flags |= FLAG_WATCHER_NEEDS_EVREF + + @property + def callback(self): + return self._callback + + @callback.setter + def callback(self, object callback): + if callback is not None and not callable(callback): + raise TypeError("Expected callable, not %r" % (callback, )) + self._callback = callback + + @property + def priority(self): + return libev.ev_priority(self.__watcher) + + @priority.setter + def priority(self, int priority): + cdef libev.ev_watcher* w = self.__watcher + if libev.ev_is_active(w): + raise AttributeError("Cannot set priority of an active watcher") + libev.ev_set_priority(w, priority) + + @property + def active(self): + return True if libev.ev_is_active(self.__watcher) else False + + @property + def pending(self): + return True if libev.ev_is_pending(self.__watcher) else False + + def start(self, object callback, *args): + _watcher_start(self, callback, args) + + def stop(self): + _check_loop(self.loop) + _libev_ref(self) + # The callback cannot possibly fire while we are executing, + # so this is safe. + self._callback = None + self.args = None + self.__ss.stop(self.loop._ptr, self.__watcher) + _python_decref(self) + + def feed(self, int revents, object callback, *args): + _check_loop(self.loop) + self.callback = callback + self.args = args + _libev_unref(self) + libev.ev_feed_event(self.loop._ptr, self.__watcher, revents) + _python_incref(self) + + def __repr__(self): + if Py_ReprEnter(self) != 0: + return "<...>" + try: + format = self._format() + result = "<%s at 0x%x%s" % (self.__class__.__name__, id(self), format) + if self.active: + result += " active" + if self.pending: + result += " pending" + if self.callback is not None: + result += " callback=%r" % (self.callback, ) + if self.args is not None: + result += " args=%r" % (self.args, ) + return result + ">" + finally: + Py_ReprLeave(self) + + def _format(self): + return '' + + def close(self): + self.stop() + + def __enter__(self): + return self + + def __exit__(self, t, v, tb): + self.close() + return + +cdef start_and_stop io_ss = make_ss(libev.ev_io_start, libev.ev_io_stop) + +cdef public class io(watcher) [object PyGeventIOObject, type PyGeventIO_Type]: + + cdef libev.ev_io _watcher + + def start(self, object callback, *args, pass_events=False): + if pass_events: + args = (GEVENT_CORE_EVENTS, ) + args + _watcher_start(self, callback, args) + + def __init__(self, loop loop, libev.vfd_socket_t fd, int events, ref=True, priority=None): + watcher.__init__(self, loop, ref, priority) + + def __cinit__(self, loop loop, libev.vfd_socket_t fd, int events, ref=True, priority=None): + if fd < 0: + raise ValueError('fd must be non-negative: %r' % fd) + if events & ~(libev.EV__IOFDSET | libev.EV_READ | libev.EV_WRITE): + raise ValueError('illegal event mask: %r' % events) + # All the vfd_functions are no-ops on POSIX + cdef int vfd = libev.vfd_open(fd) + libev.ev_io_init(&self._watcher, gevent_callback_io, vfd, events) + self.__watcher = &self._watcher + self.__ss = &io_ss + + def __dealloc__(self): + libev.vfd_free(self._watcher.fd) + + @property + def fd(self): + return libev.vfd_get(self._watcher.fd) + + @fd.setter + def fd(self, long fd): + if libev.ev_is_active(&self._watcher): + raise AttributeError("'io' watcher attribute 'fd' is read-only while watcher is active") + cdef int vfd = libev.vfd_open(fd) + libev.vfd_free(self._watcher.fd) + libev.ev_io_init(&self._watcher, gevent_callback_io, vfd, self._watcher.events) + + @property + def events(self): + return self._watcher.events + + @events.setter + def events(self, int events): + if libev.ev_is_active(&self._watcher): + raise AttributeError("'io' watcher attribute 'events' is read-only while watcher is active") + libev.ev_io_init(&self._watcher, gevent_callback_io, self._watcher.fd, events) + + @property + def events_str(self): + return _events_to_str(self._watcher.events) + + def _format(self): + return ' fd=%s events=%s' % (self.fd, self.events_str) + +cdef start_and_stop timer_ss = make_ss(libev.ev_timer_start, libev.ev_timer_stop) + +cdef public class timer(watcher) [object PyGeventTimerObject, type PyGeventTimer_Type]: + + cdef libev.ev_timer _watcher + + def __cinit__(self, loop loop, double after=0.0, double repeat=0.0, ref=True, priority=None): + if repeat < 0.0: + raise ValueError("repeat must be positive or zero: %r" % repeat) + libev.ev_timer_init(&self._watcher, gevent_callback_timer, after, repeat) + self.__watcher = &self._watcher + self.__ss = &timer_ss + + def __init__(self, loop loop, double after=0.0, double repeat=0.0, ref=True, priority=None): + watcher.__init__(self, loop, ref, priority) + + def start(self, object callback, *args, update=None): + update = update if update is not None else self.loop.starting_timer_may_update_loop_time + if update: + self.loop.update_now() + _watcher_start(self, callback, args) + + @property + def at(self): + return self._watcher.at + + # QQQ: add 'after' and 'repeat' properties? + + def again(self, object callback, *args, update=True): + _check_loop(self.loop) + self.callback = callback + self.args = args + _libev_unref(self) + if update: + libev.ev_now_update(self.loop._ptr) + libev.ev_timer_again(self.loop._ptr, &self._watcher) + _python_incref(self) + + + +cdef start_and_stop signal_ss = make_ss(libev.ev_signal_start, libev.ev_signal_stop) + +cdef public class signal(watcher) [object PyGeventSignalObject, type PyGeventSignal_Type]: + + cdef libev.ev_signal _watcher + + def __cinit__(self, loop loop, int signalnum, ref=True, priority=None): + if signalnum < 1 or signalnum >= signalmodule.NSIG: + raise ValueError('illegal signal number: %r' % signalnum) + # still possible to crash on one of libev's asserts: + # 1) "libev: ev_signal_start called with illegal signal number" + # EV_NSIG might be different from signal.NSIG on some platforms + # 2) "libev: a signal must not be attached to two different loops" + # we probably could check that in LIBEV_EMBED mode, but not in general + libev.ev_signal_init(&self._watcher, gevent_callback_signal, signalnum) + self.__watcher = &self._watcher + self.__ss = &signal_ss + + def __init__(self, loop loop, int signalnum, ref=True, priority=None): + watcher.__init__(self, loop, ref, priority) + + + +cdef start_and_stop idle_ss = make_ss(libev.ev_idle_start, libev.ev_idle_stop) + +cdef public class idle(watcher) [object PyGeventIdleObject, type PyGeventIdle_Type]: + + cdef libev.ev_idle _watcher + + def __cinit__(self, loop loop, ref=True, priority=None): + libev.ev_idle_init(&self._watcher, gevent_callback_idle) + self.__watcher = &self._watcher + self.__ss = &idle_ss + + + +cdef start_and_stop prepare_ss = make_ss(libev.ev_prepare_start, libev.ev_prepare_stop) + +cdef public class prepare(watcher) [object PyGeventPrepareObject, type PyGeventPrepare_Type]: + + cdef libev.ev_prepare _watcher + + def __cinit__(self, loop loop, ref=True, priority=None): + libev.ev_prepare_init(&self._watcher, gevent_callback_prepare) + self.__watcher = &self._watcher + self.__ss = &prepare_ss + + + +cdef start_and_stop check_ss = make_ss(libev.ev_check_start, libev.ev_check_stop) + +cdef public class check(watcher) [object PyGeventCheckObject, type PyGeventCheck_Type]: + + cdef libev.ev_check _watcher + + def __cinit__(self, loop loop, ref=True, priority=None): + libev.ev_check_init(&self._watcher, gevent_callback_check) + self.__watcher = &self._watcher + self.__ss = &check_ss + + + +cdef start_and_stop fork_ss = make_ss(libev.ev_fork_start, libev.ev_fork_stop) + +cdef public class fork(watcher) [object PyGeventForkObject, type PyGeventFork_Type]: + + cdef libev.ev_fork _watcher + + def __cinit__(self, loop loop, ref=True, priority=None): + libev.ev_fork_init(&self._watcher, gevent_callback_fork) + self.__watcher = &self._watcher + self.__ss = &fork_ss + + +cdef start_and_stop async_ss = make_ss(libev.ev_async_start, libev.ev_async_stop) + +cdef public class async_(watcher) [object PyGeventAsyncObject, type PyGeventAsync_Type]: + + cdef libev.ev_async _watcher + + @property + def pending(self): + # Note the use of ev_async_pending instead of ev_is_pending + return True if libev.ev_async_pending(&self._watcher) else False + + def __cinit__(self, loop loop, ref=True, priority=None): + libev.ev_async_init(&self._watcher, gevent_callback_async) + self.__watcher = &self._watcher + self.__ss = &async_ss + + + def send(self): + _check_loop(self.loop) + libev.ev_async_send(self.loop._ptr, &self._watcher) + +async = async_ + +cdef start_and_stop child_ss = make_ss(libev.ev_child_start, libev.ev_child_stop) + +cdef public class child(watcher) [object PyGeventChildObject, type PyGeventChild_Type]: + + cdef libev.ev_child _watcher + + def __cinit__(self, loop loop, int pid, bint trace=0, ref=True): + if sys.platform == 'win32': + raise AttributeError("Child watchers are not supported on Windows") + if not loop.default: + raise TypeError('child watchers are only available on the default loop') + libev.gevent_install_sigchld_handler() + libev.ev_child_init(&self._watcher, gevent_callback_child, pid, trace) + self.__watcher = &self._watcher + self.__ss = &child_ss + + def __init__(self, loop loop, int pid, bint trace=0, ref=True): + watcher.__init__(self, loop, ref, None) + + + def _format(self): + return ' pid=%r rstatus=%r' % (self.pid, self.rstatus) + + @property + def pid(self): + return self._watcher.pid + + @property + def rpid(self): + return self._watcher.rpid + + @rpid.setter + def rpid(self, int value): + self._watcher.rpid = value + + @property + def rstatus(self): + return self._watcher.rstatus + + @rstatus.setter + def rstatus(self, int value): + self._watcher.rstatus = value + +cdef start_and_stop stat_ss = make_ss(libev.ev_stat_start, libev.ev_stat_stop) + +cdef public class stat(watcher) [object PyGeventStatObject, type PyGeventStat_Type]: + + cdef libev.ev_stat _watcher + cdef readonly str path + cdef readonly bytes _paths + + def __cinit__(self, loop loop, str path, float interval=0.0, ref=True, priority=None): + self.path = path + cdef bytes paths + if isinstance(path, unicode): + # the famous Python3 filesystem encoding debacle hits us here. Can we do better? + # We must keep a reference to the encoded string so that its bytes don't get freed + # and overwritten, leading to strange errors from libev ("no such file or directory") + paths = (path).encode(sys.getfilesystemencoding()) + self._paths = paths + else: + paths = path + self._paths = paths + libev.ev_stat_init(&self._watcher, gevent_callback_stat, paths, interval) + self.__watcher = &self._watcher + self.__ss = &stat_ss + + def __init__(self, loop loop, str path, float interval=0.0, ref=True, priority=None): + watcher.__init__(self, loop, ref, priority) + + + @property + def attr(self): + if not self._watcher.attr.st_nlink: + return + return _pystat_fromstructstat(&self._watcher.attr) + + @property + def prev(self): + if not self._watcher.prev.st_nlink: + return + return _pystat_fromstructstat(&self._watcher.prev) + + @property + def interval(self): + return self._watcher.interval + + + +__SYSERR_CALLBACK = None + + +cdef void _syserr_cb(char* msg) with gil: + try: + __SYSERR_CALLBACK(msg, errno) + except: + set_syserr_cb(None) + print_exc = getattr(traceback, 'print_exc', None) + if print_exc is not None: + print_exc() + + +cpdef set_syserr_cb(callback): + global __SYSERR_CALLBACK + if callback is None: + libev.ev_set_syserr_cb(NULL) + __SYSERR_CALLBACK = None + elif callable(callback): + libev.ev_set_syserr_cb(_syserr_cb) + __SYSERR_CALLBACK = callback + else: + raise TypeError('Expected callable or None, got %r' % (callback, )) + + + +LIBEV_EMBED = bool(libev.LIBEV_EMBED) +EV_USE_FLOOR = libev.EV_USE_FLOOR +EV_USE_CLOCK_SYSCALL = libev.EV_USE_CLOCK_SYSCALL +EV_USE_REALTIME = libev.EV_USE_REALTIME +EV_USE_MONOTONIC = libev.EV_USE_MONOTONIC +EV_USE_NANOSLEEP = libev.EV_USE_NANOSLEEP +EV_USE_INOTIFY = libev.EV_USE_INOTIFY +EV_USE_SIGNALFD = libev.EV_USE_SIGNALFD +EV_USE_EVENTFD = libev.EV_USE_EVENTFD +EV_USE_4HEAP = libev.EV_USE_4HEAP + +# Things used in callbacks.c + +from cpython cimport PyErr_Fetch +from cpython cimport PyObject + +cdef public void gevent_handle_error(loop loop, object context): + cdef PyObject* typep + cdef PyObject* valuep + cdef PyObject* tracebackp + + cdef object type + cdef object value = None + cdef object traceback = None + cdef object result + + # If it was set, this will clear it, and we will own + # the references. + PyErr_Fetch(&typep, &valuep, &tracebackp) + # TODO: Should we call PyErr_Normalize? There's code in + # Hub.handle_error that works around what looks like an + # unnormalized exception. + + if not typep: + return + # This assignment will do a Py_INCREF + # on the value. We already own the reference + # returned from PyErr_Fetch, + # so we must decref immediately + type = typep + Py_DECREF(type) + + if valuep: + value = valuep + Py_DECREF(value) + if tracebackp: + traceback = tracebackp + Py_DECREF(traceback) + + # If this method fails by raising an exception, + # cython will print it for us because we don't return a + # Python object and we don't declare an `except` clause. + loop.handle_error(context, type, value, traceback) + +cdef public tuple _empty_tuple = () + +cdef public object gevent_loop_run_callbacks(loop loop): + return loop._run_callbacks() diff --git a/src/gevent/libev/corecffi.py b/src/gevent/libev/corecffi.py new file mode 100644 index 0000000..51701c4 --- /dev/null +++ b/src/gevent/libev/corecffi.py @@ -0,0 +1,427 @@ +# pylint: disable=too-many-lines, protected-access, redefined-outer-name, not-callable +# pylint: disable=no-member +from __future__ import absolute_import, print_function +import sys + +# pylint: disable=undefined-all-variable +__all__ = [ + 'get_version', + 'get_header_version', + 'supported_backends', + 'recommended_backends', + 'embeddable_backends', + 'time', + 'loop', +] + +from gevent._util import implementer +from gevent._interfaces import ILoop + +from gevent.libev import _corecffi # pylint:disable=no-name-in-module,import-error + +ffi = _corecffi.ffi # pylint:disable=no-member +libev = _corecffi.lib # pylint:disable=no-member + +if hasattr(libev, 'vfd_open'): + # Must be on windows + assert sys.platform.startswith("win"), "vfd functions only needed on windows" + vfd_open = libev.vfd_open + vfd_free = libev.vfd_free + vfd_get = libev.vfd_get +else: + vfd_open = vfd_free = vfd_get = lambda fd: fd + +##### +## NOTE on Windows: +# The C implementation does several things specially for Windows; +# a possibly incomplete list is: +# +# - the loop runs a periodic signal checker; +# - the io watcher constructor is different and it has a destructor; +# - the child watcher is not defined +# +# The CFFI implementation does none of these things, and so +# is possibly NOT FUNCTIONALLY CORRECT on Win32 +##### + + +from gevent._ffi.loop import AbstractCallbacks +from gevent._ffi.loop import assign_standard_callbacks + +class _Callbacks(AbstractCallbacks): + # pylint:disable=arguments-differ + + def python_check_callback(self, _loop, watcher_ptr, _events): + pass + + def python_prepare_callback(self, _loop_ptr, watcher_ptr, _events): + AbstractCallbacks.python_prepare_callback(self, watcher_ptr) + + def _find_loop_from_c_watcher(self, watcher_ptr): + loop_handle = ffi.cast('struct ev_watcher*', watcher_ptr).data + return self.from_handle(loop_handle) + +_callbacks = assign_standard_callbacks(ffi, libev, _Callbacks) + + +UNDEF = libev.EV_UNDEF +NONE = libev.EV_NONE +READ = libev.EV_READ +WRITE = libev.EV_WRITE +TIMER = libev.EV_TIMER +PERIODIC = libev.EV_PERIODIC +SIGNAL = libev.EV_SIGNAL +CHILD = libev.EV_CHILD +STAT = libev.EV_STAT +IDLE = libev.EV_IDLE +PREPARE = libev.EV_PREPARE +CHECK = libev.EV_CHECK +EMBED = libev.EV_EMBED +FORK = libev.EV_FORK +CLEANUP = libev.EV_CLEANUP +ASYNC = libev.EV_ASYNC +CUSTOM = libev.EV_CUSTOM +ERROR = libev.EV_ERROR + +READWRITE = libev.EV_READ | libev.EV_WRITE + +MINPRI = libev.EV_MINPRI +MAXPRI = libev.EV_MAXPRI + +BACKEND_PORT = libev.EVBACKEND_PORT +BACKEND_KQUEUE = libev.EVBACKEND_KQUEUE +BACKEND_EPOLL = libev.EVBACKEND_EPOLL +BACKEND_POLL = libev.EVBACKEND_POLL +BACKEND_SELECT = libev.EVBACKEND_SELECT +FORKCHECK = libev.EVFLAG_FORKCHECK +NOINOTIFY = libev.EVFLAG_NOINOTIFY +SIGNALFD = libev.EVFLAG_SIGNALFD +NOSIGMASK = libev.EVFLAG_NOSIGMASK + + +from gevent._ffi.loop import EVENTS +GEVENT_CORE_EVENTS = EVENTS + + +def get_version(): + return 'libev-%d.%02d' % (libev.ev_version_major(), libev.ev_version_minor()) + + +def get_header_version(): + return 'libev-%d.%02d' % (libev.EV_VERSION_MAJOR, libev.EV_VERSION_MINOR) + +_flags = [(libev.EVBACKEND_PORT, 'port'), + (libev.EVBACKEND_KQUEUE, 'kqueue'), + (libev.EVBACKEND_EPOLL, 'epoll'), + (libev.EVBACKEND_POLL, 'poll'), + (libev.EVBACKEND_SELECT, 'select'), + (libev.EVFLAG_NOENV, 'noenv'), + (libev.EVFLAG_FORKCHECK, 'forkcheck'), + (libev.EVFLAG_SIGNALFD, 'signalfd'), + (libev.EVFLAG_NOSIGMASK, 'nosigmask')] + +_flags_str2int = dict((string, flag) for (flag, string) in _flags) + + + +def _flags_to_list(flags): + result = [] + for code, value in _flags: + if flags & code: + result.append(value) + flags &= ~code + if not flags: + break + if flags: + result.append(flags) + return result + +if sys.version_info[0] >= 3: + basestring = (bytes, str) + integer_types = (int,) +else: + import __builtin__ # pylint:disable=import-error + basestring = (__builtin__.basestring,) + integer_types = (int, __builtin__.long) + + +def _flags_to_int(flags): + # Note, that order does not matter, libev has its own predefined order + if not flags: + return 0 + if isinstance(flags, integer_types): + return flags + result = 0 + try: + if isinstance(flags, basestring): + flags = flags.split(',') + for value in flags: + value = value.strip().lower() + if value: + result |= _flags_str2int[value] + except KeyError as ex: + raise ValueError('Invalid backend or flag: %s\nPossible values: %s' % (ex, ', '.join(sorted(_flags_str2int.keys())))) + return result + + +def _str_hex(flag): + if isinstance(flag, integer_types): + return hex(flag) + return str(flag) + + +def _check_flags(flags): + as_list = [] + flags &= libev.EVBACKEND_MASK + if not flags: + return + if not flags & libev.EVBACKEND_ALL: + raise ValueError('Invalid value for backend: 0x%x' % flags) + if not flags & libev.ev_supported_backends(): + as_list = [_str_hex(x) for x in _flags_to_list(flags)] + raise ValueError('Unsupported backend: %s' % '|'.join(as_list)) + + +def supported_backends(): + return _flags_to_list(libev.ev_supported_backends()) + + +def recommended_backends(): + return _flags_to_list(libev.ev_recommended_backends()) + + +def embeddable_backends(): + return _flags_to_list(libev.ev_embeddable_backends()) + + +def time(): + return libev.ev_time() + +from gevent._ffi.loop import AbstractLoop + + +from gevent.libev import watcher as _watchers +_events_to_str = _watchers._events_to_str # exported + + +@implementer(ILoop) +class loop(AbstractLoop): + # pylint:disable=too-many-public-methods + + # libuv parameters simply won't accept anything lower than 1ms + # (0.001s), but libev takes fractional seconds. In practice, on + # one machine, libev can sleep for very small periods of time: + # + # sleep(0.00001) -> 0.000024 + # sleep(0.0001) -> 0.000156 + # sleep(0.001) -> 0.00136 (which is comparable to libuv) + + approx_timer_resolution = 0.00001 + + error_handler = None + + _CHECK_POINTER = 'struct ev_check *' + + _PREPARE_POINTER = 'struct ev_prepare *' + + _TIMER_POINTER = 'struct ev_timer *' + + def __init__(self, flags=None, default=None): + AbstractLoop.__init__(self, ffi, libev, _watchers, flags, default) + self._default = bool(libev.ev_is_default_loop(self._ptr)) + + def _init_loop(self, flags, default): + c_flags = _flags_to_int(flags) + _check_flags(c_flags) + c_flags |= libev.EVFLAG_NOENV + c_flags |= libev.EVFLAG_FORKCHECK + if default is None: + default = True + if default: + ptr = libev.gevent_ev_default_loop(c_flags) + if not ptr: + raise SystemError("ev_default_loop(%s) failed" % (c_flags, )) + else: + ptr = libev.ev_loop_new(c_flags) + if not ptr: + raise SystemError("ev_loop_new(%s) failed" % (c_flags, )) + if default or globals()["__SYSERR_CALLBACK"] is None: + set_syserr_cb(self._handle_syserr) + + # Mark this loop as being used. + libev.ev_set_userdata(ptr, ptr) + return ptr + + def _init_and_start_check(self): + libev.ev_check_init(self._check, libev.python_check_callback) + self._check.data = self._handle_to_self + libev.ev_check_start(self._ptr, self._check) + self.unref() + + def _init_and_start_prepare(self): + libev.ev_prepare_init(self._prepare, libev.python_prepare_callback) + libev.ev_prepare_start(self._ptr, self._prepare) + self.unref() + + def _init_callback_timer(self): + libev.ev_timer_init(self._timer0, libev.gevent_noop, 0.0, 0.0) + + def _stop_callback_timer(self): + libev.ev_timer_stop(self._ptr, self._timer0) + + def _start_callback_timer(self): + libev.ev_timer_start(self._ptr, self._timer0) + + def _stop_aux_watchers(self): + if libev.ev_is_active(self._prepare): + self.ref() + libev.ev_prepare_stop(self._ptr, self._prepare) + if libev.ev_is_active(self._check): + self.ref() + libev.ev_check_stop(self._ptr, self._check) + if libev.ev_is_active(self._timer0): + libev.ev_timer_stop(self._timer0) + + def _setup_for_run_callback(self): + self.ref() # we should go through the loop now + + def destroy(self): + if self._ptr: + super(loop, self).destroy() + # pylint:disable=comparison-with-callable + if globals()["__SYSERR_CALLBACK"] == self._handle_syserr: + set_syserr_cb(None) + + + def _can_destroy_loop(self, ptr): + # Is it marked as destroyed? + return libev.ev_userdata(ptr) + + def _destroy_loop(self, ptr): + # Mark as destroyed. + libev.ev_set_userdata(ptr, ffi.NULL) + libev.ev_loop_destroy(ptr) + + libev.gevent_zero_prepare(self._prepare) + libev.gevent_zero_check(self._check) + libev.gevent_zero_timer(self._timer0) + + del self._prepare + del self._check + del self._timer0 + + + @property + def MAXPRI(self): + return libev.EV_MAXPRI + + @property + def MINPRI(self): + return libev.EV_MINPRI + + def _default_handle_error(self, context, type, value, tb): # pylint:disable=unused-argument + super(loop, self)._default_handle_error(context, type, value, tb) + libev.ev_break(self._ptr, libev.EVBREAK_ONE) + + def run(self, nowait=False, once=False): + flags = 0 + if nowait: + flags |= libev.EVRUN_NOWAIT + if once: + flags |= libev.EVRUN_ONCE + + libev.ev_run(self._ptr, flags) + + def reinit(self): + libev.ev_loop_fork(self._ptr) + + def ref(self): + libev.ev_ref(self._ptr) + + def unref(self): + libev.ev_unref(self._ptr) + + def break_(self, how=libev.EVBREAK_ONE): + libev.ev_break(self._ptr, how) + + def verify(self): + libev.ev_verify(self._ptr) + + def now(self): + return libev.ev_now(self._ptr) + + def update_now(self): + libev.ev_now_update(self._ptr) + + def __repr__(self): + return '<%s at 0x%x %s>' % (self.__class__.__name__, id(self), self._format()) + + @property + def iteration(self): + return libev.ev_iteration(self._ptr) + + @property + def depth(self): + return libev.ev_depth(self._ptr) + + @property + def backend_int(self): + return libev.ev_backend(self._ptr) + + @property + def backend(self): + backend = libev.ev_backend(self._ptr) + for key, value in _flags: + if key == backend: + return value + return backend + + @property + def pendingcnt(self): + return libev.ev_pending_count(self._ptr) + + if sys.platform != "win32": + + def install_sigchld(self): + libev.gevent_install_sigchld_handler() + + def reset_sigchld(self): + libev.gevent_reset_sigchld_handler() + + def fileno(self): + if self._ptr: + fd = self._ptr.backend_fd + if fd >= 0: + return fd + + @property + def activecnt(self): + if not self._ptr: + raise ValueError('operation on destroyed loop') + return self._ptr.activecnt + + +@ffi.def_extern() +def _syserr_cb(msg): + try: + msg = ffi.string(msg) + __SYSERR_CALLBACK(msg, ffi.errno) + except: + set_syserr_cb(None) + raise # let cffi print the traceback + + +def set_syserr_cb(callback): + global __SYSERR_CALLBACK + if callback is None: + libev.ev_set_syserr_cb(ffi.NULL) + __SYSERR_CALLBACK = None + elif callable(callback): + libev.ev_set_syserr_cb(libev._syserr_cb) + __SYSERR_CALLBACK = callback + else: + raise TypeError('Expected callable or None, got %r' % (callback, )) + +__SYSERR_CALLBACK = None + +LIBEV_EMBED = True diff --git a/src/gevent/libev/libev.h b/src/gevent/libev/libev.h new file mode 100644 index 0000000..66959d9 --- /dev/null +++ b/src/gevent/libev/libev.h @@ -0,0 +1,100 @@ +#if defined(LIBEV_EMBED) +#include "ev.c" +#undef LIBEV_EMBED +#define LIBEV_EMBED 1 +#define gevent_ev_loop_origflags(loop) ((loop)->origflags) +#define gevent_ev_loop_sig_pending(loop) ((loop))->sig_pending +#define gevent_ev_loop_backend_fd(loop) ((loop))->backend_fd +#define gevent_ev_loop_activecnt(loop) ((loop))->activecnt +#if EV_USE_SIGNALFD +#define gevent_ev_loop_sigfd(loop) ((loop))->sigfd +#else +#define gevent_ev_loop_sigfd(loop) -1 +#endif /* !EV_USE_SIGNALFD */ +#else /* !LIBEV_EMBED */ +#include "ev.h" + +#define gevent_ev_loop_origflags(loop) -1 +#define gevent_ev_loop_sig_pending(loop) -1 +#define gevent_ev_loop_backend_fd(loop) -1 +#define gevent_ev_loop_activecnt(loop) -1 +#define gevent_ev_loop_sigfd(loop) -1 + +#define LIBEV_EMBED 0 +#define EV_USE_FLOOR -1 +#define EV_USE_CLOCK_SYSCALL -1 +#define EV_USE_REALTIME -1 +#define EV_USE_MONOTONIC -1 +#define EV_USE_NANOSLEEP -1 +#define EV_USE_INOTIFY -1 +#define EV_USE_SIGNALFD -1 +#define EV_USE_EVENTFD -1 +#define EV_USE_4HEAP -1 + + +#ifndef _WIN32 +#include +#endif /* !_WIN32 */ + +#endif /* LIBEV_EMBED */ + +#ifndef _WIN32 + +static struct sigaction libev_sigchld; +/* + * Track the state of whether we have installed + * the libev sigchld handler specifically. + * If it's non-zero, libev_sigchld will be valid and set to the action + * that libev needs to do. + * If it's 1, we need to install libev_sigchld to make libev + * child handlers work (on request). + */ +static int sigchld_state = 0; + +static struct ev_loop* gevent_ev_default_loop(unsigned int flags) +{ + struct ev_loop* result; + struct sigaction tmp; + + if (sigchld_state) + return ev_default_loop(flags); + + // Request the old SIGCHLD handler + sigaction(SIGCHLD, NULL, &tmp); + // Get the loop, which will install a SIGCHLD handler + result = ev_default_loop(flags); + // XXX what if SIGCHLD received there? + // Now restore the previous SIGCHLD handler + sigaction(SIGCHLD, &tmp, &libev_sigchld); + sigchld_state = 1; + return result; +} + + +static void gevent_install_sigchld_handler(void) { + if (sigchld_state == 1) { + sigaction(SIGCHLD, &libev_sigchld, NULL); + sigchld_state = 2; + } +} + +static void gevent_reset_sigchld_handler(void) { + // We could have any state at this point, depending on + // whether the default loop has been used. If it has, + // then always be in state 1 ("need to install) + if (sigchld_state) { + sigchld_state = 1; + } +} + +#else /* !_WIN32 */ + +#define gevent_ev_default_loop ev_default_loop +static void gevent_install_sigchld_handler(void) { } +static void gevent_reset_sigchld_handler(void) { } + +// Fake child functions that we can link to. +static void ev_child_start(struct ev_loop* loop, ev_child* w) {}; +static void ev_child_stop(struct ev_loop* loop, ev_child* w) {}; + +#endif /* _WIN32 */ diff --git a/src/gevent/libev/libev.pxd b/src/gevent/libev/libev.pxd new file mode 100644 index 0000000..e3be525 --- /dev/null +++ b/src/gevent/libev/libev.pxd @@ -0,0 +1,235 @@ +# From cython/includes/libc/stdint.pxd +# Longness only used for type promotion. +# Actual compile time size used for conversions. +# We don't have stdint.h on visual studio 9.0 (2008) on windows, sigh, +# so go with Py_ssize_t +# ssize_t -> intptr_t + +cdef extern from "libev_vfd.h": +# cython doesn't process pre-processor directives, so they +# don't matter in this file. It just takes the last definition it sees. + ctypedef Py_ssize_t intptr_t + ctypedef intptr_t vfd_socket_t + + vfd_socket_t vfd_get(int) + int vfd_open(long) except -1 + void vfd_free(int) + +cdef extern from "libev.h" nogil: + int LIBEV_EMBED + int EV_MINPRI + int EV_MAXPRI + + int EV_VERSION_MAJOR + int EV_VERSION_MINOR + + int EV_USE_FLOOR + int EV_USE_CLOCK_SYSCALL + int EV_USE_REALTIME + int EV_USE_MONOTONIC + int EV_USE_NANOSLEEP + int EV_USE_SELECT + int EV_USE_POLL + int EV_USE_EPOLL + int EV_USE_KQUEUE + int EV_USE_PORT + int EV_USE_INOTIFY + int EV_USE_SIGNALFD + int EV_USE_EVENTFD + int EV_USE_4HEAP + int EV_USE_IOCP + int EV_SELECT_IS_WINSOCKET + + int EV_UNDEF + int EV_NONE + int EV_READ + int EV_WRITE + int EV__IOFDSET + int EV_TIMER + int EV_PERIODIC + int EV_SIGNAL + int EV_CHILD + int EV_STAT + int EV_IDLE + int EV_PREPARE + int EV_CHECK + int EV_EMBED + int EV_FORK + int EV_CLEANUP + int EV_ASYNC + int EV_CUSTOM + int EV_ERROR + + int EVFLAG_AUTO + int EVFLAG_NOENV + int EVFLAG_FORKCHECK + int EVFLAG_NOINOTIFY + int EVFLAG_SIGNALFD + int EVFLAG_NOSIGMASK + + int EVBACKEND_SELECT + int EVBACKEND_POLL + int EVBACKEND_EPOLL + int EVBACKEND_KQUEUE + int EVBACKEND_DEVPOLL + int EVBACKEND_PORT + int EVBACKEND_IOCP + int EVBACKEND_ALL + int EVBACKEND_MASK + + int EVRUN_NOWAIT + int EVRUN_ONCE + + int EVBREAK_CANCEL + int EVBREAK_ONE + int EVBREAK_ALL + + struct ev_loop: + int activecnt + int sig_pending + int backend_fd + int sigfd + unsigned int origflags + + struct ev_watcher: + void* data; + + struct ev_io: + int fd + int events + + struct ev_timer: + double at + + struct ev_signal: + pass + + struct ev_idle: + pass + + struct ev_prepare: + pass + + struct ev_check: + pass + + struct ev_fork: + pass + + struct ev_async: + pass + + struct ev_child: + int pid + int rpid + int rstatus + + struct stat: + int st_nlink + + struct ev_stat: + stat attr + stat prev + double interval + + union ev_any_watcher: + ev_watcher w + ev_io io + ev_timer timer + ev_signal signal + ev_idle idle + + int ev_version_major() + int ev_version_minor() + + unsigned int ev_supported_backends() + unsigned int ev_recommended_backends() + unsigned int ev_embeddable_backends() + + ctypedef double ev_tstamp + + ev_tstamp ev_time() + void ev_set_syserr_cb(void *) + + int ev_priority(void*) + void ev_set_priority(void*, int) + + int ev_is_pending(void*) + int ev_is_active(void*) + void ev_io_init(ev_io*, void* callback, int fd, int events) + void ev_io_start(ev_loop*, ev_io*) + void ev_io_stop(ev_loop*, ev_io*) + void ev_feed_event(ev_loop*, void*, int) + + void ev_timer_init(ev_timer*, void* callback, double, double) + void ev_timer_start(ev_loop*, ev_timer*) + void ev_timer_stop(ev_loop*, ev_timer*) + void ev_timer_again(ev_loop*, ev_timer*) + + void ev_signal_init(ev_signal*, void* callback, int) + void ev_signal_start(ev_loop*, ev_signal*) + void ev_signal_stop(ev_loop*, ev_signal*) + + void ev_idle_init(ev_idle*, void* callback) + void ev_idle_start(ev_loop*, ev_idle*) + void ev_idle_stop(ev_loop*, ev_idle*) + + void ev_prepare_init(ev_prepare*, void* callback) + void ev_prepare_start(ev_loop*, ev_prepare*) + void ev_prepare_stop(ev_loop*, ev_prepare*) + + void ev_check_init(ev_check*, void* callback) + void ev_check_start(ev_loop*, ev_check*) + void ev_check_stop(ev_loop*, ev_check*) + + void ev_fork_init(ev_fork*, void* callback) + void ev_fork_start(ev_loop*, ev_fork*) + void ev_fork_stop(ev_loop*, ev_fork*) + + void ev_async_init(ev_async*, void* callback) + void ev_async_start(ev_loop*, ev_async*) + void ev_async_stop(ev_loop*, ev_async*) + void ev_async_send(ev_loop*, ev_async*) + int ev_async_pending(ev_async*) + + void ev_child_init(ev_child*, void* callback, int, int) + void ev_child_start(ev_loop*, ev_child*) + void ev_child_stop(ev_loop*, ev_child*) + + void ev_stat_init(ev_stat*, void* callback, char*, double) + void ev_stat_start(ev_loop*, ev_stat*) + void ev_stat_stop(ev_loop*, ev_stat*) + + ev_loop* ev_default_loop(unsigned int flags) + ev_loop* ev_loop_new(unsigned int flags) + void* ev_userdata(ev_loop*) + void ev_set_userdata(ev_loop*, void*) + void ev_loop_destroy(ev_loop*) + void ev_loop_fork(ev_loop*) + int ev_is_default_loop(ev_loop*) + unsigned int ev_iteration(ev_loop*) + unsigned int ev_depth(ev_loop*) + unsigned int ev_backend(ev_loop*) + void ev_verify(ev_loop*) + void ev_run(ev_loop*, int flags) nogil + + ev_tstamp ev_now(ev_loop*) + void ev_now_update(ev_loop*) + + void ev_ref(ev_loop*) + void ev_unref(ev_loop*) + void ev_break(ev_loop*, int) + unsigned int ev_pending_count(ev_loop*) + + # gevent extra functions. These are defined in libev.h. + ev_loop* gevent_ev_default_loop(unsigned int flags) + void gevent_install_sigchld_handler() + void gevent_reset_sigchld_handler() + + # These compensate for lack of access to ev_loop struct definition + # when LIBEV_EMBED is false. + unsigned int gevent_ev_loop_origflags(ev_loop*); + int gevent_ev_loop_sig_pending(ev_loop*); + int gevent_ev_loop_backend_fd(ev_loop*); + int gevent_ev_loop_activecnt(ev_loop*); + int gevent_ev_loop_sigfd(ev_loop*); diff --git a/src/gevent/libev/libev_vfd.h b/src/gevent/libev/libev_vfd.h new file mode 100644 index 0000000..ff30fd8 --- /dev/null +++ b/src/gevent/libev/libev_vfd.h @@ -0,0 +1,225 @@ +#ifdef _WIN32 +/* see discussion in the libuv directory: this is a SOCKET which is a + HANDLE which is a PVOID (even though they're really small ints), + and CPython and PyPy return that SOCKET cast to an int from + fileno() +*/ +typedef intptr_t vfd_socket_t; +#define vfd_socket_object PyLong_FromLongLong + +#ifdef LIBEV_EMBED +/* + * If libev on win32 is embedded, then we can use an + * arbitrary mapping between integer fds and OS + * handles. Then by defining special macros libev + * will use our functions. + */ + +#define WIN32_LEAN_AND_MEAN +#include +#include + +typedef struct vfd_entry_t +{ + vfd_socket_t handle; /* OS handle, i.e. SOCKET */ + int count; /* Reference count, 0 if free */ + int next; /* Next free fd, -1 if last */ +} vfd_entry; + +#define VFD_INCREMENT 128 +static int vfd_num = 0; /* num allocated fds */ +static int vfd_max = 0; /* max allocated fds */ +static int vfd_next = -1; /* next free fd for reuse */ +static PyObject* vfd_map = NULL; /* map OS handle -> virtual fd */ +static vfd_entry* vfd_entries = NULL; /* list of virtual fd entries */ + +#ifdef WITH_THREAD +static CRITICAL_SECTION* volatile vfd_lock = NULL; +static CRITICAL_SECTION* vfd_make_lock() +{ + if (vfd_lock == NULL) { + /* must use malloc and not PyMem_Malloc here */ + CRITICAL_SECTION* lock = malloc(sizeof(CRITICAL_SECTION)); + InitializeCriticalSection(lock); + if (InterlockedCompareExchangePointer(&vfd_lock, lock, NULL) != NULL) { + /* another thread initialized lock first */ + DeleteCriticalSection(lock); + free(lock); + } + } + return vfd_lock; +} +#define VFD_LOCK_ENTER EnterCriticalSection(vfd_make_lock()) +#define VFD_LOCK_LEAVE LeaveCriticalSection(vfd_lock) +#define VFD_GIL_DECLARE PyGILState_STATE ___save +#define VFD_GIL_ENSURE ___save = PyGILState_Ensure() +#define VFD_GIL_RELEASE PyGILState_Release(___save) +#else /* ! WITH_THREAD */ +#define VFD_LOCK_ENTER +#define VFD_LOCK_LEAVE +#define VFD_GIL_DECLARE +#define VFD_GIL_ENSURE +#define VFD_GIL_RELEASE +#endif /*_WITH_THREAD */ + +/* + * Given a virtual fd returns an OS handle or -1 + * This function is speed critical, so it cannot use GIL + */ +static vfd_socket_t vfd_get(int fd) +{ + vfd_socket_t handle = -1; + VFD_LOCK_ENTER; + if (vfd_entries != NULL && fd >= 0 && fd < vfd_num) + handle = vfd_entries[fd].handle; + VFD_LOCK_LEAVE; + return handle; +} + +#define EV_FD_TO_WIN32_HANDLE(fd) vfd_get((fd)) + +/* + * Given an OS handle finds or allocates a virtual fd + * Returns -1 on failure and sets Python exception if pyexc is non-zero + */ +static int vfd_open_(vfd_socket_t handle, int pyexc) +{ + VFD_GIL_DECLARE; + int fd = -1; + unsigned long arg; + PyObject* key = NULL; + PyObject* value; + + if (!pyexc) { + VFD_GIL_ENSURE; + } + if (ioctlsocket(handle, FIONREAD, &arg) != 0) { + if (pyexc) + PyErr_Format(PyExc_IOError, +#ifdef _WIN64 + "%lld is not a socket (files are not supported)", +#else + "%ld is not a socket (files are not supported)", +#endif + handle); + goto done; + } + if (vfd_map == NULL) { + vfd_map = PyDict_New(); + if (vfd_map == NULL) + goto done; + } + key = vfd_socket_object(handle); + /* check if it's already in the dict */ + value = PyDict_GetItem(vfd_map, key); + if (value != NULL) { + /* is it safe to use PyInt_AS_LONG(value) here? */ + fd = PyInt_AsLong(value); + if (fd >= 0) { + ++vfd_entries[fd].count; + goto done; + } + } + /* use the free entry, if available */ + if (vfd_next >= 0) { + fd = vfd_next; + vfd_next = vfd_entries[fd].next; + VFD_LOCK_ENTER; + goto allocated; + } + /* check if it would be out of bounds */ + if (vfd_num >= FD_SETSIZE) { + /* libev's select doesn't support more that FD_SETSIZE fds */ + if (pyexc) + PyErr_Format(PyExc_IOError, "cannot watch more than %d sockets", (int)FD_SETSIZE); + goto done; + } + /* allocate more space if needed */ + VFD_LOCK_ENTER; + if (vfd_num >= vfd_max) { + int newsize = vfd_max + VFD_INCREMENT; + vfd_entry* entries = PyMem_Realloc(vfd_entries, sizeof(vfd_entry) * newsize); + if (entries == NULL) { + VFD_LOCK_LEAVE; + if (pyexc) + PyErr_NoMemory(); + goto done; + } + vfd_entries = entries; + vfd_max = newsize; + } + fd = vfd_num++; +allocated: + /* vfd_lock must be acquired when entering here */ + vfd_entries[fd].handle = handle; + vfd_entries[fd].count = 1; + VFD_LOCK_LEAVE; + value = PyInt_FromLong(fd); + PyDict_SetItem(vfd_map, key, value); + Py_DECREF(value); +done: + Py_XDECREF(key); + if (!pyexc) { + VFD_GIL_RELEASE; + } + return fd; +} + +#define vfd_open(fd) vfd_open_((fd), 1) +#define EV_WIN32_HANDLE_TO_FD(handle) vfd_open_((handle), 0) + +static void vfd_free_(int fd, int needclose) +{ + VFD_GIL_DECLARE; + PyObject* key; + + if (needclose) { + VFD_GIL_ENSURE; + } + if (fd < 0 || fd >= vfd_num) + goto done; /* out of bounds */ + if (vfd_entries[fd].count <= 0) + goto done; /* free entry, ignore */ + if (!--vfd_entries[fd].count) { + /* fd has just been freed */ + vfd_socket_t handle = vfd_entries[fd].handle; + vfd_entries[fd].handle = -1; + vfd_entries[fd].next = vfd_next; + vfd_next = fd; + if (needclose) + closesocket(handle); + /* vfd_map is assumed to be != NULL */ + key = vfd_socket_object(handle); + PyDict_DelItem(vfd_map, key); + Py_DECREF(key); + } +done: + if (needclose) { + VFD_GIL_RELEASE; + } +} + +#define vfd_free(fd) vfd_free_((fd), 0) +#define EV_WIN32_CLOSE_FD(fd) vfd_free_((fd), 1) + +#else /* !LIBEV_EMBED */ +/* + * If libev on win32 is not embedded in gevent, then + * the only way to map vfds is to use the default of + * using runtime fds in libev. Note that it will leak + * fds, because there's no way of closing them safely + */ +#define vfd_get(fd) _get_osfhandle((fd)) +#define vfd_open(fd) _open_osfhandle((fd), 0) +#define vfd_free(fd) +#endif /* LIBEV_EMBED */ + +#else /* !_WIN32 */ +/* + * On non-win32 platforms vfd_* are noop macros + */ +typedef int vfd_socket_t; +#define vfd_get(fd) (fd) +#define vfd_open(fd) (fd) +#define vfd_free(fd) +#endif /* _WIN32 */ diff --git a/src/gevent/libev/stathelper.c b/src/gevent/libev/stathelper.c new file mode 100644 index 0000000..1a70b55 --- /dev/null +++ b/src/gevent/libev/stathelper.c @@ -0,0 +1,187 @@ +/* copied from Python-2.7.2/Modules/posixmodule.c */ +#include "structseq.h" + +#define STRUCT_STAT struct stat + +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE +#define ST_BLKSIZE_IDX 13 +#else +#define ST_BLKSIZE_IDX 12 +#endif + +#ifdef HAVE_STRUCT_STAT_ST_BLOCKS +#define ST_BLOCKS_IDX (ST_BLKSIZE_IDX+1) +#else +#define ST_BLOCKS_IDX ST_BLKSIZE_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_RDEV +#define ST_RDEV_IDX (ST_BLOCKS_IDX+1) +#else +#define ST_RDEV_IDX ST_BLOCKS_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_FLAGS +#define ST_FLAGS_IDX (ST_RDEV_IDX+1) +#else +#define ST_FLAGS_IDX ST_RDEV_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_GEN +#define ST_GEN_IDX (ST_FLAGS_IDX+1) +#else +#define ST_GEN_IDX ST_FLAGS_IDX +#endif + +#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME +#define ST_BIRTHTIME_IDX (ST_GEN_IDX+1) +#else +#define ST_BIRTHTIME_IDX ST_GEN_IDX +#endif + + + +static PyObject* posixmodule = NULL; +static PyTypeObject* pStatResultType = NULL; + + +static PyObject* import_posixmodule(void) +{ + if (!posixmodule) { + posixmodule = PyImport_ImportModule("posix"); + } + return posixmodule; +} + + +static PyObject* import_StatResultType(void) +{ + PyObject* p = NULL; + if (!pStatResultType) { + PyObject* module; + module = import_posixmodule(); + if (module) { + p = PyObject_GetAttrString(module, "stat_result"); + } + } + return p; +} + +static void +fill_time(PyObject *v, int index, time_t sec, unsigned long nsec) +{ + PyObject *fval,*ival; +#if SIZEOF_TIME_T > SIZEOF_LONG + ival = PyLong_FromLongLong((PY_LONG_LONG)sec); +#else + ival = PyInt_FromLong((long)sec); +#endif + if (!ival) + return; + fval = PyFloat_FromDouble(sec + 1e-9*nsec); + PyStructSequence_SET_ITEM(v, index, ival); + PyStructSequence_SET_ITEM(v, index+3, fval); +} + +/* pack a system stat C structure into the Python stat tuple + (used by posix_stat() and posix_fstat()) */ +static PyObject* +_pystat_fromstructstat(STRUCT_STAT *st) +{ + unsigned long ansec, mnsec, cnsec; + PyObject *v; + + PyTypeObject* StatResultType = (PyTypeObject*)import_StatResultType(); + if (StatResultType == NULL) { + return NULL; + } + + v = PyStructSequence_New(StatResultType); + if (v == NULL) + return NULL; + + PyStructSequence_SET_ITEM(v, 0, PyInt_FromLong((long)st->st_mode)); +#ifdef HAVE_LARGEFILE_SUPPORT + PyStructSequence_SET_ITEM(v, 1, + PyLong_FromLongLong((PY_LONG_LONG)st->st_ino)); +#else + PyStructSequence_SET_ITEM(v, 1, PyInt_FromLong((long)st->st_ino)); +#endif +#if defined(HAVE_LONG_LONG) && !defined(MS_WINDOWS) + PyStructSequence_SET_ITEM(v, 2, + PyLong_FromLongLong((PY_LONG_LONG)st->st_dev)); +#else + PyStructSequence_SET_ITEM(v, 2, PyInt_FromLong((long)st->st_dev)); +#endif + PyStructSequence_SET_ITEM(v, 3, PyInt_FromLong((long)st->st_nlink)); + PyStructSequence_SET_ITEM(v, 4, PyInt_FromLong((long)st->st_uid)); + PyStructSequence_SET_ITEM(v, 5, PyInt_FromLong((long)st->st_gid)); +#ifdef HAVE_LARGEFILE_SUPPORT + PyStructSequence_SET_ITEM(v, 6, + PyLong_FromLongLong((PY_LONG_LONG)st->st_size)); +#else + PyStructSequence_SET_ITEM(v, 6, PyInt_FromLong(st->st_size)); +#endif + +#if defined(HAVE_STAT_TV_NSEC) + ansec = st->st_atim.tv_nsec; + mnsec = st->st_mtim.tv_nsec; + cnsec = st->st_ctim.tv_nsec; +#elif defined(HAVE_STAT_TV_NSEC2) + ansec = st->st_atimespec.tv_nsec; + mnsec = st->st_mtimespec.tv_nsec; + cnsec = st->st_ctimespec.tv_nsec; +#elif defined(HAVE_STAT_NSEC) + ansec = st->st_atime_nsec; + mnsec = st->st_mtime_nsec; + cnsec = st->st_ctime_nsec; +#else + ansec = mnsec = cnsec = 0; +#endif + fill_time(v, 7, st->st_atime, ansec); + fill_time(v, 8, st->st_mtime, mnsec); + fill_time(v, 9, st->st_ctime, cnsec); + +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE + PyStructSequence_SET_ITEM(v, ST_BLKSIZE_IDX, + PyInt_FromLong((long)st->st_blksize)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_BLOCKS + PyStructSequence_SET_ITEM(v, ST_BLOCKS_IDX, + PyInt_FromLong((long)st->st_blocks)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_RDEV + PyStructSequence_SET_ITEM(v, ST_RDEV_IDX, + PyInt_FromLong((long)st->st_rdev)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_GEN + PyStructSequence_SET_ITEM(v, ST_GEN_IDX, + PyInt_FromLong((long)st->st_gen)); +#endif +#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME + { + PyObject *val; + unsigned long bsec,bnsec; + bsec = (long)st->st_birthtime; +#ifdef HAVE_STAT_TV_NSEC2 + bnsec = st->st_birthtimespec.tv_nsec; +#else + bnsec = 0; +#endif + val = PyFloat_FromDouble(bsec + 1e-9*bnsec); + PyStructSequence_SET_ITEM(v, ST_BIRTHTIME_IDX, + val); + } +#endif +#ifdef HAVE_STRUCT_STAT_ST_FLAGS + PyStructSequence_SET_ITEM(v, ST_FLAGS_IDX, + PyInt_FromLong((long)st->st_flags)); +#endif + + if (PyErr_Occurred()) { + Py_DECREF(v); + return NULL; + } + + return v; +} diff --git a/src/gevent/libev/watcher.py b/src/gevent/libev/watcher.py new file mode 100644 index 0000000..74db0c1 --- /dev/null +++ b/src/gevent/libev/watcher.py @@ -0,0 +1,286 @@ +# pylint: disable=too-many-lines, protected-access, redefined-outer-name, not-callable +# pylint: disable=no-member +from __future__ import absolute_import, print_function +import sys + +from gevent.libev import _corecffi # pylint:disable=no-name-in-module,import-error + +# Nothing public here +__all__ = [] + + +ffi = _corecffi.ffi # pylint:disable=no-member +libev = _corecffi.lib # pylint:disable=no-member + +if hasattr(libev, 'vfd_open'): + # Must be on windows + assert sys.platform.startswith("win"), "vfd functions only needed on windows" + vfd_open = libev.vfd_open + vfd_free = libev.vfd_free + vfd_get = libev.vfd_get +else: + vfd_open = vfd_free = vfd_get = lambda fd: fd + +##### +## NOTE on Windows: +# The C implementation does several things specially for Windows; +# a possibly incomplete list is: +# +# - the loop runs a periodic signal checker; +# - the io watcher constructor is different and it has a destructor; +# - the child watcher is not defined +# +# The CFFI implementation does none of these things, and so +# is possibly NOT FUNCTIONALLY CORRECT on Win32 +##### +_NOARGS = () +_events = [(libev.EV_READ, 'READ'), + (libev.EV_WRITE, 'WRITE'), + (libev.EV__IOFDSET, '_IOFDSET'), + (libev.EV_PERIODIC, 'PERIODIC'), + (libev.EV_SIGNAL, 'SIGNAL'), + (libev.EV_CHILD, 'CHILD'), + (libev.EV_STAT, 'STAT'), + (libev.EV_IDLE, 'IDLE'), + (libev.EV_PREPARE, 'PREPARE'), + (libev.EV_CHECK, 'CHECK'), + (libev.EV_EMBED, 'EMBED'), + (libev.EV_FORK, 'FORK'), + (libev.EV_CLEANUP, 'CLEANUP'), + (libev.EV_ASYNC, 'ASYNC'), + (libev.EV_CUSTOM, 'CUSTOM'), + (libev.EV_ERROR, 'ERROR')] + +from gevent._ffi import watcher as _base + +def _events_to_str(events): + return _base.events_to_str(events, _events) + + + +class watcher(_base.watcher): + _FFI = ffi + _LIB = libev + _watcher_prefix = 'ev' + + # Flags is a bitfield with the following meaning: + # 0000 -> default, referenced (when active) + # 0010 -> ev_unref has been called + # 0100 -> not referenced; independent of 0010 + _flags = 0 + + def __init__(self, _loop, ref=True, priority=None, args=_base._NOARGS): + if ref: + self._flags = 0 + else: + self._flags = 4 + + super(watcher, self).__init__(_loop, ref=ref, priority=priority, args=args) + + def _watcher_ffi_set_priority(self, priority): + libev.ev_set_priority(self._watcher, priority) + + def _watcher_ffi_init(self, args): + self._watcher_init(self._watcher, + self._watcher_callback, + *args) + + def _watcher_ffi_start(self): + self._watcher_start(self.loop._ptr, self._watcher) + + def _watcher_ffi_ref(self): + if self._flags & 2: # we've told libev we're not referenced + self.loop.ref() + self._flags &= ~2 + + def _watcher_ffi_unref(self): + if self._flags & 6 == 4: + # We're not referenced, but we haven't told libev that + self.loop.unref() + self._flags |= 2 # now we've told libev + + def _get_ref(self): + return not self._flags & 4 + + def _set_ref(self, value): + if value: + if not self._flags & 4: + return # ref is already True + if self._flags & 2: # ev_unref was called, undo + self.loop.ref() + self._flags &= ~6 # do not want unref, no outstanding unref + else: + if self._flags & 4: + return # ref is already False + self._flags |= 4 # we're not referenced + if not self._flags & 2 and libev.ev_is_active(self._watcher): + # we haven't told libev we're not referenced, but it thinks we're + # active so we need to undo that + self.loop.unref() + self._flags |= 2 # libev knows we're not referenced + + ref = property(_get_ref, _set_ref) + + + def _get_priority(self): + return libev.ev_priority(self._watcher) + + @_base.not_while_active + def _set_priority(self, priority): + libev.ev_set_priority(self._watcher, priority) + + priority = property(_get_priority, _set_priority) + + def feed(self, revents, callback, *args): + self.callback = callback + self.args = args or _NOARGS + if self._flags & 6 == 4: + self.loop.unref() + self._flags |= 2 + libev.ev_feed_event(self.loop._ptr, self._watcher, revents) + if not self._flags & 1: + # Py_INCREF(self) + self._flags |= 1 + + @property + def pending(self): + return bool(self._watcher and libev.ev_is_pending(self._watcher)) + + +class io(_base.IoMixin, watcher): + + EVENT_MASK = libev.EV__IOFDSET | libev.EV_READ | libev.EV_WRITE + + def _get_fd(self): + return vfd_get(self._watcher.fd) + + @_base.not_while_active + def _set_fd(self, fd): + vfd = vfd_open(fd) + vfd_free(self._watcher.fd) + self._watcher_init(self._watcher, self._watcher_callback, vfd, self._watcher.events) + + fd = property(_get_fd, _set_fd) + + def _get_events(self): + return self._watcher.events + + @_base.not_while_active + def _set_events(self, events): + self._watcher_init(self._watcher, self._watcher_callback, self._watcher.fd, events) + + events = property(_get_events, _set_events) + + @property + def events_str(self): + return _events_to_str(self._watcher.events) + + def _format(self): + return ' fd=%s events=%s' % (self.fd, self.events_str) + + +class timer(_base.TimerMixin, watcher): + + @property + def at(self): + return self._watcher.at + + def again(self, callback, *args, **kw): + # Exactly the same as start(), just with a different initializer + # function + self._watcher_start = libev.ev_timer_again + try: + self.start(callback, *args, **kw) + finally: + del self._watcher_start + + +class signal(_base.SignalMixin, watcher): + pass + +class idle(_base.IdleMixin, watcher): + pass + +class prepare(_base.PrepareMixin, watcher): + pass + +class check(_base.CheckMixin, watcher): + pass + +class fork(_base.ForkMixin, watcher): + pass + + +class async_(_base.AsyncMixin, watcher): + + def send(self): + libev.ev_async_send(self.loop._ptr, self._watcher) + + @property + def pending(self): + return bool(libev.ev_async_pending(self._watcher)) + +# Provide BWC for those that have async +locals()['async'] = async_ + +class _ClosedWatcher(object): + __slots__ = ('pid', 'rpid', 'rstatus') + + def __init__(self, other): + self.pid = other.pid + self.rpid = other.rpid + self.rstatus = other.rstatus + + def __bool__(self): + return False + __nonzero__ = __bool__ + +class child(_base.ChildMixin, watcher): + _watcher_type = 'child' + + def close(self): + # Capture the properties we defer to our _watcher, because + # we're about to discard it. + closed_watcher = _ClosedWatcher(self._watcher) + super(child, self).close() + self._watcher = closed_watcher + + @property + def pid(self): + return self._watcher.pid + + @property + def rpid(self): + return self._watcher.rpid + + @rpid.setter + def rpid(self, value): + self._watcher.rpid = value + + @property + def rstatus(self): + return self._watcher.rstatus + + @rstatus.setter + def rstatus(self, value): + self._watcher.rstatus = value + + +class stat(_base.StatMixin, watcher): + _watcher_type = 'stat' + + @property + def attr(self): + if not self._watcher.attr.st_nlink: + return + return self._watcher.attr + + @property + def prev(self): + if not self._watcher.prev.st_nlink: + return + return self._watcher.prev + + @property + def interval(self): + return self._watcher.interval diff --git a/src/gevent/libuv/__init__.py b/src/gevent/libuv/__init__.py new file mode 100644 index 0000000..412d64c --- /dev/null +++ b/src/gevent/libuv/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# Nothing public here +__all__ = [] diff --git a/src/gevent/libuv/_corecffi_build.py b/src/gevent/libuv/_corecffi_build.py new file mode 100644 index 0000000..5194d09 --- /dev/null +++ b/src/gevent/libuv/_corecffi_build.py @@ -0,0 +1,256 @@ +# pylint: disable=no-member + +# This module is only used to create and compile the gevent._corecffi module; +# nothing should be directly imported from it except `ffi`, which should only be +# used for `ffi.compile()`; programs should import gevent._corecfffi. +# However, because we are using "out-of-line" mode, it is necessary to examine +# this file to know what functions are created and available on the generated +# module. +from __future__ import absolute_import, print_function +import sys +import os +import os.path # pylint:disable=no-name-in-module +import struct + +__all__ = [] + +WIN = sys.platform.startswith('win32') + +def system_bits(): + return struct.calcsize('P') * 8 + + +def st_nlink_type(): + if sys.platform == "darwin" or sys.platform.startswith("freebsd"): + return "short" + if system_bits() == 32: + return "unsigned long" + return "long long" + + +from cffi import FFI +ffi = FFI() + +thisdir = os.path.dirname(os.path.abspath(__file__)) +def read_source(name): + with open(os.path.join(thisdir, name), 'r') as f: + return f.read() + +_cdef = read_source('_corecffi_cdef.c') +_source = read_source('_corecffi_source.c') + +_cdef = _cdef.replace('#define GEVENT_ST_NLINK_T int', '') +_cdef = _cdef.replace('#define GEVENT_STRUCT_DONE int', '') +_cdef = _cdef.replace('#define GEVENT_UV_OS_SOCK_T int', '') + +_cdef = _cdef.replace('GEVENT_ST_NLINK_T', st_nlink_type()) +_cdef = _cdef.replace("GEVENT_STRUCT_DONE _;", '...;') +# uv_os_sock_t is int on POSIX and SOCKET on Win32, but socket is +# just another name for handle, which is just another name for 'void*' +# which we will treat as an 'unsigned long' or 'unsigned long long' +# since it comes through 'fileno()' where it has been cast as an int. +# See class watcher.io +_void_pointer_as_integer = 'intptr_t' +_cdef = _cdef.replace("GEVENT_UV_OS_SOCK_T", 'int' if not WIN else _void_pointer_as_integer) + + +setup_py_dir = os.path.abspath(os.path.join(thisdir, '..', '..', '..')) +libuv_dir = os.path.abspath(os.path.join(setup_py_dir, 'deps', 'libuv')) + + +LIBUV_INCLUDE_DIRS = [ + thisdir, # libev_vfd.h + os.path.join(libuv_dir, 'include'), + os.path.join(libuv_dir, 'src'), +] + +# Initially based on https://github.com/saghul/pyuv/blob/v1.x/setup_libuv.py + +def _libuv_source(rel_path): + # Certain versions of setuptools, notably on windows, are *very* + # picky about what we feed to sources= "setup() arguments must + # *always* be /-separated paths relative to the setup.py + # directory, *never* absolute paths." POSIX doesn't have that issue. + path = os.path.join('deps', 'libuv', 'src', rel_path) + return path + +LIBUV_SOURCES = [ + _libuv_source('fs-poll.c'), + _libuv_source('inet.c'), + _libuv_source('threadpool.c'), + _libuv_source('uv-common.c'), + _libuv_source('version.c'), + _libuv_source('uv-data-getter-setters.c'), + _libuv_source('timer.c'), + _libuv_source('idna.c'), +] + +if WIN: + LIBUV_SOURCES += [ + _libuv_source('win/async.c'), + _libuv_source('win/core.c'), + _libuv_source('win/detect-wakeup.c'), + _libuv_source('win/dl.c'), + _libuv_source('win/error.c'), + _libuv_source('win/fs-event.c'), + _libuv_source('win/fs.c'), + # getaddrinfo.c refers to ConvertInterfaceIndexToLuid + # and ConvertInterfaceLuidToNameA, which are supposedly in iphlpapi.h + # and iphlpapi.lib/dll. But on Windows 10 with Python 3.5 and VC 14 (Visual Studio 2015), + # I get an undefined warning from the compiler for those functions and + # a link error from the linker, so this file can't be included. + # This is possibly because the functions are defined for Windows Vista, and + # Python 3.5 builds with at earlier SDK? + # Fortunately we don't use those functions. + #_libuv_source('win/getaddrinfo.c'), + # getnameinfo.c refers to uv__getaddrinfo_translate_error from + # getaddrinfo.c, which we don't have. + #_libuv_source('win/getnameinfo.c'), + _libuv_source('win/handle.c'), + _libuv_source('win/loop-watcher.c'), + _libuv_source('win/pipe.c'), + _libuv_source('win/poll.c'), + _libuv_source('win/process-stdio.c'), + _libuv_source('win/process.c'), + _libuv_source('win/signal.c'), + _libuv_source('win/snprintf.c'), + _libuv_source('win/stream.c'), + _libuv_source('win/tcp.c'), + _libuv_source('win/thread.c'), + _libuv_source('win/tty.c'), + _libuv_source('win/udp.c'), + _libuv_source('win/util.c'), + _libuv_source('win/winapi.c'), + _libuv_source('win/winsock.c'), + ] +else: + LIBUV_SOURCES += [ + _libuv_source('unix/async.c'), + _libuv_source('unix/core.c'), + _libuv_source('unix/dl.c'), + _libuv_source('unix/fs.c'), + _libuv_source('unix/getaddrinfo.c'), + _libuv_source('unix/getnameinfo.c'), + _libuv_source('unix/loop-watcher.c'), + _libuv_source('unix/loop.c'), + _libuv_source('unix/pipe.c'), + _libuv_source('unix/poll.c'), + _libuv_source('unix/process.c'), + _libuv_source('unix/signal.c'), + _libuv_source('unix/stream.c'), + _libuv_source('unix/tcp.c'), + _libuv_source('unix/thread.c'), + _libuv_source('unix/tty.c'), + _libuv_source('unix/udp.c'), + ] + + +if sys.platform.startswith('linux'): + LIBUV_SOURCES += [ + _libuv_source('unix/linux-core.c'), + _libuv_source('unix/linux-inotify.c'), + _libuv_source('unix/linux-syscalls.c'), + _libuv_source('unix/procfs-exepath.c'), + _libuv_source('unix/proctitle.c'), + _libuv_source('unix/sysinfo-loadavg.c'), + _libuv_source('unix/sysinfo-memory.c'), + ] +elif sys.platform == 'darwin': + LIBUV_SOURCES += [ + _libuv_source('unix/bsd-ifaddrs.c'), + _libuv_source('unix/darwin.c'), + _libuv_source('unix/darwin-proctitle.c'), + _libuv_source('unix/fsevents.c'), + _libuv_source('unix/kqueue.c'), + _libuv_source('unix/proctitle.c'), + ] +elif sys.platform.startswith(('freebsd', 'dragonfly')): + LIBUV_SOURCES += [ + _libuv_source('unix/bsd-ifaddrs.c'), + _libuv_source('unix/freebsd.c'), + _libuv_source('unix/kqueue.c'), + _libuv_source('unix/posix-hrtime.c'), + _libuv_source('unix/bsd-proctitle.c'), + ] +elif sys.platform.startswith('openbsd'): + LIBUV_SOURCES += [ + _libuv_source('unix/bsd-ifaddrs.c'), + _libuv_source('unix/kqueue.c'), + _libuv_source('unix/openbsd.c'), + _libuv_source('unix/posix-hrtime.c'), + _libuv_source('unix/bsd-proctitle.c'), + ] +elif sys.platform.startswith('netbsd'): + LIBUV_SOURCES += [ + _libuv_source('unix/bsd-ifaddrs.c'), + _libuv_source('unix/kqueue.c'), + _libuv_source('unix/netbsd.c'), + _libuv_source('unix/posix-hrtime.c'), + _libuv_source('unix/bsd-proctitle.c'), + ] + +elif sys.platform.startswith('sunos'): + LIBUV_SOURCES += [ + _libuv_source('unix/no-proctitle.c'), + _libuv_source('unix/sunos.c'), + ] + + +LIBUV_MACROS = [] + +def _define_macro(name, value): + LIBUV_MACROS.append((name, value)) + +LIBUV_LIBRARIES = [] + +def _add_library(name): + LIBUV_LIBRARIES.append(name) + +if sys.platform != 'win32': + _define_macro('_LARGEFILE_SOURCE', 1) + _define_macro('_FILE_OFFSET_BITS', 64) + +if sys.platform.startswith('linux'): + _add_library('dl') + _add_library('rt') + _define_macro('_GNU_SOURCE', 1) + _define_macro('_POSIX_C_SOURCE', '200112') +elif sys.platform == 'darwin': + _define_macro('_DARWIN_USE_64_BIT_INODE', 1) + _define_macro('_DARWIN_UNLIMITED_SELECT', 1) +elif sys.platform.startswith('netbsd'): + _add_library('kvm') +elif sys.platform.startswith('sunos'): + _define_macro('__EXTENSIONS__', 1) + _define_macro('_XOPEN_SOURCE', 500) + _add_library('kstat') + _add_library('nsl') + _add_library('sendfile') + _add_library('socket') +elif WIN: + _define_macro('_GNU_SOURCE', 1) + _define_macro('WIN32', 1) + _define_macro('_CRT_SECURE_NO_DEPRECATE', 1) + _define_macro('_CRT_NONSTDC_NO_DEPRECATE', 1) + _define_macro('_CRT_SECURE_NO_WARNINGS', 1) + _define_macro('_WIN32_WINNT', '0x0600') + _define_macro('WIN32_LEAN_AND_MEAN', 1) + _add_library('advapi32') + _add_library('iphlpapi') + _add_library('psapi') + _add_library('shell32') + _add_library('user32') + _add_library('userenv') + _add_library('ws2_32') + +ffi.cdef(_cdef) +ffi.set_source('gevent.libuv._corecffi', + _source, + sources=LIBUV_SOURCES, + depends=LIBUV_SOURCES, + include_dirs=LIBUV_INCLUDE_DIRS, + libraries=list(LIBUV_LIBRARIES), + define_macros=list(LIBUV_MACROS)) + +if __name__ == '__main__': + ffi.compile() diff --git a/src/gevent/libuv/_corecffi_cdef.c b/src/gevent/libuv/_corecffi_cdef.c new file mode 100644 index 0000000..0735aea --- /dev/null +++ b/src/gevent/libuv/_corecffi_cdef.c @@ -0,0 +1,393 @@ +/* markers for the CFFI parser. Replaced when the string is read. */ +#define GEVENT_STRUCT_DONE int +#define GEVENT_ST_NLINK_T int +#define GEVENT_UV_OS_SOCK_T int + +#define UV_EBUSY ... + +#define UV_VERSION_MAJOR ... +#define UV_VERSION_MINOR ... +#define UV_VERSION_PATCH ... + +typedef enum { + UV_RUN_DEFAULT = 0, + UV_RUN_ONCE, + UV_RUN_NOWAIT +} uv_run_mode; + +typedef enum { + UV_UNKNOWN_HANDLE = 0, + UV_ASYNC, + UV_CHECK, + UV_FS_EVENT, + UV_FS_POLL, + UV_HANDLE, + UV_IDLE, + UV_NAMED_PIPE, + UV_POLL, + UV_PREPARE, + UV_PROCESS, + UV_STREAM, + UV_TCP, + UV_TIMER, + UV_TTY, + UV_UDP, + UV_SIGNAL, + UV_FILE, + UV_HANDLE_TYPE_MAX +} uv_handle_type; + +enum uv_poll_event { + UV_READABLE = 1, + UV_WRITABLE = 2, + /* new in 1.9 */ + UV_DISCONNECT = 4, + /* new in 1.14.0 */ + UV_PRIORITIZED = 8, +}; + +enum uv_fs_event { + UV_RENAME = 1, + UV_CHANGE = 2 +}; + +enum uv_fs_event_flags { + /* + * By default, if the fs event watcher is given a directory name, we will + * watch for all events in that directory. This flags overrides this behavior + * and makes fs_event report only changes to the directory entry itself. This + * flag does not affect individual files watched. + * This flag is currently not implemented yet on any backend. + */ + UV_FS_EVENT_WATCH_ENTRY = 1, + /* + * By default uv_fs_event will try to use a kernel interface such as inotify + * or kqueue to detect events. This may not work on remote filesystems such + * as NFS mounts. This flag makes fs_event fall back to calling stat() on a + * regular interval. + * This flag is currently not implemented yet on any backend. + */ + UV_FS_EVENT_STAT = 2, + /* + * By default, event watcher, when watching directory, is not registering + * (is ignoring) changes in it's subdirectories. + * This flag will override this behaviour on platforms that support it. + */ + UV_FS_EVENT_RECURSIVE = 4 +}; + +const char* uv_strerror(int); +const char* uv_err_name(int); +const char* uv_version_string(void); +const char* uv_handle_type_name(uv_handle_type type); + +// handle structs and types +struct uv_loop_s { + void* data; + GEVENT_STRUCT_DONE _; +}; +struct uv_handle_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; +struct uv_idle_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; +struct uv_prepare_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; +struct uv_timer_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; +struct uv_signal_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; +struct uv_poll_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; + +struct uv_check_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; + +struct uv_async_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + void (*async_cb)(struct uv_async_s *); + GEVENT_STRUCT_DONE _; +}; + +struct uv_fs_event_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; + +struct uv_fs_poll_s { + struct uv_loop_s* loop; + uv_handle_type type; + void *data; + GEVENT_STRUCT_DONE _; +}; + +typedef struct uv_loop_s uv_loop_t; +typedef struct uv_handle_s uv_handle_t; +typedef struct uv_idle_s uv_idle_t; +typedef struct uv_prepare_s uv_prepare_t; +typedef struct uv_timer_s uv_timer_t; +typedef struct uv_signal_s uv_signal_t; +typedef struct uv_poll_s uv_poll_t; +typedef struct uv_check_s uv_check_t; +typedef struct uv_async_s uv_async_t; +typedef struct uv_fs_event_s uv_fs_event_t; +typedef struct uv_fs_poll_s uv_fs_poll_t; + + +size_t uv_handle_size(uv_handle_type); + +// callbacks with the same signature +typedef void (*uv_close_cb)(uv_handle_t *handle); +typedef void (*uv_idle_cb)(uv_idle_t *handle); +typedef void (*uv_timer_cb)(uv_timer_t *handle); +typedef void (*uv_check_cb)(uv_check_t* handle); +typedef void (*uv_async_cb)(uv_async_t* handle); +typedef void (*uv_prepare_cb)(uv_prepare_t *handle); + +// callbacks with distinct sigs +typedef void (*uv_walk_cb)(uv_handle_t *handle, void *arg); +typedef void (*uv_poll_cb)(uv_poll_t *handle, int status, int events); +typedef void (*uv_signal_cb)(uv_signal_t *handle, int signum); + +// Callback passed to uv_fs_event_start() which will be called +// repeatedly after the handle is started. If the handle was started +// with a directory the filename parameter will be a relative path to +// a file contained in the directory. The events parameter is an ORed +// mask of uv_fs_event elements. +typedef void (*uv_fs_event_cb)(uv_fs_event_t* handle, const char* filename, int events, int status); + +typedef struct { + long tv_sec; + long tv_nsec; +} uv_timespec_t; + +typedef struct { + uint64_t st_dev; + uint64_t st_mode; + uint64_t st_nlink; + uint64_t st_uid; + uint64_t st_gid; + uint64_t st_rdev; + uint64_t st_ino; + uint64_t st_size; + uint64_t st_blksize; + uint64_t st_blocks; + uint64_t st_flags; + uint64_t st_gen; + uv_timespec_t st_atim; + uv_timespec_t st_mtim; + uv_timespec_t st_ctim; + uv_timespec_t st_birthtim; +} uv_stat_t; + +typedef void (*uv_fs_poll_cb)(uv_fs_poll_t* handle, int status, const uv_stat_t* prev, const uv_stat_t* curr); + +// loop functions +uv_loop_t *uv_default_loop(); +uv_loop_t* uv_loop_new(); // not documented; neither is uv_loop_delete +int uv_loop_init(uv_loop_t* loop); +int uv_loop_fork(uv_loop_t* loop); +int uv_loop_alive(const uv_loop_t *loop); +int uv_loop_close(uv_loop_t* loop); +uint64_t uv_backend_timeout(uv_loop_t* loop); +int uv_run(uv_loop_t *, uv_run_mode mode); +int uv_backend_fd(const uv_loop_t* loop); +// The narrative docs for the two time functions say 'const', +// but the header does not. +void uv_update_time(uv_loop_t* loop); +uint64_t uv_now(uv_loop_t* loop); +void uv_stop(uv_loop_t *); +void uv_walk(uv_loop_t *loop, uv_walk_cb walk_cb, void *arg); + +// handle functions +// uv_handle_t is the base type for all libuv handle types. + +void uv_ref(void *); +void uv_unref(void *); +int uv_has_ref(void *); +void uv_close(void *handle, uv_close_cb close_cb); +int uv_is_active(void *handle); +int uv_is_closing(void *handle); + +// idle functions +// Idle handles will run the given callback once per loop iteration, right +// before the uv_prepare_t handles. Note: The notable difference with prepare +// handles is that when there are active idle handles, the loop will perform a +// zero timeout poll instead of blocking for i/o. Warning: Despite the name, +// idle handles will get their callbacks called on every loop iteration, not +// when the loop is actually "idle". +int uv_idle_init(uv_loop_t *, uv_idle_t *idle); +int uv_idle_start(uv_idle_t *idle, uv_idle_cb cb); +int uv_idle_stop(uv_idle_t *idle); + +// prepare functions +// Prepare handles will run the given callback once per loop iteration, right +// before polling for i/o. +int uv_prepare_init(uv_loop_t *, uv_prepare_t *prepare); +int uv_prepare_start(uv_prepare_t *prepare, uv_prepare_cb cb); +int uv_prepare_stop(uv_prepare_t *prepare); + +// check functions +// Check handles will run the given callback once per loop iteration, right +int uv_check_init(uv_loop_t *, uv_check_t *check); +int uv_check_start(uv_check_t *check, uv_check_cb cb); +int uv_check_stop(uv_check_t *check); + +// async functions +// Async handles allow the user to "wakeup" the event loop and get a callback called from another thread. + +int uv_async_init(uv_loop_t *, uv_async_t*, uv_async_cb); +int uv_async_send(uv_async_t*); + +// timer functions +// Timer handles are used to schedule callbacks to be called in the future. +int uv_timer_init(uv_loop_t *, uv_timer_t *handle); +int uv_timer_start(uv_timer_t *handle, uv_timer_cb cb, uint64_t timeout, uint64_t repeat); +int uv_timer_stop(uv_timer_t *handle); +int uv_timer_again(uv_timer_t *handle); +void uv_timer_set_repeat(uv_timer_t *handle, uint64_t repeat); +uint64_t uv_timer_get_repeat(const uv_timer_t *handle); + +// signal functions +// Signal handles implement Unix style signal handling on a per-event loop +// bases. +int uv_signal_init(uv_loop_t *loop, uv_signal_t *handle); +int uv_signal_start(uv_signal_t *handle, uv_signal_cb signal_cb, int signum); +int uv_signal_stop(uv_signal_t *handle); + +// poll functions Poll handles are used to watch file descriptors for +// readability and writability, similar to the purpose of poll(2). It +// is not okay to have multiple active poll handles for the same +// socket, this can cause libuv to busyloop or otherwise malfunction. +// +// The purpose of poll handles is to enable integrating external +// libraries that rely on the event loop to signal it about the socket +// status changes, like c-ares or libssh2. Using uv_poll_t for any +// other purpose is not recommended; uv_tcp_t, uv_udp_t, etc. provide +// an implementation that is faster and more scalable than what can be +// achieved with uv_poll_t, especially on Windows. +// +// Note On windows only sockets can be polled with poll handles. On +// Unix any file descriptor that would be accepted by poll(2) can be +// used. +int uv_poll_init(uv_loop_t *loop, uv_poll_t *handle, int fd); + +// Initialize the handle using a socket descriptor. On Unix this is +// identical to uv_poll_init(). On windows it takes a SOCKET handle; +// SOCKET handles are another name for HANDLE objects in win32, and +// those are defined as PVOID, even though they are not actually +// pointers (they're small integers). CPython and PyPy both return +// the SOCKET (as cast to an int) from the socket.fileno() method. +// libuv uses ``uv_os_sock_t`` for this type, which is defined as an +// int on unix. +int uv_poll_init_socket(uv_loop_t* loop, uv_poll_t* handle, GEVENT_UV_OS_SOCK_T socket); +int uv_poll_start(uv_poll_t *handle, int events, uv_poll_cb cb); +int uv_poll_stop(uv_poll_t *handle); + +// FS Event handles allow the user to monitor a given path for +// changes, for example, if the file was renamed or there was a +// generic change in it. This handle uses the best backend for the job +// on each platform. +// +// Thereas also uv_fs_poll_t that uses stat for filesystems where +// the kernel event isn't available. +int uv_fs_event_init(uv_loop_t*, uv_fs_event_t*); +int uv_fs_event_start(uv_fs_event_t*, uv_fs_event_cb, const char* path, unsigned int flags); +int uv_fs_event_stop(uv_fs_event_t*); +int uv_fs_event_getpath(uv_fs_event_t*, char* buffer, size_t* size); + +// FS Poll handles allow the user to monitor a given path for changes. +// Unlike uv_fs_event_t, fs poll handles use stat to detect when a +// file has changed so they can work on file systems where fs event +// handles can't. +// +// This is a closer match to libev. +int uv_fs_poll_init(void*, void*); +int uv_fs_poll_start(void*, uv_fs_poll_cb, const char* path, unsigned int); +int uv_fs_poll_stop(void*); + + +/* Standard library */ +void* memset(void *b, int c, size_t len); + + +/* gevent callbacks */ +// Implemented in Python code as 'def_extern'. In the case of poll callbacks and fs +// callbacks, if *status* is less than 0, it will be passed in the revents +// field. In cases of no extra arguments, revents will be 0. +// These will be created as static functions at the end of the +// _source.c and must be pre-declared at the top of that file if we +// call them +typedef void* GeventWatcherObject; +extern "Python" { + // Standard gevent._ffi.loop callbacks. + int python_callback(GeventWatcherObject handle, int revents); + void python_handle_error(GeventWatcherObject handle, int revents); + void python_stop(GeventWatcherObject handle); + + void python_check_callback(uv_check_t* handle); + void python_prepare_callback(uv_prepare_t* handle); + void python_timer0_callback(uv_check_t* handle); + + // libuv specific callback + void _uv_close_callback(uv_handle_t* handle); + void python_sigchld_callback(uv_signal_t* handle, int signum); + void python_queue_callback(uv_handle_t* handle, int revents); +} +// A variable we fill in. +static void (*gevent_noop)(void* handle); + +static void _gevent_signal_callback1(uv_signal_t* handle, int arg); +static void _gevent_async_callback0(uv_async_t* handle); +static void _gevent_prepare_callback0(uv_prepare_t* handle); +static void _gevent_timer_callback0(uv_timer_t* handle); +static void _gevent_check_callback0(uv_check_t* handle); +static void _gevent_idle_callback0(uv_idle_t* handle); +static void _gevent_poll_callback2(uv_poll_t* handle, int status, int events); +static void _gevent_fs_event_callback3(uv_fs_event_t* handle, const char* filename, int events, int status); + +typedef struct _gevent_fs_poll_s { + uv_fs_poll_t handle; + uv_stat_t curr; + uv_stat_t prev; +} gevent_fs_poll_t; + +static void _gevent_fs_poll_callback3(uv_fs_poll_t* handle, int status, const uv_stat_t* prev, const uv_stat_t* curr); + +static void gevent_uv_walk_callback_close(uv_handle_t* handle, void* arg); +static void gevent_close_all_handles(uv_loop_t* loop); +static void gevent_zero_timer(uv_timer_t* handle); +static void gevent_zero_prepare(uv_prepare_t* handle); +static void gevent_zero_check(uv_check_t* handle); +static void gevent_zero_loop(uv_loop_t* handle); diff --git a/src/gevent/libuv/_corecffi_source.c b/src/gevent/libuv/_corecffi_source.c new file mode 100644 index 0000000..83fe82e --- /dev/null +++ b/src/gevent/libuv/_corecffi_source.c @@ -0,0 +1,181 @@ +#include +#include "uv.h" + +typedef void* GeventWatcherObject; + +static int python_callback(GeventWatcherObject handle, int revents); +static void python_queue_callback(uv_handle_t* watcher_ptr, int revents); +static void python_handle_error(GeventWatcherObject handle, int revents); +static void python_stop(GeventWatcherObject handle); + +static void _gevent_noop(void* handle) {} + +static void (*gevent_noop)(void* handle) = &_gevent_noop; + +static void _gevent_generic_callback1_unused(uv_handle_t* watcher, int arg) +{ + // Python code may set this to NULL or even change it + // out from under us, which would tend to break things. + GeventWatcherObject handle = watcher->data; + const int cb_result = python_callback(handle, arg); + switch(cb_result) { + case -1: + // in case of exception, call self.loop.handle_error; + // this function is also responsible for stopping the watcher + // and allowing memory to be freed + python_handle_error(handle, arg); + break; + case 1: + // Code to stop the event IF NEEDED. Note that if python_callback + // has disposed of the last reference to the handle, + // `watcher` could now be invalid/disposed memory! + if (!uv_is_active(watcher)) { + if (watcher->data != handle) { + if (watcher->data) { + // If Python set the data to NULL, then they + // expected to be stopped. That's fine. + // Otherwise, something weird happened. + fprintf(stderr, + "WARNING: gevent: watcher handle changed in callback " + "from %p to %p for watcher at %p of type %d\n", + handle, watcher->data, watcher, watcher->type); + // There's a very good chance that the object the + // handle referred to has been changed and/or the + // old handle has been deallocated (most common), so + // passing the old handle will crash. Instead we + // pass a sigil to let python distinguish this case. + python_stop(NULL); + } + } + else { + python_stop(handle); + } + } + break; + case 2: + // watcher is already stopped and dead, nothing to do. + break; + default: + fprintf(stderr, + "WARNING: gevent: Unexpected return value %d from Python callback " + "for watcher %p (of type %d) and handle %p\n", + cb_result, + watcher, watcher->type, handle); + // XXX: Possible leaking of resources here? Should we be + // closing the watcher? + } +} + + +static void _gevent_generic_callback1(uv_handle_t* watcher, int arg) +{ + python_queue_callback(watcher, arg); +} + +static void _gevent_generic_callback0(uv_handle_t* handle) +{ + _gevent_generic_callback1(handle, 0); +} + +static void _gevent_async_callback0(uv_async_t* handle) +{ + _gevent_generic_callback0((uv_handle_t*)handle); +} + +static void _gevent_timer_callback0(uv_timer_t* handle) +{ + _gevent_generic_callback0((uv_handle_t*)handle); +} + +static void _gevent_prepare_callback0(uv_prepare_t* handle) +{ + _gevent_generic_callback0((uv_handle_t*)handle); +} + +static void _gevent_check_callback0(uv_check_t* handle) +{ + _gevent_generic_callback0((uv_handle_t*)handle); +} + +static void _gevent_idle_callback0(uv_idle_t* handle) +{ + _gevent_generic_callback0((uv_handle_t*)handle); +} + +static void _gevent_signal_callback1(uv_signal_t* handle, int signum) +{ + _gevent_generic_callback1((uv_handle_t*)handle, signum); +} + + +static void _gevent_poll_callback2(void* handle, int status, int events) +{ + _gevent_generic_callback1(handle, status < 0 ? status : events); +} + +static void _gevent_fs_event_callback3(void* handle, const char* filename, int events, int status) +{ + _gevent_generic_callback1(handle, status < 0 ? status : events); +} + + +typedef struct _gevent_fs_poll_s { + uv_fs_poll_t handle; + uv_stat_t curr; + uv_stat_t prev; +} gevent_fs_poll_t; + +static void _gevent_fs_poll_callback3(void* handlep, int status, const uv_stat_t* prev, const uv_stat_t* curr) +{ + // stat pointers are valid for this callback only. + // if given, copy them into our structure, where they can be reached + // from python, just like libev's watcher does, before calling + // the callback. + + // The callback is invoked with status < 0 if path does not exist + // or is inaccessible. The watcher is not stopped but your + // callback is not called again until something changes (e.g. when + // the file is created or the error reason changes). + // In that case the fields will be 0 in curr/prev. + + + gevent_fs_poll_t* handle = (gevent_fs_poll_t*)handlep; + assert(status == 0); + + handle->curr = *curr; + handle->prev = *prev; + + _gevent_generic_callback1((uv_handle_t*)handle, 0); +} + +static void gevent_uv_walk_callback_close(uv_handle_t* handle, void* arg) +{ + if( handle && !uv_is_closing(handle) ) { + uv_close(handle, NULL); + } +} + +static void gevent_close_all_handles(uv_loop_t* loop) +{ + uv_walk(loop, gevent_uv_walk_callback_close, NULL); +} + +static void gevent_zero_timer(uv_timer_t* handle) +{ + memset(handle, 0, sizeof(uv_timer_t)); +} + +static void gevent_zero_check(uv_check_t* handle) +{ + memset(handle, 0, sizeof(uv_check_t)); +} + +static void gevent_zero_prepare(uv_prepare_t* handle) +{ + memset(handle, 0, sizeof(uv_prepare_t)); +} + +static void gevent_zero_loop(uv_loop_t* handle) +{ + memset(handle, 0, sizeof(uv_loop_t)); +} diff --git a/src/gevent/libuv/loop.py b/src/gevent/libuv/loop.py new file mode 100644 index 0000000..5f7ff81 --- /dev/null +++ b/src/gevent/libuv/loop.py @@ -0,0 +1,602 @@ +""" +libuv loop implementation +""" +# pylint: disable=no-member +from __future__ import absolute_import, print_function + +import os +from collections import defaultdict +from collections import namedtuple +from operator import delitem +import signal + +from gevent._ffi import _dbg # pylint: disable=unused-import +from gevent._ffi.loop import AbstractLoop +from gevent.libuv import _corecffi # pylint:disable=no-name-in-module,import-error +from gevent._ffi.loop import assign_standard_callbacks +from gevent._ffi.loop import AbstractCallbacks +from gevent._util import implementer +from gevent._interfaces import ILoop + +ffi = _corecffi.ffi +libuv = _corecffi.lib + +__all__ = [ +] + + +class _Callbacks(AbstractCallbacks): + + def _find_loop_from_c_watcher(self, watcher_ptr): + loop_handle = ffi.cast('uv_handle_t*', watcher_ptr).data + return self.from_handle(loop_handle) if loop_handle else None + + def python_sigchld_callback(self, watcher_ptr, _signum): + self.from_handle(ffi.cast('uv_handle_t*', watcher_ptr).data)._sigchld_callback() + + def python_timer0_callback(self, watcher_ptr): + return self.python_prepare_callback(watcher_ptr) + + def python_queue_callback(self, watcher_ptr, revents): + watcher_handle = watcher_ptr.data + the_watcher = self.from_handle(watcher_handle) + + the_watcher.loop._queue_callback(watcher_ptr, revents) + + +_callbacks = assign_standard_callbacks( + ffi, libuv, _Callbacks, + [('python_sigchld_callback', None), + ('python_timer0_callback', None), + ('python_queue_callback', None)]) + +from gevent._ffi.loop import EVENTS +GEVENT_CORE_EVENTS = EVENTS # export + +from gevent.libuv import watcher as _watchers # pylint:disable=no-name-in-module + +_events_to_str = _watchers._events_to_str # export + +READ = libuv.UV_READABLE +WRITE = libuv.UV_WRITABLE + +def get_version(): + uv_bytes = ffi.string(libuv.uv_version_string()) + if not isinstance(uv_bytes, str): + # Py3 + uv_str = uv_bytes.decode("ascii") + else: + uv_str = uv_bytes + + return 'libuv-' + uv_str + +def get_header_version(): + return 'libuv-%d.%d.%d' % (libuv.UV_VERSION_MAJOR, libuv.UV_VERSION_MINOR, libuv.UV_VERSION_PATCH) + +def supported_backends(): + return ['default'] + +@implementer(ILoop) +class loop(AbstractLoop): + + # libuv parameters simply won't accept anything lower than 1ms. In + # practice, looping on gevent.sleep(0.001) takes about 0.00138 s + # (+- 0.000036s) + approx_timer_resolution = 0.001 # 1ms + + error_handler = None + + _CHECK_POINTER = 'uv_check_t *' + + _PREPARE_POINTER = 'uv_prepare_t *' + _PREPARE_CALLBACK_SIG = "void(*)(void*)" + + _TIMER_POINTER = _CHECK_POINTER # This is poorly named. It's for the callback "timer" + + def __init__(self, flags=None, default=None): + AbstractLoop.__init__(self, ffi, libuv, _watchers, flags, default) + self.__loop_pid = os.getpid() + self._child_watchers = defaultdict(list) + self._io_watchers = dict() + self._fork_watchers = set() + self._pid = os.getpid() + self._default = self._ptr == libuv.uv_default_loop() + self._queued_callbacks = [] + + def _queue_callback(self, watcher_ptr, revents): + self._queued_callbacks.append((watcher_ptr, revents)) + + def _init_loop(self, flags, default): + if default is None: + default = True + # Unlike libev, libuv creates a new default + # loop automatically if the old default loop was + # closed. + + if default: + # XXX: If the default loop had been destroyed, this + # will create a new one, but we won't destroy it + ptr = libuv.uv_default_loop() + else: + ptr = libuv.uv_loop_new() + + + if not ptr: + raise SystemError("Failed to get loop") + + # Track whether or not any object has destroyed + # this loop. See _can_destroy_default_loop + ptr.data = ptr + return ptr + + _signal_idle = None + + def _init_and_start_check(self): + libuv.uv_check_init(self._ptr, self._check) + libuv.uv_check_start(self._check, libuv.python_check_callback) + libuv.uv_unref(self._check) + + # We also have to have an idle watcher to be able to handle + # signals in a timely manner. Without them, libuv won't loop again + # and call into its check and prepare handlers. + # Note that this basically forces us into a busy-loop + # XXX: As predicted, using an idle watcher causes our process + # to eat 100% CPU time. We instead use a timer with a max of a .3 second + # delay to notice signals. Note that this timeout also implements fork + # watchers, effectively. + + # XXX: Perhaps we could optimize this to notice when there are other + # timers in the loop and start/stop it then. When we have a callback + # scheduled, this should also be the same and unnecessary? + # libev does takes this basic approach on Windows. + self._signal_idle = ffi.new("uv_timer_t*") + libuv.uv_timer_init(self._ptr, self._signal_idle) + self._signal_idle.data = self._handle_to_self + sig_cb = ffi.cast('void(*)(uv_timer_t*)', libuv.python_check_callback) + libuv.uv_timer_start(self._signal_idle, + sig_cb, + 300, + 300) + libuv.uv_unref(self._signal_idle) + + def _run_callbacks(self): + # Manually handle fork watchers. + curpid = os.getpid() + if curpid != self._pid: + self._pid = curpid + for watcher in self._fork_watchers: + watcher._on_fork() + + + # The contents of queued_callbacks at this point should be timers + # that expired when the loop began along with any idle watchers. + # We need to run them so that any manual callbacks they want to schedule + # get added to the list and ran next before we go on to poll for IO. + # This is critical for libuv on linux: closing a socket schedules some manual + # callbacks to actually stop the watcher; if those don't run before + # we poll for IO, then libuv can abort the process for the closed file descriptor. + + # XXX: There's still a race condition here because we may not run *all* the manual + # callbacks. We need a way to prioritize those. + + # Running these before the manual callbacks lead to some + # random test failures. In test__event.TestEvent_SetThenClear + # we would get a LoopExit sometimes. The problem occurred when + # a timer expired on entering the first loop; we would process + # it there, and then process the callback that it created + # below, leaving nothing for the loop to do. Having the + # self.run() manually process manual callbacks before + # continuing solves the problem. (But we must still run callbacks + # here again.) + self._prepare_ran_callbacks = self.__run_queued_callbacks() + + super(loop, self)._run_callbacks() + + def _init_and_start_prepare(self): + libuv.uv_prepare_init(self._ptr, self._prepare) + libuv.uv_prepare_start(self._prepare, libuv.python_prepare_callback) + libuv.uv_unref(self._prepare) + + def _init_callback_timer(self): + libuv.uv_check_init(self._ptr, self._timer0) + + def _stop_callback_timer(self): + libuv.uv_check_stop(self._timer0) + + def _start_callback_timer(self): + # The purpose of the callback timer is to ensure that we run + # callbacks as soon as possible on the next iteration of the event loop. + + # In libev, we set a 0 duration timer with a no-op callback. + # This executes immediately *after* the IO poll is done (it + # actually determines the time that the IO poll will block + # for), so having the timer present simply spins the loop, and + # our normal prepare watcher kicks in to run the callbacks. + + # In libuv, however, timers are run *first*, before prepare + # callbacks and before polling for IO. So a no-op 0 duration + # timer actually does *nothing*. (Also note that libev queues all + # watchers found during IO poll to run at the end (I think), while libuv + # runs them in uv__io_poll itself.) + + # From the loop inside uv_run: + # while True: + # uv__update_time(loop); + # uv__run_timers(loop); + # # we don't use pending watchers. They are how libuv + # # implements the pipe/udp/tcp streams. + # ran_pending = uv__run_pending(loop); + # uv__run_idle(loop); + # uv__run_prepare(loop); + # ... + # uv__io_poll(loop, timeout); # <--- IO watchers run here! + # uv__run_check(loop); + + # libev looks something like this (pseudo code because the real code is + # hard to read): + # + # do { + # run_fork_callbacks(); + # run_prepare_callbacks(); + # timeout = min(time of all timers or normal block time) + # io_poll() # <--- Only queues IO callbacks + # update_now(); calculate_expired_timers(); + # run callbacks in this order: (although specificying priorities changes it) + # check + # stat + # child + # signal + # timer + # io + # } + + # So instead of running a no-op and letting the side-effect of spinning + # the loop run the callbacks, we must explicitly run them here. + + # If we don't, test__systemerror:TestCallback will be flaky, failing + # one time out of ~20, depending on timing. + + # To get them to run immediately after this current loop, + # we use a check watcher, instead of a 0 duration timer entirely. + # If we use a 0 duration timer, we can get stuck in a timer loop. + # Python 3.6 fails in test_ftplib.py + + # As a final note, if we have not yet entered the loop *at + # all*, and a timer was created with a duration shorter than + # the amount of time it took for us to enter the loop in the + # first place, it may expire and get called before our callback + # does. This could also lead to test__systemerror:TestCallback + # appearing to be flaky. + + # As yet another final note, if we are currently running a + # timer callback, meaning we're inside uv__run_timers() in C, + # and the Python starts a new timer, if the Python code then + # update's the loop's time, it's possible that timer will + # expire *and be run in the same iteration of the loop*. This + # is trivial to do: In sequential code, anything after + # `gevent.sleep(0.1)` is running in a timer callback. Starting + # a new timer---e.g., another gevent.sleep() call---will + # update the time, *before* uv__run_timers exits, meaning + # other timers get a chance to run before our check or prepare + # watcher callbacks do. Therefore, we do indeed have to have a 0 + # timer to run callbacks---it gets inserted before any other user + # timers---ideally, this should be especially careful about how much time + # it runs for. + + # AND YET: We can't actually do that. We get timeouts that I haven't fully + # investigated if we do. Probably stuck in a timer loop. + + # As a partial remedy to this, unlike libev, our timer watcher + # class doesn't update the loop time by default. + + libuv.uv_check_start(self._timer0, libuv.python_timer0_callback) + + + def _stop_aux_watchers(self): + assert self._prepare + assert self._check + assert self._signal_idle + libuv.uv_prepare_stop(self._prepare) + libuv.uv_ref(self._prepare) # Why are we doing this? + + libuv.uv_check_stop(self._check) + libuv.uv_ref(self._check) + + libuv.uv_timer_stop(self._signal_idle) + libuv.uv_ref(self._signal_idle) + + libuv.uv_check_stop(self._timer0) + + def _setup_for_run_callback(self): + self._start_callback_timer() + libuv.uv_ref(self._timer0) + + + def _can_destroy_loop(self, ptr): + # We're being asked to destroy a loop that's, + # at the time it was constructed, was the default loop. + # If loop objects were constructed more than once, + # it may have already been destroyed, though. + # We track this in the data member. + return ptr.data + + def _destroy_loop(self, ptr): + ptr.data = ffi.NULL + libuv.uv_stop(ptr) + + libuv.gevent_close_all_handles(ptr) + + closed_failed = libuv.uv_loop_close(ptr) + if closed_failed: + assert closed_failed == libuv.UV_EBUSY + # We already closed all the handles. Run the loop + # once to let them be cut off from the loop. + ran_has_more_callbacks = libuv.uv_run(ptr, libuv.UV_RUN_ONCE) + if ran_has_more_callbacks: + libuv.uv_run(ptr, libuv.UV_RUN_NOWAIT) + closed_failed = libuv.uv_loop_close(ptr) + assert closed_failed == 0, closed_failed + + # Destroy the native resources *after* we have closed + # the loop. If we do it before, walking the handles + # attached to the loop is likely to segfault. + + libuv.gevent_zero_check(self._check) + libuv.gevent_zero_check(self._timer0) + libuv.gevent_zero_prepare(self._prepare) + libuv.gevent_zero_timer(self._signal_idle) + del self._check + del self._prepare + del self._signal_idle + del self._timer0 + + libuv.gevent_zero_loop(ptr) + + # Destroy any watchers we're still holding on to. + del self._io_watchers + del self._fork_watchers + del self._child_watchers + + + def debug(self): + """ + Return all the handles that are open and their ref status. + """ + handle_state = namedtuple("HandleState", + ['handle', + 'type', + 'watcher', + 'ref', + 'active', + 'closing']) + handles = [] + + # XXX: Convert this to a modern callback. + def walk(handle, _arg): + data = handle.data + if data: + watcher = ffi.from_handle(data) + else: + watcher = None + handles.append(handle_state(handle, + ffi.string(libuv.uv_handle_type_name(handle.type)), + watcher, + libuv.uv_has_ref(handle), + libuv.uv_is_active(handle), + libuv.uv_is_closing(handle))) + + libuv.uv_walk(self._ptr, + ffi.callback("void(*)(uv_handle_t*,void*)", + walk), + ffi.NULL) + return handles + + def ref(self): + pass + + def unref(self): + # XXX: Called by _run_callbacks. + pass + + def break_(self, how=None): + libuv.uv_stop(self._ptr) + + def reinit(self): + # TODO: How to implement? We probably have to simply + # re-__init__ this whole class? Does it matter? + # OR maybe we need to uv_walk() and close all the handles? + + # XXX: libuv < 1.12 simply CANNOT handle a fork unless you immediately + # exec() in the child. There are multiple calls to abort() that + # will kill the child process: + # - The OS X poll implementation (kqueue) aborts on an error return + # value; since kqueue FDs can't be inherited, then the next call + # to kqueue in the child will fail and get aborted; fork() is likely + # to be called during the gevent loop, meaning we're deep inside the + # runloop already, so we can't even close the loop that we're in: + # it's too late, the next call to kqueue is already scheduled. + # - The threadpool, should it be in use, also aborts + # (https://github.com/joyent/libuv/pull/1136) + # - There global shared state that breaks signal handling + # and leads to an abort() in the child, EVEN IF the loop in the parent + # had already been closed + # (https://github.com/joyent/libuv/issues/1405) + + # In 1.12, the uv_loop_fork function was added (by gevent!) + libuv.uv_loop_fork(self._ptr) + + _prepare_ran_callbacks = False + + def __run_queued_callbacks(self): + if not self._queued_callbacks: + return False + + cbs = list(self._queued_callbacks) + self._queued_callbacks = [] + + for watcher_ptr, arg in cbs: + handle = watcher_ptr.data + if not handle: + # It's been stopped and possibly closed + assert not libuv.uv_is_active(watcher_ptr) + continue + val = _callbacks.python_callback(handle, arg) + if val == -1: + _callbacks.python_handle_error(handle, arg) + elif val == 1: + if not libuv.uv_is_active(watcher_ptr): + if watcher_ptr.data != handle: + if watcher_ptr.data: + _callbacks.python_stop(None) + else: + _callbacks.python_stop(handle) + return True + + + def run(self, nowait=False, once=False): + # we can only respect one flag or the other. + # nowait takes precedence because it can't block + mode = libuv.UV_RUN_DEFAULT + if once: + mode = libuv.UV_RUN_ONCE + if nowait: + mode = libuv.UV_RUN_NOWAIT + + if mode == libuv.UV_RUN_DEFAULT: + while self._ptr and self._ptr.data: + # This is here to better preserve order guarantees. See _run_callbacks + # for details. + # It may get run again from the prepare watcher, so potentially we + # could take twice as long as the switch interval. + self._run_callbacks() + self._prepare_ran_callbacks = False + ran_status = libuv.uv_run(self._ptr, libuv.UV_RUN_ONCE) + # Note that we run queued callbacks when the prepare watcher runs, + # thus accounting for timers that expired before polling for IO, + # and idle watchers. This next call should get IO callbacks and + # callbacks from timers that expired *after* polling for IO. + ran_callbacks = self.__run_queued_callbacks() + + if not ran_status and not ran_callbacks and not self._prepare_ran_callbacks: + # A return of 0 means there are no referenced and + # active handles. The loop is over. + # If we didn't run any callbacks, then we couldn't schedule + # anything to switch in the future, so there's no point + # running again. + return ran_status + return 0 # Somebody closed the loop + + result = libuv.uv_run(self._ptr, mode) + self.__run_queued_callbacks() + return result + + def now(self): + # libuv's now is expressed as an integer number of + # milliseconds, so to get it compatible with time.time units + # that this method is supposed to return, we have to divide by 1000.0 + now = libuv.uv_now(self._ptr) + return now / 1000.0 + + def update_now(self): + libuv.uv_update_time(self._ptr) + + def fileno(self): + if self._ptr: + fd = libuv.uv_backend_fd(self._ptr) + if fd >= 0: + return fd + + _sigchld_watcher = None + _sigchld_callback_ffi = None + + def install_sigchld(self): + if not self.default: + return + + if self._sigchld_watcher: + return + + self._sigchld_watcher = ffi.new('uv_signal_t*') + libuv.uv_signal_init(self._ptr, self._sigchld_watcher) + self._sigchld_watcher.data = self._handle_to_self + + libuv.uv_signal_start(self._sigchld_watcher, + libuv.python_sigchld_callback, + signal.SIGCHLD) + + def reset_sigchld(self): + if not self.default or not self._sigchld_watcher: + return + + libuv.uv_signal_stop(self._sigchld_watcher) + # Must go through this to manage the memory lifetime + # correctly. Alternately, we could just stop it and restart + # it in install_sigchld? + _watchers.watcher._watcher_ffi_close(self._sigchld_watcher) + del self._sigchld_watcher + + + def _sigchld_callback(self): + # Signals can arrive at (relatively) any time. To eliminate + # race conditions, and behave more like libev, we "queue" + # sigchld to run when we run callbacks. + while True: + try: + pid, status, _usage = os.wait3(os.WNOHANG) + except OSError: + # Python 3 raises ChildProcessError + break + + if pid == 0: + break + children_watchers = self._child_watchers.get(pid, []) + self._child_watchers.get(0, []) + for watcher in children_watchers: + self.run_callback(watcher._set_waitpid_status, pid, status) + + # Don't invoke child watchers for 0 more than once + self._child_watchers[0] = [] + + def _register_child_watcher(self, watcher): + self._child_watchers[watcher._pid].append(watcher) + + def _unregister_child_watcher(self, watcher): + try: + # stop() should be idempotent + self._child_watchers[watcher._pid].remove(watcher) + except ValueError: + pass + + # Now's a good time to clean up any dead lists we don't need + # anymore + for pid in list(self._child_watchers): + if not self._child_watchers[pid]: + del self._child_watchers[pid] + + def io(self, fd, events, ref=True, priority=None): + # We rely on hard references here and explicit calls to + # close() on the returned object to correctly manage + # the watcher lifetimes. + + io_watchers = self._io_watchers + try: + io_watcher = io_watchers[fd] + assert io_watcher._multiplex_watchers, ("IO Watcher %s unclosed but should be dead" % io_watcher) + except KeyError: + # Start the watcher with just the events that we're interested in. + # as multiplexers are added, the real event mask will be updated to keep in sync. + # If we watch for too much, we get spurious wakeups and busy loops. + io_watcher = self._watchers.io(self, fd, 0) + io_watchers[fd] = io_watcher + io_watcher._no_more_watchers = lambda: delitem(io_watchers, fd) + + return io_watcher.multiplex(events) + + def prepare(self, ref=True, priority=None): + # We run arbitrary code in python_prepare_callback. That could switch + # greenlets. If it does that while also manipulating the active prepare + # watchers, we could corrupt the process state, since the prepare watcher + # queue is iterated on the stack (on unix). We could workaround this by implementing + # prepare watchers in pure Python. + # See https://github.com/gevent/gevent/issues/1126 + raise TypeError("prepare watchers are not currently supported in libuv. " + "If you need them, please contact the maintainers.") diff --git a/src/gevent/libuv/watcher.py b/src/gevent/libuv/watcher.py new file mode 100644 index 0000000..84a8ca4 --- /dev/null +++ b/src/gevent/libuv/watcher.py @@ -0,0 +1,735 @@ +# pylint: disable=too-many-lines, protected-access, redefined-outer-name, not-callable +# pylint: disable=no-member +from __future__ import absolute_import, print_function + +import functools +import sys + +from gevent.libuv import _corecffi # pylint:disable=no-name-in-module,import-error + +# Nothing public here +__all__ = [] + +ffi = _corecffi.ffi +libuv = _corecffi.lib + +from gevent._ffi import watcher as _base +from gevent._ffi import _dbg + +_closing_watchers = set() + +# In debug mode, it would be nice to be able to clear the memory of +# the watcher (its size determined by +# libuv.uv_handle_size(ffi_watcher.type)) using memset so that if we +# are using it after it's supposedly been closed and deleted, we'd +# catch it sooner. BUT doing so breaks test__threadpool. We get errors +# about `pthread_mutex_lock[3]: Invalid argument` (and sometimes we +# crash) suggesting either that we're writing on memory that doesn't +# belong to us, somehow, or that we haven't actually lost all +# references... +_uv_close_callback = ffi.def_extern(name='_uv_close_callback')(_closing_watchers.remove) + + +_events = [(libuv.UV_READABLE, "READ"), + (libuv.UV_WRITABLE, "WRITE")] + +def _events_to_str(events): # export + return _base.events_to_str(events, _events) + +class UVFuncallError(ValueError): + pass + +class libuv_error_wrapper(object): + # Makes sure that everything stored as a function + # on the wrapper instances (classes, actually, + # because this is used by the metaclass) + # checks its return value and raises an error. + # This expects that everything we call has an int + # or void return value and follows the conventions + # of error handling (that negative values are errors) + def __init__(self, uv): + self._libuv = uv + + def __getattr__(self, name): + libuv_func = getattr(self._libuv, name) + + @functools.wraps(libuv_func) + def wrap(*args, **kwargs): + if args and isinstance(args[0], watcher): + args = args[1:] + res = libuv_func(*args, **kwargs) + if res is not None and res < 0: + raise UVFuncallError( + str(ffi.string(libuv.uv_err_name(res)).decode('ascii') + + ' ' + + ffi.string(libuv.uv_strerror(res)).decode('ascii')) + + " Args: " + repr(args) + " KWARGS: " + repr(kwargs) + ) + return res + + setattr(self, name, wrap) + + return wrap + + +class ffi_unwrapper(object): + # undoes the wrapping of libuv_error_wrapper for + # the methods used by the metaclass that care + + def __init__(self, ff): + self._ffi = ff + + def __getattr__(self, name): + return getattr(self._ffi, name) + + def addressof(self, lib, name): + assert isinstance(lib, libuv_error_wrapper) + return self._ffi.addressof(libuv, name) + + +class watcher(_base.watcher): + _FFI = ffi_unwrapper(ffi) + _LIB = libuv_error_wrapper(libuv) + + _watcher_prefix = 'uv' + _watcher_struct_pattern = '%s_t' + + @classmethod + def _watcher_ffi_close(cls, ffi_watcher): + # Managing the lifetime of _watcher is tricky. + # They have to be uv_close()'d, but that only + # queues them to be closed in the *next* loop iteration. + # The memory must stay valid for at least that long, + # or assert errors are triggered. We can't use a ffi.gc() + # pointer to queue the uv_close, because by the time the + # destructor is called, there's no way to keep the memory alive + # and it could be re-used. + # So here we resort to resurrecting the pointer object out + # of our scope, keeping it alive past this object's lifetime. + # We then use the uv_close callback to handle removing that + # reference. There's no context passed to the close callback, + # so we have to do this globally. + + # Sadly, doing this causes crashes if there were multiple + # watchers for a given FD, so we have to take special care + # about that. See https://github.com/gevent/gevent/issues/790#issuecomment-208076604 + + # Note that this cannot be a __del__ method, because we store + # the CFFI handle to self on self, which is a cycle, and + # objects with a __del__ method cannot be collected on CPython < 3.4 + + # Instead, this is arranged as a callback to GC when the + # watcher class dies. Obviously it's important to keep the ffi + # watcher alive. + # We can pass in "subclasses" if uv_handle_t that line up at the C level, + # but that don't in CFFI without a cast. But be careful what we use the cast + # for, don't pass it back to C. + ffi_handle_watcher = cls._FFI.cast('uv_handle_t*', ffi_watcher) + if ffi_handle_watcher.type and not libuv.uv_is_closing(ffi_watcher): + # If the type isn't set, we were never properly initialized, + # and trying to close it results in libuv terminating the process. + # Sigh. Same thing if it's already in the process of being + # closed. + _closing_watchers.add(ffi_watcher) + libuv.uv_close(ffi_watcher, libuv._uv_close_callback) + + ffi_handle_watcher.data = ffi.NULL + + + def _watcher_ffi_set_init_ref(self, ref): + self.ref = ref + + def _watcher_ffi_init(self, args): + # TODO: we could do a better job chokepointing this + return self._watcher_init(self.loop.ptr, + self._watcher, + *args) + + def _watcher_ffi_start(self): + self._watcher_start(self._watcher, self._watcher_callback) + + def _watcher_ffi_stop(self): + if self._watcher: + # The multiplexed io watcher deletes self._watcher + # when it closes down. If that's in the process of + # an error handler, AbstractCallbacks.unhandled_onerror + # will try to close us again. + self._watcher_stop(self._watcher) + + @_base.only_if_watcher + def _watcher_ffi_ref(self): + libuv.uv_ref(self._watcher) + + @_base.only_if_watcher + def _watcher_ffi_unref(self): + libuv.uv_unref(self._watcher) + + def _watcher_ffi_start_unref(self): + pass + + def _watcher_ffi_stop_ref(self): + pass + + def _get_ref(self): + # Convert 1/0 to True/False + if self._watcher is None: + return None + return bool(libuv.uv_has_ref(self._watcher)) + + def _set_ref(self, value): + if value: + self._watcher_ffi_ref() + else: + self._watcher_ffi_unref() + + ref = property(_get_ref, _set_ref) + + def feed(self, _revents, _callback, *_args): + raise Exception("Not implemented") + +class io(_base.IoMixin, watcher): + _watcher_type = 'poll' + _watcher_callback_name = '_gevent_poll_callback2' + + # On Windows is critical to be able to garbage collect these + # objects in a timely fashion so that they don't get reused + # for multiplexing completely different sockets. This is because + # uv_poll_init_socket does a lot of setup for the socket to make + # polling work. If get reused for another socket that has the same + # fileno, things break badly. (In theory this could be a problem + # on posix too, but in practice it isn't). + + # TODO: We should probably generalize this to all + # ffi watchers. Avoiding GC cycles as much as possible + # is a good thing, and potentially allocating new handles + # as needed gets us better memory locality. + + # Especially on Windows, we must also account for the case that a + # reference to this object has leaked (e.g., the socket object is + # still around), but the fileno has been closed and a new one + # opened. We must still get a new native watcher at that point. We + # handle this case by simply making sure that we don't even have + # a native watcher until the object is started, and we shut it down + # when the object is stopped. + + # XXX: I was able to solve at least Windows test_ftplib.py issues + # with more of a careful use of io objects in socket.py, so + # delaying this entirely is at least temporarily on hold. Instead + # sticking with the _watcher_create function override for the + # moment. + + # XXX: Note 2: Moving to a deterministic close model, which was necessary + # for PyPy, also seems to solve the Windows issues. So we're completely taking + # this object out of the loop's registration; we don't want GC callbacks and + # uv_close anywhere *near* this object. + + _watcher_registers_with_loop_on_create = False + + EVENT_MASK = libuv.UV_READABLE | libuv.UV_WRITABLE | libuv.UV_DISCONNECT + + _multiplex_watchers = () + + def __init__(self, loop, fd, events, ref=True, priority=None): + super(io, self).__init__(loop, fd, events, ref=ref, priority=priority, _args=(fd,)) + self._fd = fd + self._events = events + self._multiplex_watchers = [] + + def _get_fd(self): + return self._fd + + @_base.not_while_active + def _set_fd(self, fd): + self._fd = fd + self._watcher_ffi_init((fd,)) + + def _get_events(self): + return self._events + + def _set_events(self, events): + if events == self._events: + return + self._events = events + if self.active: + # We're running but libuv specifically says we can + # call start again to change our event mask. + assert self._handle is not None + self._watcher_start(self._watcher, self._events, self._watcher_callback) + + events = property(_get_events, _set_events) + + def _watcher_ffi_start(self): + self._watcher_start(self._watcher, self._events, self._watcher_callback) + + if sys.platform.startswith('win32'): + # uv_poll can only handle sockets on Windows, but the plain + # uv_poll_init we call on POSIX assumes that the fileno + # argument is already a C fileno, as created by + # _get_osfhandle. C filenos are limited resources, must be + # closed with _close. So there are lifetime issues with that: + # calling the C function _close to dispose of the fileno + # *also* closes the underlying win32 handle, possibly + # prematurely. (XXX: Maybe could do something with weak + # references? But to what?) + + # All libuv wants to do with the fileno in uv_poll_init is + # turn it back into a Win32 SOCKET handle. + + # Now, libuv provides uv_poll_init_socket, which instead of + # taking a C fileno takes the SOCKET, avoiding the need to dance with + # the C runtime. + + # It turns out that SOCKET (win32 handles in general) can be + # represented with `intptr_t`. It further turns out that + # CPython *directly* exposes the SOCKET handle as the value of + # fileno (32-bit PyPy does some munging on it, which should + # rarely matter). So we can pass socket.fileno() through + # to uv_poll_init_socket. + + # See _corecffi_build. + _watcher_init = watcher._LIB.uv_poll_init_socket + + + class _multiplexwatcher(object): + + callback = None + args = () + pass_events = False + ref = True + + def __init__(self, events, watcher): + self._events = events + + # References: + # These objects must keep the original IO object alive; + # the IO object SHOULD NOT keep these alive to avoid cycles + # We MUST NOT rely on GC to clean up the IO objects, but the explicit + # calls to close(); see _multiplex_closed. + self._watcher_ref = watcher + + events = property( + lambda self: self._events, + _base.not_while_active(lambda self, nv: setattr(self, '_events', nv))) + + def start(self, callback, *args, **kwargs): + self.pass_events = kwargs.get("pass_events") + self.callback = callback + self.args = args + + watcher = self._watcher_ref + if watcher is not None: + if not watcher.active: + watcher._io_start() + else: + # Make sure we're in the event mask + watcher._calc_and_update_events() + + def stop(self): + self.callback = None + self.pass_events = None + self.args = None + watcher = self._watcher_ref + if watcher is not None: + watcher._io_maybe_stop() + + def close(self): + if self._watcher_ref is not None: + self._watcher_ref._multiplex_closed(self) + self._watcher_ref = None + + @property + def active(self): + return self.callback is not None + + @property + def _watcher(self): + # For testing. + return self._watcher_ref._watcher + + # ares.pyx depends on this property, + # and test__core uses it too + fd = property(lambda self: getattr(self._watcher_ref, '_fd', -1), + lambda self, nv: self._watcher_ref._set_fd(nv)) + + def _io_maybe_stop(self): + self._calc_and_update_events() + for w in self._multiplex_watchers: + if w.callback is not None: + # There's still a reference to it, and it's started, + # so we can't stop. + return + # If we get here, nothing was started + # so we can take ourself out of the polling set + self.stop() + + def _io_start(self): + self._calc_and_update_events() + self.start(self._io_callback, pass_events=True) + + def _calc_and_update_events(self): + events = 0 + for watcher in self._multiplex_watchers: + if watcher.callback is not None: + # Only ask for events that are active. + events |= watcher.events + self._set_events(events) + + + def multiplex(self, events): + watcher = self._multiplexwatcher(events, self) + self._multiplex_watchers.append(watcher) + self._calc_and_update_events() + return watcher + + def close(self): + super(io, self).close() + del self._multiplex_watchers + + def _multiplex_closed(self, watcher): + self._multiplex_watchers.remove(watcher) + if not self._multiplex_watchers: + self.stop() # should already be stopped + self._no_more_watchers() + # It is absolutely critical that we control when the call + # to uv_close() gets made. uv_close() of a uv_poll_t + # handle winds up calling uv__platform_invalidate_fd, + # which, as the name implies, destroys any outstanding + # events for the *fd* that haven't been delivered yet, and also removes + # the *fd* from the poll set. So if this happens later, at some + # non-deterministic time when (cyclic or otherwise) GC runs, + # *and* we've opened a new watcher for the fd, that watcher will + # suddenly and mysteriously stop seeing events. So we do this now; + # this method is smart enough not to close the handle twice. + self.close() + else: + self._calc_and_update_events() + + def _no_more_watchers(self): + # The loop sets this on an individual watcher to delete it from + # the active list where it keeps hard references. + pass + + def _io_callback(self, events): + if events < 0: + # actually a status error code + _dbg("Callback error on", self._fd, + ffi.string(libuv.uv_err_name(events)), + ffi.string(libuv.uv_strerror(events))) + # XXX: We've seen one half of a FileObjectPosix pair + # (the read side of a pipe) report errno 11 'bad file descriptor' + # after the write side was closed and its watcher removed. But + # we still need to attempt to read from it to clear out what's in + # its buffers--if we return with the watcher inactive before proceeding to wake up + # the reader, we get a LoopExit. So we can't return here and arguably shouldn't print it + # either. The negative events mask will match the watcher's mask. + # See test__fileobject.py:Test.test_newlines for an example. + + # On Windows (at least with PyPy), we can get ENOTSOCK (socket operation on non-socket) + # if a socket gets closed. If we don't pass the events on, we hang. + # See test__makefile_ref.TestSSL for examples. + # return + + for watcher in self._multiplex_watchers: + if not watcher.callback: + # Stopped + continue + assert watcher._watcher_ref is self, (self, watcher._watcher_ref) + + send_event = (events & watcher.events) or events < 0 + if send_event: + if not watcher.pass_events: + watcher.callback(*watcher.args) + else: + watcher.callback(events, *watcher.args) + +class _SimulatedWithAsyncMixin(object): + _watcher_skip_ffi = True + + def __init__(self, loop, *args, **kwargs): + self._async = loop.async_() + try: + super(_SimulatedWithAsyncMixin, self).__init__(loop, *args, **kwargs) + except: + self._async.close() + raise + + def _watcher_create(self, _args): + return + + @property + def _watcher_handle(self): + return None + + def _watcher_ffi_init(self, _args): + return + + def _watcher_ffi_set_init_ref(self, ref): + self._async.ref = ref + + @property + def active(self): + return self._async.active + + def start(self, cb, *args): + self._register_loop_callback() + self.callback = cb + self.args = args + self._async.start(cb, *args) + #watcher.start(self, cb, *args) + + def stop(self): + self._unregister_loop_callback() + self.callback = None + self.args = None + self._async.stop() + + def close(self): + if self._async is not None: + a = self._async + #self._async = None + a.close() + + def _register_loop_callback(self): + # called from start() + raise NotImplementedError() + + def _unregister_loop_callback(self): + # called from stop + raise NotImplementedError() + +class fork(_SimulatedWithAsyncMixin, + _base.ForkMixin, + watcher): + # We'll have to implement this one completely manually + # Right now it doesn't matter much since libuv doesn't survive + # a fork anyway. (That's a work in progress) + _watcher_skip_ffi = False + + def _register_loop_callback(self): + self.loop._fork_watchers.add(self) + + def _unregister_loop_callback(self): + try: + # stop() should be idempotent + self.loop._fork_watchers.remove(self) + except KeyError: + pass + + def _on_fork(self): + self._async.send() + + +class child(_SimulatedWithAsyncMixin, + _base.ChildMixin, + watcher): + _watcher_skip_ffi = True + # We'll have to implement this one completely manually. + # Our approach is to use a SIGCHLD handler and the original + # os.waitpid call. + + # On Unix, libuv's uv_process_t and uv_spawn use SIGCHLD, + # just like libev does for its child watchers. So + # we're not adding any new SIGCHLD related issues not already + # present in libev. + + + def _register_loop_callback(self): + self.loop._register_child_watcher(self) + + def _unregister_loop_callback(self): + self.loop._unregister_child_watcher(self) + + def _set_waitpid_status(self, pid, status): + self._rpid = pid + self._rstatus = status + self._async.send() + + +class async_(_base.AsyncMixin, watcher): + _watcher_callback_name = '_gevent_async_callback0' + + def _watcher_ffi_init(self, args): + # It's dangerous to have a raw, non-initted struct + # around; it will crash in uv_close() when we get GC'd, + # and send() will also crash. + # NOTE: uv_async_init is NOT idempotent. Calling it more than + # once adds the uv_async_t to the internal queue multiple times, + # and uv_close only cleans up one of them, meaning that we tend to + # crash. Thus we have to be very careful not to allow that. + return self._watcher_init(self.loop.ptr, self._watcher, ffi.NULL) + + def _watcher_ffi_start(self): + # we're created in a started state, but we didn't provide a + # callback (because if we did and we don't have a value in our + # callback attribute, then python_callback would crash.) Note that + # uv_async_t->async_cb is not technically documented as public. + self._watcher.async_cb = self._watcher_callback + + def _watcher_ffi_stop(self): + self._watcher.async_cb = ffi.NULL + # We have to unref this because we're setting the cb behind libuv's + # back, basically: once a async watcher is started, it can't ever be + # stopped through libuv interfaces, so it would never lose its active + # status, and thus if it stays reffed it would keep the event loop + # from exiting. + self._watcher_ffi_unref() + + def send(self): + if libuv.uv_is_closing(self._watcher): + raise Exception("Closing handle") + libuv.uv_async_send(self._watcher) + + @property + def pending(self): + return None + +locals()['async'] = async_ + +class timer(_base.TimerMixin, watcher): + + _watcher_callback_name = '_gevent_timer_callback0' + + # In libuv, timer callbacks continue running while any timer is + # expired, including newly added timers. Newly added non-zero + # timers (especially of small duration) can be seen to be expired + # if the loop time is updated while we are in a timer callback. + # This can lead to us being stuck running timers for a terribly + # long time, which is not good. So default to not updating the + # time. + + # Also, newly-added timers of 0 duration can *also* stall the + # loop, because they'll be seen to be expired immediately. + # Updating the time can prevent that, *if* there was already a + # timer for a longer duration scheduled. + + # To mitigate the above problems, our loop implementation turns + # zero duration timers into check watchers instead using OneShotCheck. + # This ensures the loop cycles. Of course, the 'again' method does + # nothing on them and doesn't exist. In practice that's not an issue. + + _again = False + + def _watcher_ffi_init(self, args): + self._watcher_init(self.loop._ptr, self._watcher) + self._after, self._repeat = args + if self._after and self._after < 0.001: + import warnings + # XXX: The stack level is hard to determine, could be getting here + # through a number of different ways. + warnings.warn("libuv only supports millisecond timer resolution; " + "all times less will be set to 1 ms", + stacklevel=6) + # The alternative is to effectively pass in int(0.1) == 0, which + # means no sleep at all, which leads to excessive wakeups + self._after = 0.001 + if self._repeat and self._repeat < 0.001: + import warnings + warnings.warn("libuv only supports millisecond timer resolution; " + "all times less will be set to 1 ms", + stacklevel=6) + self._repeat = 0.001 + + def _watcher_ffi_start(self): + if self._again: + libuv.uv_timer_again(self._watcher) + else: + try: + self._watcher_start(self._watcher, self._watcher_callback, + int(self._after * 1000), + int(self._repeat * 1000)) + except ValueError: + # in case of non-ints in _after/_repeat + raise TypeError() + + def again(self, callback, *args, **kw): + if not self.active: + # If we've never been started, this is the same as starting us. + # libuv makes the distinction, libev doesn't. + self.start(callback, *args, **kw) + return + + self._again = True + try: + self.start(callback, *args, **kw) + finally: + del self._again + + +class stat(_base.StatMixin, watcher): + _watcher_type = 'fs_poll' + _watcher_struct_name = 'gevent_fs_poll_t' + _watcher_callback_name = '_gevent_fs_poll_callback3' + + def _watcher_set_data(self, the_watcher, data): + the_watcher.handle.data = data + return data + + def _watcher_ffi_init(self, args): + return self._watcher_init(self.loop._ptr, self._watcher) + + MIN_STAT_INTERVAL = 0.1074891 # match libev; 0.0 is default + + def _watcher_ffi_start(self): + # libev changes this when the watcher is started + if self._interval < self.MIN_STAT_INTERVAL: + self._interval = self.MIN_STAT_INTERVAL + self._watcher_start(self._watcher, self._watcher_callback, + self._cpath, + int(self._interval * 1000)) + + @property + def _watcher_handle(self): + return self._watcher.handle.data + + @property + def attr(self): + if not self._watcher.curr.st_nlink: + return + return self._watcher.curr + + @property + def prev(self): + if not self._watcher.prev.st_nlink: + return + return self._watcher.prev + + +class signal(_base.SignalMixin, watcher): + _watcher_callback_name = '_gevent_signal_callback1' + + def _watcher_ffi_init(self, args): + self._watcher_init(self.loop._ptr, self._watcher) + self.ref = False # libev doesn't ref these by default + + + def _watcher_ffi_start(self): + self._watcher_start(self._watcher, self._watcher_callback, + self._signalnum) + + +class idle(_base.IdleMixin, watcher): + # Because libuv doesn't support priorities, idle watchers are + # potentially quite a bit different than under libev + _watcher_callback_name = '_gevent_idle_callback0' + + +class check(_base.CheckMixin, watcher): + _watcher_callback_name = '_gevent_check_callback0' + +class OneShotCheck(check): + + _watcher_skip_ffi = True + + def __make_cb(self, func): + stop = self.stop + @functools.wraps(func) + def cb(*args): + stop() + return func(*args) + return cb + + def start(self, callback, *args): + return check.start(self, self.__make_cb(callback), *args) + +class prepare(_base.PrepareMixin, watcher): + _watcher_callback_name = '_gevent_prepare_callback0' diff --git a/src/gevent/local.py b/src/gevent/local.py new file mode 100644 index 0000000..f29bb23 --- /dev/null +++ b/src/gevent/local.py @@ -0,0 +1,605 @@ +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False +""" +Greenlet-local objects. + +This module is based on `_threading_local.py`__ from the standard +library of Python 3.4. + +__ https://github.com/python/cpython/blob/3.4/Lib/_threading_local.py + +Greenlet-local objects support the management of greenlet-local data. +If you have data that you want to be local to a greenlet, simply create +a greenlet-local object and use its attributes: + + >>> mydata = local() + >>> mydata.number = 42 + >>> mydata.number + 42 + +You can also access the local-object's dictionary: + + >>> mydata.__dict__ + {'number': 42} + >>> mydata.__dict__.setdefault('widgets', []) + [] + >>> mydata.widgets + [] + +What's important about greenlet-local objects is that their data are +local to a greenlet. If we access the data in a different greenlet: + + >>> log = [] + >>> def f(): + ... items = list(mydata.__dict__.items()) + ... items.sort() + ... log.append(items) + ... mydata.number = 11 + ... log.append(mydata.number) + >>> greenlet = gevent.spawn(f) + >>> greenlet.join() + >>> log + [[], 11] + +we get different data. Furthermore, changes made in the other greenlet +don't affect data seen in this greenlet: + + >>> mydata.number + 42 + +Of course, values you get from a local object, including a __dict__ +attribute, are for whatever greenlet was current at the time the +attribute was read. For that reason, you generally don't want to save +these values across greenlets, as they apply only to the greenlet they +came from. + +You can create custom local objects by subclassing the local class: + + >>> class MyLocal(local): + ... number = 2 + ... initialized = False + ... def __init__(self, **kw): + ... if self.initialized: + ... raise SystemError('__init__ called too many times') + ... self.initialized = True + ... self.__dict__.update(kw) + ... def squared(self): + ... return self.number ** 2 + +This can be useful to support default values, methods and +initialization. Note that if you define an __init__ method, it will be +called each time the local object is used in a separate greenlet. This +is necessary to initialize each greenlet's dictionary. + +Now if we create a local object: + + >>> mydata = MyLocal(color='red') + +Now we have a default number: + + >>> mydata.number + 2 + +an initial color: + + >>> mydata.color + 'red' + >>> del mydata.color + +And a method that operates on the data: + + >>> mydata.squared() + 4 + +As before, we can access the data in a separate greenlet: + + >>> log = [] + >>> greenlet = gevent.spawn(f) + >>> greenlet.join() + >>> log + [[('color', 'red'), ('initialized', True)], 11] + +without affecting this greenlet's data: + + >>> mydata.number + 2 + >>> mydata.color + Traceback (most recent call last): + ... + AttributeError: 'MyLocal' object has no attribute 'color' + +Note that subclasses can define slots, but they are not greenlet +local. They are shared across greenlets:: + + >>> class MyLocal(local): + ... __slots__ = 'number' + + >>> mydata = MyLocal() + >>> mydata.number = 42 + >>> mydata.color = 'red' + +So, the separate greenlet: + + >>> greenlet = gevent.spawn(f) + >>> greenlet.join() + +affects what we see: + + >>> mydata.number + 11 + +>>> del mydata + +.. versionchanged:: 1.1a2 + Update the implementation to match Python 3.4 instead of Python 2.5. + This results in locals being eligible for garbage collection as soon + as their greenlet exits. + +.. versionchanged:: 1.2.3 + Use a weak-reference to clear the greenlet link we establish in case + the local object dies before the greenlet does. + +.. versionchanged:: 1.3a1 + Implement the methods for attribute access directly, handling + descriptors directly here. This allows removing the use of a lock + and facilitates greatly improved performance. + +.. versionchanged:: 1.3a1 + The ``__init__`` method of subclasses of ``local`` is no longer + called with a lock held. CPython does not use such a lock in its + native implementation. This could potentially show as a difference + if code that uses multiple dependent attributes in ``__slots__`` + (which are shared across all greenlets) switches during ``__init__``. + +""" +from __future__ import print_function + +from copy import copy +from weakref import ref + + +locals()['getcurrent'] = __import__('greenlet').getcurrent +locals()['greenlet_init'] = lambda: None + +__all__ = [ + "local", +] + +# The key used in the Thread objects' attribute dicts. +# We keep it a string for speed but make it unlikely to clash with +# a "real" attribute. +key_prefix = '_gevent_local_localimpl_' + +# The overall structure is as follows: +# For each local() object: +# greenlet.__dict__[key_prefix + str(id(local))] +# => _localimpl.dicts[id(greenlet)] => (ref(greenlet), {}) + +# That final tuple is actually a localimpl_dict_entry object. + +def all_local_dicts_for_greenlet(greenlet): + """ + Internal debug helper for getting the local values associated + with a greenlet. This is subject to change or removal at any time. + + :return: A list of ((type, id), {}) pairs, where the first element + is the type and id of the local object and the second object is its + instance dictionary, as seen from this greenlet. + + .. versionadded:: 1.3a2 + """ + + result = [] + id_greenlet = id(greenlet) + greenlet_dict = greenlet.__dict__ + for k, v in greenlet_dict.items(): + if not k.startswith(key_prefix): + continue + local_impl = v() + if local_impl is None: + continue + entry = local_impl.dicts.get(id_greenlet) + if entry is None: + # Not yet used in this greenlet. + continue + assert entry.wrgreenlet() is greenlet + result.append((local_impl.localtypeid, entry.localdict)) + + return result + + +class _wrefdict(dict): + """A dict that can be weak referenced""" + +class _greenlet_deleted(object): + """ + A weakref callback for when the greenlet + is deleted. + + If the greenlet is a `gevent.greenlet.Greenlet` and + supplies ``rawlink``, that will be used instead of a + weakref. + """ + __slots__ = ('idt', 'wrdicts') + + def __init__(self, idt, wrdicts): + self.idt = idt + self.wrdicts = wrdicts + + def __call__(self, _unused): + dicts = self.wrdicts() + if dicts: + dicts.pop(self.idt, None) + +class _local_deleted(object): + __slots__ = ('key', 'wrthread', 'greenlet_deleted') + + def __init__(self, key, wrthread, greenlet_deleted): + self.key = key + self.wrthread = wrthread + self.greenlet_deleted = greenlet_deleted + + def __call__(self, _unused): + thread = self.wrthread() + if thread is not None: + try: + unlink = thread.unlink + except AttributeError: + pass + else: + unlink(self.greenlet_deleted) + del thread.__dict__[self.key] + +class _localimpl(object): + """A class managing thread-local dicts""" + __slots__ = ('key', 'dicts', + 'localargs', 'localkwargs', + 'localtypeid', + '__weakref__',) + + def __init__(self, args, kwargs, local_type, id_local): + self.key = key_prefix + str(id(self)) + # { id(greenlet) -> _localimpl_dict_entry(ref(greenlet), greenlet-local dict) } + self.dicts = _wrefdict() + self.localargs = args + self.localkwargs = kwargs + self.localtypeid = local_type, id_local + + # We need to create the thread dict in anticipation of + # __init__ being called, to make sure we don't call it + # again ourselves. MUST do this before setting any attributes. + greenlet = getcurrent() # pylint:disable=undefined-variable + _localimpl_create_dict(self, greenlet, id(greenlet)) + +class _localimpl_dict_entry(object): + """ + The object that goes in the ``dicts`` of ``_localimpl`` + object for each thread. + """ + # This is a class, not just a tuple, so that cython can optimize + # attribute access + __slots__ = ('wrgreenlet', 'localdict') + + def __init__(self, wrgreenlet, localdict): + self.wrgreenlet = wrgreenlet + self.localdict = localdict + +# We use functions instead of methods so that they can be cdef'd in +# local.pxd; if they were cdef'd as methods, they would cause +# the creation of a pointer and a vtable. This happens +# even if we declare the class @cython.final. functions thus save memory overhead +# (but not pointer chasing overhead; the vtable isn't used when we declare +# the class final). + + +def _localimpl_create_dict(self, greenlet, id_greenlet): + """Create a new dict for the current thread, and return it.""" + localdict = {} + key = self.key + + wrdicts = ref(self.dicts) + + # When the greenlet is deleted, remove the local dict. + # Note that this is suboptimal if the greenlet object gets + # caught in a reference loop. We would like to be called + # as soon as the OS-level greenlet ends instead. + + # If we are working with a gevent.greenlet.Greenlet, we + # can pro-actively clear out with a link, avoiding the + # issue described above. Use rawlink to avoid spawning any + # more greenlets. + greenlet_deleted = _greenlet_deleted(id_greenlet, wrdicts) + + rawlink = getattr(greenlet, 'rawlink', None) + if rawlink is not None: + rawlink(greenlet_deleted) + wrthread = ref(greenlet) + else: + wrthread = ref(greenlet, greenlet_deleted) + + + # When the localimpl is deleted, remove the thread attribute. + local_deleted = _local_deleted(key, wrthread, greenlet_deleted) + + + wrlocal = ref(self, local_deleted) + greenlet.__dict__[key] = wrlocal + + self.dicts[id_greenlet] = _localimpl_dict_entry(wrthread, localdict) + return localdict + + +_marker = object() + +def _local_get_dict(self): + impl = self._local__impl + # Cython can optimize dict[], but not dict.get() + greenlet = getcurrent() # pylint:disable=undefined-variable + idg = id(greenlet) + try: + entry = impl.dicts[idg] + dct = entry.localdict + except KeyError: + dct = _localimpl_create_dict(impl, greenlet, idg) + self.__init__(*impl.localargs, **impl.localkwargs) + return dct + +def _init(): + greenlet_init() # pylint:disable=undefined-variable + +_local_attrs = { + '_local__impl', + '_local_type_get_descriptors', + '_local_type_set_or_del_descriptors', + '_local_type_del_descriptors', + '_local_type_set_descriptors', + '_local_type', + '_local_type_vars', + '__class__', + '__cinit__', +} + +class local(object): + """ + An object whose attributes are greenlet-local. + """ + __slots__ = tuple(_local_attrs - {'__class__', '__cinit__'}) + + def __cinit__(self, *args, **kw): + if args or kw: + if type(self).__init__ == object.__init__: + raise TypeError("Initialization arguments are not supported", args, kw) + impl = _localimpl(args, kw, type(self), id(self)) + # pylint:disable=attribute-defined-outside-init + self._local__impl = impl + get, dels, sets_or_dels, sets = _local_find_descriptors(self) + self._local_type_get_descriptors = get + self._local_type_set_or_del_descriptors = sets_or_dels + self._local_type_del_descriptors = dels + self._local_type_set_descriptors = sets + self._local_type = type(self) + self._local_type_vars = set(dir(self._local_type)) + + def __getattribute__(self, name): # pylint:disable=too-many-return-statements + if name in _local_attrs: + # The _local__impl, __cinit__, etc, won't be hit by the + # Cython version, if we've done things right. If we haven't, + # they will be, and this will produce an error. + return object.__getattribute__(self, name) + + dct = _local_get_dict(self) + + if name == '__dict__': + return dct + # If there's no possible way we can switch, because this + # attribute is *not* found in the class where it might be a + # data descriptor (property), and it *is* in the dict + # then we don't need to swizzle the dict and take the lock. + + # We don't have to worry about people overriding __getattribute__ + # because if they did, the dict-swizzling would only last as + # long as we were in here anyway. + # Similarly, a __getattr__ will still be called by _oga() if needed + # if it's not in the dict. + + # Optimization: If we're not subclassed, then + # there can be no descriptors except for methods, which will + # never need to use __dict__. + if self._local_type is local: + return dct[name] if name in dct else object.__getattribute__(self, name) + + # NOTE: If this is a descriptor, this will invoke its __get__. + # A broken descriptor that doesn't return itself when called with + # a None for the instance argument could mess us up here. + # But this is faster than a loop over mro() checking each class __dict__ + # manually. + if name in dct: + if name not in self._local_type_vars: + # If there is a dict value, and nothing in the type, + # it can't possibly be a descriptor, so it is just returned. + return dct[name] + + # It's in the type *and* in the dict. If the type value is + # a data descriptor (defines __get__ *and* either __set__ or + # __delete__), then the type wins. If it's a non-data descriptor + # (defines just __get__), then the instance wins. If it's not a + # descriptor at all (doesn't have __get__), the instance wins. + # NOTE that the docs for descriptors say that these methods must be + # defined on the *class* of the object in the type. + if name not in self._local_type_get_descriptors: + # Entirely not a descriptor. Instance wins. + return dct[name] + if name in self._local_type_set_or_del_descriptors: + # A data descriptor. + # arbitrary code execution while these run. If they touch self again, + # they'll call back into us and we'll repeat the dance. + type_attr = getattr(self._local_type, name) + return type(type_attr).__get__(type_attr, self, self._local_type) + # Last case is a non-data descriptor. Instance wins. + return dct[name] + + if name in self._local_type_vars: + # Not in the dictionary, but is found in the type. It could be + # a non-data descriptor still. Some descriptors, like @staticmethod, + # return objects (functions, in this case), that are *themselves* + # descriptors, which when invoked, again, would do the wrong thing. + # So we can't rely on getattr() on the type for them, we have to + # look through the MRO dicts ourself. + if name not in self._local_type_get_descriptors: + # Not a descriptor, can't execute code. So all we need is + # the return value of getattr() on our type. + return getattr(self._local_type, name) + + for base in self._local_type.mro(): + bd = base.__dict__ + if name in bd: + attr_on_type = bd[name] + result = type(attr_on_type).__get__(attr_on_type, self, self._local_type) + return result + + # It wasn't in the dict and it wasn't in the type. + # So the next step is to invoke type(self)__getattr__, if it + # exists, otherwise raise an AttributeError. + # we will invoke type(self).__getattr__ or raise an attribute error. + if hasattr(self._local_type, '__getattr__'): + return self._local_type.__getattr__(self, name) + raise AttributeError("%r object has no attribute '%s'" + % (self._local_type.__name__, name)) + + def __setattr__(self, name, value): + if name == '__dict__': + raise AttributeError( + "%r object attribute '__dict__' is read-only" + % type(self)) + + if name in _local_attrs: + object.__setattr__(self, name, value) + return + + dct = _local_get_dict(self) + + if self._local_type is local: + # Optimization: If we're not subclassed, we can't + # have data descriptors, so this goes right in the dict. + dct[name] = value + return + + if name in self._local_type_vars: + if name in self._local_type_set_descriptors: + type_attr = getattr(self._local_type, name, _marker) + # A data descriptor, like a property or a slot. + type(type_attr).__set__(type_attr, self, value) + return + # Otherwise it goes directly in the dict + dct[name] = value + + def __delattr__(self, name): + if name == '__dict__': + raise AttributeError( + "%r object attribute '__dict__' is read-only" + % self.__class__.__name__) + + if name in self._local_type_vars: + if name in self._local_type_del_descriptors: + # A data descriptor, like a property or a slot. + type_attr = getattr(self._local_type, name, _marker) + type(type_attr).__delete__(type_attr, self) + return + # Otherwise it goes directly in the dict + + # Begin inlined function _get_dict() + dct = _local_get_dict(self) + + try: + del dct[name] + except KeyError: + raise AttributeError(name) + + def __copy__(self): + impl = self._local__impl + entry = impl.dicts[id(getcurrent())] # pylint:disable=undefined-variable + + dct = entry.localdict + duplicate = copy(dct) + + cls = type(self) + instance = cls(*impl.localargs, **impl.localkwargs) + _local__copy_dict_from(instance, impl, duplicate) + return instance + +def _local__copy_dict_from(self, impl, duplicate): + current = getcurrent() # pylint:disable=undefined-variable + currentId = id(current) + new_impl = self._local__impl + assert new_impl is not impl + entry = new_impl.dicts[currentId] + new_impl.dicts[currentId] = _localimpl_dict_entry(entry.wrgreenlet, duplicate) + +def _local_find_descriptors(self): + type_self = type(self) + gets = set() + dels = set() + set_or_del = set() + sets = set() + mro = list(type_self.mro()) + + for attr_name in dir(type_self): + # Conventionally, descriptors when called on a class + # return themself, but not all do. Notable exceptions are + # in the zope.interface package, where things like __provides__ + # return other class attributes. So we can't use getattr, and instead + # walk up the dicts + for base in mro: + bd = base.__dict__ + if attr_name in bd: + attr = bd[attr_name] + break + else: + raise AttributeError(attr_name) + + type_attr = type(attr) + if hasattr(type_attr, '__get__'): + gets.add(attr_name) + if hasattr(type_attr, '__delete__'): + dels.add(attr_name) + set_or_del.add(attr_name) + if hasattr(type_attr, '__set__'): + sets.add(attr_name) + + return (gets, dels, set_or_del, sets) + +# Cython doesn't let us use __new__, it requires +# __cinit__. But we need __new__ if we're not compiled +# (e.g., on PyPy). So we set it at runtime. Cython +# will raise an error if we're compiled. +def __new__(cls, *args, **kw): + self = super(local, cls).__new__(cls) + # We get the cls in *args for some reason + # too when we do it this way....except on PyPy3, which does + # not *unless* it's wrapped in a classmethod (which it is) + self.__cinit__(*args[1:], **kw) + return self + +try: + # PyPy2/3 and CPython handle adding a __new__ to the class + # in different ways. In CPython and PyPy3, it must be wrapped with classmethod; + # in PyPy2, it must not. In either case, the args that get passed to + # it are stil wrong. + local.__new__ = 'None' +except TypeError: # pragma: no cover + # Must be compiled + pass +else: + from gevent._compat import PYPY + from gevent._compat import PY2 + if PYPY and PY2: + local.__new__ = __new__ + else: + local.__new__ = classmethod(__new__) + + del PYPY + del PY2 + +_init() + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent._local') diff --git a/src/gevent/lock.py b/src/gevent/lock.py new file mode 100644 index 0000000..dc60d19 --- /dev/null +++ b/src/gevent/lock.py @@ -0,0 +1,260 @@ +# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details. +"""Locking primitives""" +from __future__ import absolute_import + +from gevent.hub import getcurrent +from gevent._compat import PYPY +from gevent._semaphore import Semaphore, BoundedSemaphore # pylint:disable=no-name-in-module,import-error + + +__all__ = [ + 'Semaphore', + 'DummySemaphore', + 'BoundedSemaphore', + 'RLock', +] + +# On PyPy, we don't compile the Semaphore class with Cython. Under +# Cython, each individual method holds the GIL for its entire +# duration, ensuring that no other thread can interrupt us in an +# unsafe state (only when we _do_wait do we call back into Python and +# allow switching threads). Simulate that here through the use of a manual +# lock. (We use a separate lock for each semaphore to allow sys.settrace functions +# to use locks *other* than the one being traced.) +if PYPY: + # TODO: Need to use monkey.get_original? + try: + from _thread import allocate_lock as _allocate_lock # pylint:disable=import-error,useless-suppression + from _thread import get_ident as _get_ident # pylint:disable=import-error,useless-suppression + except ImportError: + # Python 2 + from thread import allocate_lock as _allocate_lock # pylint:disable=import-error,useless-suppression + from thread import get_ident as _get_ident # pylint:disable=import-error,useless-suppression + _sem_lock = _allocate_lock() + + def untraceable(f): + # Don't allow re-entry to these functions in a single thread, as can + # happen if a sys.settrace is used + def wrapper(self): + me = _get_ident() + try: + count = self._locking[me] + except KeyError: + count = self._locking[me] = 1 + else: + count = self._locking[me] = count + 1 + if count: + return + + try: + return f(self) + finally: + count = count - 1 + if not count: + del self._locking[me] + else: + self._locking[me] = count + return wrapper + + class _OwnedLock(object): + + def __init__(self): + self._owner = None + self._block = _allocate_lock() + self._locking = {} + self._count = 0 + + @untraceable + def acquire(self): + me = _get_ident() + if self._owner == me: + self._count += 1 + return + + self._owner = me + self._block.acquire() + self._count = 1 + + @untraceable + def release(self): + self._count = count = self._count - 1 + if not count: + self._block.release() + self._owner = None + + # acquire, wait, and release all acquire the lock on entry and release it + # on exit. acquire and wait can call _do_wait, which must release it on entry + # and re-acquire it for them on exit. + class _around(object): + __slots__ = ('before', 'after') + + def __init__(self, before, after): + self.before = before + self.after = after + + def __enter__(self): + self.before() + + def __exit__(self, t, v, tb): + self.after() + + def _decorate(func, cmname): + # functools.wrap? + def wrapped(self, *args, **kwargs): + with getattr(self, cmname): + return func(self, *args, **kwargs) + return wrapped + + Semaphore._py3k_acquire = Semaphore.acquire = _decorate(Semaphore.acquire, '_lock_locked') + Semaphore.release = _decorate(Semaphore.release, '_lock_locked') + Semaphore.wait = _decorate(Semaphore.wait, '_lock_locked') + Semaphore._wait = _decorate(Semaphore._wait, '_lock_unlocked') + + _Sem_init = Semaphore.__init__ + + def __init__(self, *args, **kwargs): + l = self._lock_lock = _OwnedLock() + self._lock_locked = _around(l.acquire, l.release) + self._lock_unlocked = _around(l.release, l.acquire) + + _Sem_init(self, *args, **kwargs) + + Semaphore.__init__ = __init__ + + del _decorate + del untraceable + + +class DummySemaphore(object): + """ + DummySemaphore(value=None) -> DummySemaphore + + A Semaphore initialized with "infinite" initial value. None of its + methods ever block. + + This can be used to parameterize on whether or not to actually + guard access to a potentially limited resource. If the resource is + actually limited, such as a fixed-size thread pool, use a real + :class:`Semaphore`, but if the resource is unbounded, use an + instance of this class. In that way none of the supporting code + needs to change. + + Similarly, it can be used to parameterize on whether or not to + enforce mutual exclusion to some underlying object. If the + underlying object is known to be thread-safe itself mutual + exclusion is not needed and a ``DummySemaphore`` can be used, but + if that's not true, use a real ``Semaphore``. + """ + + # Internally this is used for exactly the purpose described in the + # documentation. gevent.pool.Pool uses it instead of a Semaphore + # when the pool size is unlimited, and + # gevent.fileobject.FileObjectThread takes a parameter that + # determines whether it should lock around IO to the underlying + # file object. + + def __init__(self, value=None): + """ + .. versionchanged:: 1.1rc3 + Accept and ignore a *value* argument for compatibility with Semaphore. + """ + + def __str__(self): + return '<%s>' % self.__class__.__name__ + + def locked(self): + """A DummySemaphore is never locked so this always returns False.""" + return False + + def release(self): + """Releasing a dummy semaphore does nothing.""" + + def rawlink(self, callback): + # XXX should still work and notify? + pass + + def unlink(self, callback): + pass + + def wait(self, timeout=None): + """Waiting for a DummySemaphore returns immediately.""" + + def acquire(self, blocking=True, timeout=None): + """ + A DummySemaphore can always be acquired immediately so this always + returns True and ignores its arguments. + + .. versionchanged:: 1.1a1 + Always return *true*. + """ + # pylint:disable=unused-argument + return True + + def __enter__(self): + pass + + def __exit__(self, typ, val, tb): + pass + + +class RLock(object): + """ + A mutex that can be acquired more than once by the same greenlet. + """ + + def __init__(self): + self._block = Semaphore(1) + self._owner = None + self._count = 0 + + def __repr__(self): + return "<%s at 0x%x _block=%s _count=%r _owner=%r)>" % ( + self.__class__.__name__, + id(self), + self._block, + self._count, + self._owner) + + def acquire(self, blocking=1): + me = getcurrent() + if self._owner is me: + self._count = self._count + 1 + return 1 + rc = self._block.acquire(blocking) + if rc: + self._owner = me + self._count = 1 + return rc + + def __enter__(self): + return self.acquire() + + def release(self): + if self._owner is not getcurrent(): + raise RuntimeError("cannot release un-acquired lock") + self._count = count = self._count - 1 + if not count: + self._owner = None + self._block.release() + + def __exit__(self, typ, value, tb): + self.release() + + # Internal methods used by condition variables + + def _acquire_restore(self, count_owner): + count, owner = count_owner + self._block.acquire() + self._count = count + self._owner = owner + + def _release_save(self): + count = self._count + self._count = 0 + owner = self._owner + self._owner = None + self._block.release() + return (count, owner) + + def _is_owned(self): + return self._owner is getcurrent() diff --git a/src/gevent/monkey.py b/src/gevent/monkey.py new file mode 100644 index 0000000..6c1fa00 --- /dev/null +++ b/src/gevent/monkey.py @@ -0,0 +1,1079 @@ +# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details. +# pylint: disable=redefined-outer-name +""" +Make the standard library cooperative. + +The primary purpose of this module is to carefully patch, in place, +portions of the standard library with gevent-friendly functions that +behave in the same way as the original (at least as closely as possible). + +The primary interface to this is the :func:`patch_all` function, which +performs all the available patches. It accepts arguments to limit the +patching to certain modules, but most programs **should** use the +default values as they receive the most wide-spread testing, and some monkey +patches have dependencies on others. + +Patching **should be done as early as possible** in the lifecycle of the +program. For example, the main module (the one that tests against +``__main__`` or is otherwise the first imported) should begin with +this code, ideally before any other imports:: + + from gevent import monkey + monkey.patch_all() + +A corollary of the above is that patching **should be done on the main +thread** and **should be done while the program is single-threaded**. + +.. tip:: + + Some frameworks, such as gunicorn, handle monkey-patching for you. + Check their documentation to be sure. + +.. warning:: + + Patching too late can lead to unreliable behaviour (for example, some + modules may still use blocking sockets) or even errors. + +Querying +======== + +Sometimes it is helpful to know if objects have been monkey-patched, and in +advanced cases even to have access to the original standard library functions. This +module provides functions for that purpose. + +- :func:`is_module_patched` +- :func:`is_object_patched` +- :func:`get_original` + +Plugins +======= + +Beginning in gevent 1.3, events are emitted during the monkey patching process. +These events are delivered first to :mod:`gevent.events` subscribers, and then +to `setuptools entry points`_. + +The following events are defined. They are listed in (roughly) the order +that a call to :func:`patch_all` will emit them. + +- :class:`gevent.events.GeventWillPatchAllEvent` +- :class:`gevent.events.GeventWillPatchModuleEvent` +- :class:`gevent.events.GeventDidPatchModuleEvent` +- :class:`gevent.events.GeventDidPatchBuiltinModulesEvent` +- :class:`gevent.events.GeventDidPatchAllEvent` + +Each event class documents the corresponding setuptools entry point name. The +entry points will be called with a single argument, the same instance of +the class that was sent to the subscribers. + +You can subscribe to the events to monitor the monkey-patching process and +to manipulate it, for example by raising :exc:`gevent.events.DoNotPatch`. + +You can also subscribe to the events to provide additional patching beyond what +gevent distributes, either for additional standard library modules, or +for third-party packages. The suggested time to do this patching is in +the subscriber for :class:`gevent.events.GeventDidPatchBuiltinModulesEvent`. +For example, to automatically patch `psycopg2`_ using `psycogreen`_ +when the call to :func:`patch_all` is made, you could write code like this:: + + # mypackage.py + def patch_psycopg(event): + from psycogreen.gevent import patch_psycopg + patch_psycopg() + +In your ``setup.py`` you would register it like this:: + + from setuptools import setup + setup( + ... + entry_points={ + 'gevent.plugins.monkey.did_patch_builtins': [ + 'psycopg2 = mypackage:patch_psycopg', + ], + }, + ... + ) + +For more complex patching, gevent provides a helper method +that you can call to replace attributes of modules with attributes of your +own modules. This function also takes care of emitting the appropriate events. + +- :func:`patch_module` + +.. _setuptools entry points: http://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins +.. _psycopg2: https://pypi.python.org/pypi/psycopg2 +.. _psycogreen: https://pypi.python.org/pypi/psycogreen + +Use as a module +=============== + +Sometimes it is useful to run existing python scripts or modules that +were not built to be gevent aware under gevent. To do so, this module +can be run as the main module, passing the script and its arguments. +For details, see the :func:`main` function. + +.. versionchanged:: 1.3b1 + Added support for plugins and began emitting will/did patch events. +""" +from __future__ import absolute_import +from __future__ import print_function +import sys + +__all__ = [ + 'patch_all', + 'patch_builtins', + 'patch_dns', + 'patch_os', + 'patch_queue', + 'patch_select', + 'patch_signal', + 'patch_socket', + 'patch_ssl', + 'patch_subprocess', + 'patch_sys', + 'patch_thread', + 'patch_time', + # query functions + 'get_original', + 'is_module_patched', + 'is_object_patched', + # plugin API + 'patch_module', + # module functions + 'main', +] + + +if sys.version_info[0] >= 3: + string_types = (str,) + PY3 = True +else: + import __builtin__ # pylint:disable=import-error + string_types = (__builtin__.basestring,) + PY3 = False + +WIN = sys.platform.startswith("win") + +class MonkeyPatchWarning(RuntimeWarning): + """ + The type of warnings we issue. + + .. versionadded:: 1.3a2 + """ + +def _notify_patch(event, _warnings=None): + # Raises DoNotPatch if we're not supposed to patch + from gevent.events import notify_and_call_entry_points + + event._warnings = _warnings + notify_and_call_entry_points(event) + +def _ignores_DoNotPatch(func): + + from functools import wraps + + @wraps(func) + def ignores(*args, **kwargs): + from gevent.events import DoNotPatch + try: + return func(*args, **kwargs) + except DoNotPatch: + return False + + return ignores + + +# maps module name -> {attribute name: original item} +# e.g. "time" -> {"sleep": built-in function sleep} +saved = {} + + +def is_module_patched(mod_name): + """ + Check if a module has been replaced with a cooperative version. + + :param str mod_name: The name of the standard library module, + e.g., ``'socket'``. + + """ + return mod_name in saved + + +def is_object_patched(mod_name, item_name): + """ + Check if an object in a module has been replaced with a + cooperative version. + + :param str mod_name: The name of the standard library module, + e.g., ``'socket'``. + :param str item_name: The name of the attribute in the module, + e.g., ``'create_connection'``. + + """ + return is_module_patched(mod_name) and item_name in saved[mod_name] + + +def _get_original(name, items): + d = saved.get(name, {}) + values = [] + module = None + for item in items: + if item in d: + values.append(d[item]) + else: + if module is None: + module = __import__(name) + values.append(getattr(module, item)) + return values + + +def get_original(mod_name, item_name): + """ + Retrieve the original object from a module. + + If the object has not been patched, then that object will still be + retrieved. + + :param str mod_name: The name of the standard library module, + e.g., ``'socket'``. + :param item_name: A string or sequence of strings naming the + attribute(s) on the module ``mod_name`` to return. + + :return: The original value if a string was given for + ``item_name`` or a sequence of original values if a + sequence was passed. + """ + if isinstance(item_name, string_types): + return _get_original(mod_name, [item_name])[0] + return _get_original(mod_name, item_name) + + +_NONE = object() + + +def patch_item(module, attr, newitem): + olditem = getattr(module, attr, _NONE) + if olditem is not _NONE: + saved.setdefault(module.__name__, {}).setdefault(attr, olditem) + setattr(module, attr, newitem) + + +def remove_item(module, attr): + olditem = getattr(module, attr, _NONE) + if olditem is _NONE: + return + saved.setdefault(module.__name__, {}).setdefault(attr, olditem) + delattr(module, attr) + + +def __call_module_hook(gevent_module, name, module, items, _warnings): + # This function can raise DoNotPatch on 'will' + + def warn(message): + _queue_warning(message, _warnings) + + func_name = '_gevent_' + name + '_monkey_patch' + try: + func = getattr(gevent_module, func_name) + except AttributeError: + func = lambda *args: None + + + func(module, items, warn) + + +def patch_module(target_module, source_module, items=None, + _warnings=None, + _notify_did_subscribers=True): + """ + patch_module(target_module, source_module, items=None) + + Replace attributes in *target_module* with the attributes of the + same name in *source_module*. + + The *source_module* can provide some attributes to customize the process: + + * ``__implements__`` is a list of attribute names to copy; if not present, + the *items* keyword argument is mandatory. + * ``_gevent_will_monkey_patch(target_module, items, warn, **kwargs)`` + * ``_gevent_did_monkey_patch(target_module, items, warn, **kwargs)`` + These two functions in the *source_module* are called *if* they exist, + before and after copying attributes, respectively. The "will" function + may modify *items*. The value of *warn* is a function that should be called + with a single string argument to issue a warning to the user. If the "will" + function raises :exc:`gevent.events.DoNotPatch`, no patching will be done. These functions + are called before any event subscribers or plugins. + + :keyword list items: A list of attribute names to replace. If + not given, this will be taken from the *source_module* ``__implements__`` + attribute. + :return: A true value if patching was done, a false value if patching was canceled. + + .. versionadded:: 1.3b1 + """ + from gevent import events + + if items is None: + items = getattr(source_module, '__implements__', None) + if items is None: + raise AttributeError('%r does not have __implements__' % source_module) + + try: + __call_module_hook(source_module, 'will', target_module, items, _warnings) + _notify_patch( + events.GeventWillPatchModuleEvent(target_module.__name__, source_module, + target_module, items), + _warnings) + except events.DoNotPatch: + return False + + for attr in items: + patch_item(target_module, attr, getattr(source_module, attr)) + + __call_module_hook(source_module, 'did', target_module, items, _warnings) + + if _notify_did_subscribers: + # We allow turning off the broadcast of the 'did' event for the benefit + # of our internal functions which need to do additional work (besides copying + # attributes) before their patch can be considered complete. + _notify_patch( + events.GeventDidPatchModuleEvent(target_module.__name__, source_module, + target_module) + ) + + return True + +def _patch_module(name, items=None, _warnings=None, _notify_did_subscribers=True): + + gevent_module = getattr(__import__('gevent.' + name), name) + module_name = getattr(gevent_module, '__target__', name) + target_module = __import__(module_name) + + patch_module(target_module, gevent_module, items=items, + _warnings=_warnings, + _notify_did_subscribers=_notify_did_subscribers) + + return gevent_module, target_module + + +def _queue_warning(message, _warnings): + # Queues a warning to show after the monkey-patching process is all done. + # Done this way to avoid extra imports during the process itself, just + # in case. If we're calling a function one-off (unusual) go ahead and do it + if _warnings is None: + _process_warnings([message]) + else: + _warnings.append(message) + + +def _process_warnings(_warnings): + import warnings + for warning in _warnings: + warnings.warn(warning, MonkeyPatchWarning, stacklevel=3) + + +def _patch_sys_std(name): + from gevent.fileobject import FileObjectThread + orig = getattr(sys, name) + if not isinstance(orig, FileObjectThread): + patch_item(sys, name, FileObjectThread(orig)) + +@_ignores_DoNotPatch +def patch_sys(stdin=True, stdout=True, stderr=True): + """ + Patch sys.std[in,out,err] to use a cooperative IO via a + threadpool. + + This is relatively dangerous and can have unintended consequences + such as hanging the process or `misinterpreting control keys`_ + when :func:`input` and :func:`raw_input` are used. :func:`patch_all` + does *not* call this function by default. + + This method does nothing on Python 3. The Python 3 interpreter + wants to flush the TextIOWrapper objects that make up + stderr/stdout at shutdown time, but using a threadpool at that + time leads to a hang. + + .. _`misinterpreting control keys`: https://github.com/gevent/gevent/issues/274 + """ + # test__issue6.py demonstrates the hang if these lines are removed; + # strangely enough that test passes even without monkey-patching sys + if PY3: + items = None + else: + items = set([('stdin' if stdin else None), + ('stdout' if stdout else None), + ('stderr' if stderr else None)]) + items.discard(None) + items = list(items) + + if not items: + return + + from gevent import events + _notify_patch(events.GeventWillPatchModuleEvent('sys', None, sys, + items)) + + for item in items: + _patch_sys_std(item) + + _notify_patch(events.GeventDidPatchModuleEvent('sys', None, sys)) + +@_ignores_DoNotPatch +def patch_os(): + """ + Replace :func:`os.fork` with :func:`gevent.fork`, and, on POSIX, + :func:`os.waitpid` with :func:`gevent.os.waitpid` (if the + environment variable ``GEVENT_NOWAITPID`` is not defined). Does + nothing if fork is not available. + + .. caution:: This method must be used with :func:`patch_signal` to have proper `SIGCHLD` + handling and thus correct results from ``waitpid``. + :func:`patch_all` calls both by default. + + .. caution:: For `SIGCHLD` handling to work correctly, the event loop must run. + The easiest way to help ensure this is to use :func:`patch_all`. + """ + _patch_module('os') + + +@_ignores_DoNotPatch +def patch_queue(): + """ + On Python 3.7 and above, replace :class:`queue.SimpleQueue` (implemented + in C) with its Python counterpart. + + .. versionadded:: 1.3.5 + """ + + import gevent.queue + if 'SimpleQueue' in gevent.queue.__all__: + _patch_module('queue', items=['SimpleQueue']) + + +@_ignores_DoNotPatch +def patch_time(): + """ + Replace :func:`time.sleep` with :func:`gevent.sleep`. + """ + _patch_module('time') + + +def _patch_existing_locks(threading): + if len(list(threading.enumerate())) != 1: + return + try: + tid = threading.get_ident() + except AttributeError: + tid = threading._get_ident() + rlock_type = type(threading.RLock()) + try: + import importlib._bootstrap + except ImportError: + class _ModuleLock(object): + pass + else: + _ModuleLock = importlib._bootstrap._ModuleLock # python 2 pylint: disable=no-member + # It might be possible to walk up all the existing stack frames to find + # locked objects...at least if they use `with`. To be sure, we look at every object + # Since we're supposed to be done very early in the process, there shouldn't be + # too many. + + # By definition there's only one thread running, so the various + # owner attributes were the old (native) thread id. Make it our + # current greenlet id so that when it wants to unlock and compare + # self.__owner with _get_ident(), they match. + gc = __import__('gc') + for o in gc.get_objects(): + if isinstance(o, rlock_type): + if hasattr(o, '_owner'): # Py3 + if o._owner is not None: + o._owner = tid + else: + if o._RLock__owner is not None: + o._RLock__owner = tid + elif isinstance(o, _ModuleLock): + if o.owner is not None: + o.owner = tid + +@_ignores_DoNotPatch +def patch_thread(threading=True, _threading_local=True, Event=True, logging=True, + existing_locks=True, + _warnings=None): + """ + patch_thread(threading=True, _threading_local=True, Event=True, logging=True, existing_locks=True) -> None + + Replace the standard :mod:`thread` module to make it greenlet-based. + + :keyword bool threading: When True (the default), + also patch :mod:`threading`. + :keyword bool _threading_local: When True (the default), + also patch :class:`_threading_local.local`. + :keyword bool logging: When True (the default), also patch locks + taken if the logging module has been configured. + + :keyword bool existing_locks: When True (the default), and the + process is still single threaded, make sure that any + :class:`threading.RLock` (and, under Python 3, :class:`importlib._bootstrap._ModuleLock`) + instances that are currently locked can be properly unlocked. + + .. caution:: + Monkey-patching :mod:`thread` and using + :class:`multiprocessing.Queue` or + :class:`concurrent.futures.ProcessPoolExecutor` (which uses a + ``Queue``) will hang the process. + + .. versionchanged:: 1.1b1 + Add *logging* and *existing_locks* params. + .. versionchanged:: 1.3a2 + ``Event`` defaults to True. + """ + # XXX: Simplify + # pylint:disable=too-many-branches,too-many-locals,too-many-statements + + # Description of the hang: + # There is an incompatibility with patching 'thread' and the 'multiprocessing' module: + # The problem is that multiprocessing.queues.Queue uses a half-duplex multiprocessing.Pipe, + # which is implemented with os.pipe() and _multiprocessing.Connection. os.pipe isn't patched + # by gevent, as it returns just a fileno. _multiprocessing.Connection is an internal implementation + # class implemented in C, which exposes a 'poll(timeout)' method; under the covers, this issues a + # (blocking) select() call: hence the need for a real thread. Except for that method, we could + # almost replace Connection with gevent.fileobject.SocketAdapter, plus a trivial + # patch to os.pipe (below). Sigh, so close. (With a little work, we could replicate that method) + + # import os + # import fcntl + # os_pipe = os.pipe + # def _pipe(): + # r, w = os_pipe() + # fcntl.fcntl(r, fcntl.F_SETFL, os.O_NONBLOCK) + # fcntl.fcntl(w, fcntl.F_SETFL, os.O_NONBLOCK) + # return r, w + # os.pipe = _pipe + + # The 'threading' module copies some attributes from the + # thread module the first time it is imported. If we patch 'thread' + # before that happens, then we store the wrong values in 'saved', + # So if we're going to patch threading, we either need to import it + # before we patch thread, or manually clean up the attributes that + # are in trouble. The latter is tricky because of the different names + # on different versions. + if threading: + threading_mod = __import__('threading') + # Capture the *real* current thread object before + # we start returning DummyThread objects, for comparison + # to the main thread. + orig_current_thread = threading_mod.current_thread() + else: + threading_mod = None + gevent_threading_mod = None + orig_current_thread = None + + gevent_thread_mod, thread_mod = _patch_module('thread', + _warnings=_warnings, _notify_did_subscribers=False) + + if threading: + gevent_threading_mod, _ = _patch_module('threading', + _warnings=_warnings, _notify_did_subscribers=False) + + if Event: + from gevent.event import Event + patch_item(threading_mod, 'Event', Event) + # Python 2 had `Event` as a function returning + # the private class `_Event`. Some code may be relying + # on that. + if hasattr(threading_mod, '_Event'): + patch_item(threading_mod, '_Event', Event) + + if existing_locks: + _patch_existing_locks(threading_mod) + + if logging and 'logging' in sys.modules: + logging = __import__('logging') + patch_item(logging, '_lock', threading_mod.RLock()) + for wr in logging._handlerList: + # In py26, these are actual handlers, not weakrefs + handler = wr() if callable(wr) else wr + if handler is None: + continue + if not hasattr(handler, 'lock'): + raise TypeError("Unknown/unsupported handler %r" % handler) + handler.lock = threading_mod.RLock() + + if _threading_local: + _threading_local = __import__('_threading_local') + from gevent.local import local + patch_item(_threading_local, 'local', local) + + def make_join_func(thread, thread_greenlet): + from gevent.hub import sleep + from time import time + + def join(timeout=None): + end = None + if threading_mod.current_thread() is thread: + raise RuntimeError("Cannot join current thread") + if thread_greenlet is not None and thread_greenlet.dead: + return + if not thread.is_alive(): + return + + if timeout: + end = time() + timeout + + while thread.is_alive(): + if end is not None and time() > end: + return + sleep(0.01) + return join + + if threading: + from gevent.threading import main_native_thread + + for thread in threading_mod._active.values(): + if thread == main_native_thread(): + continue + thread.join = make_join_func(thread, None) + + if sys.version_info[:2] >= (3, 4): + + # Issue 18808 changes the nature of Thread.join() to use + # locks. This means that a greenlet spawned in the main thread + # (which is already running) cannot wait for the main thread---it + # hangs forever. We patch around this if possible. See also + # gevent.threading. + greenlet = __import__('greenlet') + + if orig_current_thread == threading_mod.main_thread(): + main_thread = threading_mod.main_thread() + _greenlet = main_thread._greenlet = greenlet.getcurrent() + + main_thread.join = make_join_func(main_thread, _greenlet) + + # Patch up the ident of the main thread to match. This + # matters if threading was imported before monkey-patching + # thread + oldid = main_thread.ident + main_thread._ident = threading_mod.get_ident() + if oldid in threading_mod._active: + threading_mod._active[main_thread.ident] = threading_mod._active[oldid] + if oldid != main_thread.ident: + del threading_mod._active[oldid] + else: + _queue_warning("Monkey-patching not on the main thread; " + "threading.main_thread().join() will hang from a greenlet", + _warnings) + + from gevent import events + _notify_patch(events.GeventDidPatchModuleEvent('thread', gevent_thread_mod, thread_mod)) + _notify_patch(events.GeventDidPatchModuleEvent('threading', gevent_threading_mod, threading_mod)) + +@_ignores_DoNotPatch +def patch_socket(dns=True, aggressive=True): + """ + Replace the standard socket object with gevent's cooperative + sockets. + + :keyword bool dns: When true (the default), also patch address + resolution functions in :mod:`socket`. See :doc:`/dns` for details. + """ + from gevent import socket + # Note: although it seems like it's not strictly necessary to monkey patch 'create_connection', + # it's better to do it. If 'create_connection' was not monkey patched, but the rest of socket module + # was, create_connection would still use "green" getaddrinfo and "green" socket. + # However, because gevent.socket.socket.connect is a Python function, the exception raised by it causes + # _socket object to be referenced by the frame, thus causing the next invocation of bind(source_address) to fail. + if dns: + items = socket.__implements__ # pylint:disable=no-member + else: + items = set(socket.__implements__) - set(socket.__dns__) # pylint:disable=no-member + _patch_module('socket', items=items) + if aggressive: + if 'ssl' not in socket.__implements__: # pylint:disable=no-member + remove_item(socket, 'ssl') + +@_ignores_DoNotPatch +def patch_dns(): + """ + Replace :doc:`DNS functions ` in :mod:`socket` with + cooperative versions. + + This is only useful if :func:`patch_socket` has been called and is + done automatically by that method if requested. + """ + from gevent import socket + _patch_module('socket', items=socket.__dns__) # pylint:disable=no-member + + +def _find_module_refs(to, excluding_names=()): + # Looks specifically for module-level references, + # i.e., 'from foo import Bar'. We define a module reference + # as a dict (subclass) that also has a __name__ attribute. + # This does not handle subclasses, but it does find them. + # Returns two sets. The first is modules (name, file) that were + # found. The second is subclasses that were found. + gc = __import__('gc') + direct_ref_modules = set() + subclass_modules = set() + + def report(mod): + return mod['__name__'], mod.get('__file__', '') + + for r in gc.get_referrers(to): + if isinstance(r, dict) and '__name__' in r: + if r['__name__'] in excluding_names: + continue + + for v in r.values(): + if v is to: + direct_ref_modules.add(report(r)) + elif isinstance(r, type) and to in r.__bases__ and 'gevent.' not in r.__module__: + subclass_modules.add(r) + + return direct_ref_modules, subclass_modules + +@_ignores_DoNotPatch +def patch_ssl(_warnings=None, _first_time=True): + """ + patch_ssl() -> None + + Replace :class:`ssl.SSLSocket` object and socket wrapping functions in + :mod:`ssl` with cooperative versions. + + This is only useful if :func:`patch_socket` has been called. + """ + may_need_warning = ( + _first_time + and sys.version_info[:2] >= (3, 6) + and 'ssl' in sys.modules + and hasattr(sys.modules['ssl'], 'SSLContext')) + # Previously, we didn't warn on Python 2 if pkg_resources has been imported + # because that imports ssl and it's commonly used for namespace packages, + # which typically means we're still in some early part of the import cycle. + # However, with our new more discriminating check, that no longer seems to be a problem. + # Prior to 3.6, we don't have the RecursionError problem, and prior to 3.7 we don't have the + # SSLContext.sslsocket_class/SSLContext.sslobject_class problem. + + gevent_mod, _ = _patch_module('ssl', _warnings=_warnings) + if may_need_warning: + direct_ref_modules, subclass_modules = _find_module_refs( + gevent_mod.orig_SSLContext, + excluding_names=('ssl', 'gevent.ssl', 'gevent._ssl3', 'gevent._sslgte279')) + if direct_ref_modules or subclass_modules: + # Normally you don't want to have dynamic warning strings, because + # the cache in the warning module is based on the string. But we + # specifically only do this the first time we patch ourself, so it's + # ok. + direct_ref_mod_str = subclass_str = '' + if direct_ref_modules: + direct_ref_mod_str = 'Modules that had direct imports (NOT patched): %s. ' % ([ + "%s (%s)" % (name, fname) + for name, fname in direct_ref_modules + ]) + if subclass_modules: + subclass_str = 'Subclasses (NOT patched): %s. ' % ([ + str(t) for t in subclass_modules + ]) + _queue_warning( + 'Monkey-patching ssl after ssl has already been imported ' + 'may lead to errors, including RecursionError on Python 3.6. ' + 'It may also silently lead to incorrect behaviour on Python 3.7. ' + 'Please monkey-patch earlier. ' + 'See https://github.com/gevent/gevent/issues/1016. ' + + direct_ref_mod_str + subclass_str, + _warnings) + + +@_ignores_DoNotPatch +def patch_select(aggressive=True): + """ + Replace :func:`select.select` with :func:`gevent.select.select` + and :func:`select.poll` with :class:`gevent.select.poll` (where available). + + If ``aggressive`` is true (the default), also remove other + blocking functions from :mod:`select` and (on Python 3.4 and + above) :mod:`selectors`: + + - :func:`select.epoll` + - :func:`select.kqueue` + - :func:`select.kevent` + - :func:`select.devpoll` (Python 3.5+) + - :class:`selectors.EpollSelector` + - :class:`selectors.KqueueSelector` + - :class:`selectors.DevpollSelector` (Python 3.5+) + """ + + source_mod, target_mod = _patch_module('select', _notify_did_subscribers=False) + if aggressive: + select = target_mod + # since these are blocking we're removing them here. This makes some other + # modules (e.g. asyncore) non-blocking, as they use select that we provide + # when none of these are available. + remove_item(select, 'epoll') + remove_item(select, 'kqueue') + remove_item(select, 'kevent') + remove_item(select, 'devpoll') + + if sys.version_info[:2] >= (3, 4): + # Python 3 wants to use `select.select` as a member function, + # leading to this error in selectors.py (because gevent.select.select is + # not a builtin and doesn't get the magic auto-static that they do) + # r, w, _ = self._select(self._readers, self._writers, [], timeout) + # TypeError: select() takes from 3 to 4 positional arguments but 5 were given + # Note that this obviously only happens if selectors was imported after we had patched + # select; but there is a code path that leads to it being imported first (but now we've + # patched select---so we can't compare them identically) + select = target_mod # Should be gevent-patched now + orig_select_select = get_original('select', 'select') + assert select.select is not orig_select_select + selectors = __import__('selectors') + if selectors.SelectSelector._select in (select.select, orig_select_select): + def _select(self, *args, **kwargs): # pylint:disable=unused-argument + return select.select(*args, **kwargs) + selectors.SelectSelector._select = _select + _select._gevent_monkey = True + + # Python 3.7 refactors the poll-like selectors to use a common + # base class and capture a reference to select.poll, etc, at + # import time. selectors tends to get imported early + # (importing 'platform' does it: platform -> subprocess -> selectors), + # so we need to clean that up. + if hasattr(selectors, 'PollSelector') and hasattr(selectors.PollSelector, '_selector_cls'): + selectors.PollSelector._selector_cls = select.poll + + if aggressive: + # If `selectors` had already been imported before we removed + # select.epoll|kqueue|devpoll, these may have been defined in terms + # of those functions. They'll fail at runtime. + remove_item(selectors, 'EpollSelector') + remove_item(selectors, 'KqueueSelector') + remove_item(selectors, 'DevpollSelector') + selectors.DefaultSelector = selectors.SelectSelector + + from gevent import events + _notify_patch(events.GeventDidPatchModuleEvent('select', source_mod, target_mod)) + +@_ignores_DoNotPatch +def patch_subprocess(): + """ + Replace :func:`subprocess.call`, :func:`subprocess.check_call`, + :func:`subprocess.check_output` and :class:`subprocess.Popen` with + :mod:`cooperative versions `. + + .. note:: + On Windows under Python 3, the API support may not completely match + the standard library. + + """ + _patch_module('subprocess') + +@_ignores_DoNotPatch +def patch_builtins(): + """ + Make the builtin :func:`__import__` function `greenlet safe`_ under Python 2. + + .. note:: + This does nothing under Python 3 as it is not necessary. Python 3 features + improved import locks that are per-module, not global. + + .. _greenlet safe: https://github.com/gevent/gevent/issues/108 + + """ + if sys.version_info[:2] < (3, 3): + _patch_module('builtins') + +@_ignores_DoNotPatch +def patch_signal(): + """ + Make the :func:`signal.signal` function work with a :func:`monkey-patched os `. + + .. caution:: This method must be used with :func:`patch_os` to have proper ``SIGCHLD`` + handling. :func:`patch_all` calls both by default. + + .. caution:: For proper ``SIGCHLD`` handling, you must yield to the event loop. + Using :func:`patch_all` is the easiest way to ensure this. + + .. seealso:: :mod:`gevent.signal` + """ + _patch_module("signal") + + +def _check_repatching(**module_settings): + _warnings = [] + key = '_gevent_saved_patch_all' + del module_settings['kwargs'] + if saved.get(key, module_settings) != module_settings: + _queue_warning("Patching more than once will result in the union of all True" + " parameters being patched", + _warnings) + + first_time = key not in saved + saved[key] = module_settings + return _warnings, first_time, module_settings + + +def _subscribe_signal_os(will_patch_all): + if will_patch_all.will_patch_module('signal') and not will_patch_all.will_patch_module('os'): + warnings = will_patch_all._warnings # Internal + _queue_warning('Patching signal but not os will result in SIGCHLD handlers' + ' installed after this not being called and os.waitpid may not' + ' function correctly if gevent.subprocess is used. This may raise an' + ' error in the future.', + warnings) + +def patch_all(socket=True, dns=True, time=True, select=True, thread=True, os=True, ssl=True, + httplib=False, # Deprecated, to be removed. + subprocess=True, sys=False, aggressive=True, Event=True, + builtins=True, signal=True, + queue=True, + **kwargs): + """ + Do all of the default monkey patching (calls every other applicable + function in this module). + + :return: A true value if patching all modules wasn't cancelled, a false + value if it was. + + .. versionchanged:: 1.1 + Issue a :mod:`warning ` if this function is called multiple times + with different arguments. The second and subsequent calls will only add more + patches, they can never remove existing patches by setting an argument to ``False``. + .. versionchanged:: 1.1 + Issue a :mod:`warning ` if this function is called with ``os=False`` + and ``signal=True``. This will cause SIGCHLD handlers to not be called. This may + be an error in the future. + .. versionchanged:: 1.3a2 + ``Event`` defaults to True. + .. versionchanged:: 1.3b1 + Defined the return values. + .. versionchanged:: 1.3b1 + Add ``**kwargs`` for the benefit of event subscribers. CAUTION: gevent may add + and interpret additional arguments in the future, so it is suggested to use prefixes + for kwarg values to be interpreted by plugins, for example, `patch_all(mylib_futures=True)`. + .. versionchanged:: 1.3.5 + Add *queue*, defaulting to True, for Python 3.7. + """ + # pylint:disable=too-many-locals,too-many-branches + + # Check to see if they're changing the patched list + _warnings, first_time, modules_to_patch = _check_repatching(**locals()) + if not _warnings and not first_time: + # Nothing to do, identical args to what we just + # did + return + + from gevent import events + try: + _notify_patch(events.GeventWillPatchAllEvent(modules_to_patch, kwargs), _warnings) + except events.DoNotPatch: + return False + + # order is important + if os: + patch_os() + if time: + patch_time() + if thread: + patch_thread(Event=Event, _warnings=_warnings) + # sys must be patched after thread. in other cases threading._shutdown will be + # initiated to _MainThread with real thread ident + if sys: + patch_sys() + if socket: + patch_socket(dns=dns, aggressive=aggressive) + if select: + patch_select(aggressive=aggressive) + if ssl: + patch_ssl(_warnings=_warnings, _first_time=first_time) + if httplib: + raise ValueError('gevent.httplib is no longer provided, httplib must be False') + if subprocess: + patch_subprocess() + if builtins: + patch_builtins() + if signal: + patch_signal() + if queue: + patch_queue() + + _notify_patch(events.GeventDidPatchBuiltinModulesEvent(modules_to_patch, kwargs), _warnings) + _notify_patch(events.GeventDidPatchAllEvent(modules_to_patch, kwargs), _warnings) + + _process_warnings(_warnings) + return True + + +def main(): + args = {} + argv = sys.argv[1:] + verbose = False + script_help, patch_all_args, modules = _get_script_help() + while argv and argv[0].startswith('--'): + option = argv[0][2:] + if option == 'verbose': + verbose = True + elif option.startswith('no-') and option.replace('no-', '') in patch_all_args: + args[option[3:]] = False + elif option in patch_all_args: + args[option] = True + if option in modules: + for module in modules: + args.setdefault(module, False) + else: + sys.exit(script_help + '\n\n' + 'Cannot patch %r' % option) + del argv[0] + # TODO: break on -- + if verbose: + import pprint + import os + print('gevent.monkey.patch_all(%s)' % ', '.join('%s=%s' % item for item in args.items())) + print('sys.version=%s' % (sys.version.strip().replace('\n', ' '), )) + print('sys.path=%s' % pprint.pformat(sys.path)) + print('sys.modules=%s' % pprint.pformat(sorted(sys.modules.keys()))) + print('cwd=%s' % os.getcwd()) + + patch_all(**args) + if argv: + sys.argv = argv + import runpy + # Use runpy.run_path to closely (exactly) match what the + # interpreter does given 'python '. This includes allowing + # passing .pyc/.pyo files and packages with a __main__ and + # potentially even zip files. Previously we used exec, which only + # worked if we directly read a python source file. + runpy.run_path(sys.argv[0], + run_name='__main__') + else: + print(script_help) + + +def _get_script_help(): + # pylint:disable=deprecated-method + import inspect + try: + getter = inspect.getfullargspec # deprecated in 3.5, un-deprecated in 3.6 + except AttributeError: + getter = inspect.getargspec + patch_all_args = getter(patch_all)[0] + modules = [x for x in patch_all_args if 'patch_' + x in globals()] + script_help = """gevent.monkey - monkey patch the standard modules to use gevent. + +USAGE: ``python -m gevent.monkey [MONKEY OPTIONS] script [SCRIPT OPTIONS]`` + +If no OPTIONS present, monkey patches all the modules it can patch. +You can exclude a module with --no-module, e.g. --no-thread. You can +specify a module to patch with --module, e.g. --socket. In the latter +case only the modules specified on the command line will be patched. + +.. versionchanged:: 1.3b1 + The *script* argument can now be any argument that can be passed to `runpy.run_path`, + just like the interpreter itself does, for example a package directory containing ``__main__.py``. + Previously it had to be the path to + a .py source file. + +MONKEY OPTIONS: ``--verbose %s``""" % ', '.join('--[no-]%s' % m for m in modules) + return script_help, patch_all_args, modules + +main.__doc__ = _get_script_help()[0] + +if __name__ == '__main__': + main() diff --git a/src/gevent/os.py b/src/gevent/os.py new file mode 100644 index 0000000..3980f32 --- /dev/null +++ b/src/gevent/os.py @@ -0,0 +1,510 @@ +""" +Low-level operating system functions from :mod:`os`. + +Cooperative I/O +=============== + +This module provides cooperative versions of :func:`os.read` and +:func:`os.write`. These functions are *not* monkey-patched; you +must explicitly call them or monkey patch them yourself. + +POSIX functions +--------------- + +On POSIX, non-blocking IO is available. + +- :func:`nb_read` +- :func:`nb_write` +- :func:`make_nonblocking` + +All Platforms +------------- + +On non-POSIX platforms (e.g., Windows), non-blocking IO is not +available. On those platforms (and on POSIX), cooperative IO can +be done with the threadpool. + +- :func:`tp_read` +- :func:`tp_write` + +Child Processes +=============== + +The functions :func:`fork` and (on POSIX) :func:`forkpty` and :func:`waitpid` can be used +to manage child processes. + +.. warning:: + + Forking a process that uses greenlets does not eliminate all non-running + greenlets. Any that were scheduled in the hub of the forking thread in the parent + remain scheduled in the child; compare this to how normal threads operate. (This behaviour + may change is a subsequent major release.) +""" + +from __future__ import absolute_import + +import os +import sys +from gevent.hub import _get_hub_noargs as get_hub +from gevent.hub import reinit +from gevent._config import config +from gevent._compat import PY3 +from gevent._util import copy_globals +import errno + +EAGAIN = getattr(errno, 'EAGAIN', 11) + +try: + import fcntl +except ImportError: + fcntl = None + +__implements__ = ['fork'] +__extensions__ = ['tp_read', 'tp_write'] + +_read = os.read +_write = os.write + + +ignored_errors = [EAGAIN, errno.EINTR] + + +if fcntl: + + __extensions__ += ['make_nonblocking', 'nb_read', 'nb_write'] + + def make_nonblocking(fd): + """Put the file descriptor *fd* into non-blocking mode if + possible. + + :return: A boolean value that evaluates to True if successful. + """ + flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) + if not bool(flags & os.O_NONBLOCK): + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + return True + + def nb_read(fd, n): + """ + Read up to *n* bytes from file descriptor *fd*. Return a + byte string containing the bytes read, which may be shorter than + *n*. If end-of-file is reached, an empty string is returned. + + The descriptor must be in non-blocking mode. + """ + hub = None + event = None + try: + while 1: + try: + result = _read(fd, n) + return result + except OSError as e: + if e.errno not in ignored_errors: + raise + if not PY3: + sys.exc_clear() + if hub is None: + hub = get_hub() + event = hub.loop.io(fd, 1) + hub.wait(event) + finally: + if event is not None: + event.close() + event = None + hub = None + + + def nb_write(fd, buf): + """ + Write some number of bytes from buffer *buf* to file + descriptor *fd*. Return the number of bytes written, which may + be less than the length of *buf*. + + The file descriptor must be in non-blocking mode. + """ + hub = None + event = None + try: + while 1: + try: + result = _write(fd, buf) + return result + except OSError as e: + if e.errno not in ignored_errors: + raise + if not PY3: + sys.exc_clear() + if hub is None: + hub = get_hub() + event = hub.loop.io(fd, 2) + hub.wait(event) + finally: + if event is not None: + event.close() + event = None + hub = None + + +def tp_read(fd, n): + """Read up to *n* bytes from file descriptor *fd*. Return a string + containing the bytes read. If end-of-file is reached, an empty string + is returned. + + Reading is done using the threadpool. + """ + return get_hub().threadpool.apply(_read, (fd, n)) + + +def tp_write(fd, buf): + """Write bytes from buffer *buf* to file descriptor *fd*. Return the + number of bytes written. + + Writing is done using the threadpool. + """ + return get_hub().threadpool.apply(_write, (fd, buf)) + + +if hasattr(os, 'fork'): + # pylint:disable=function-redefined,redefined-outer-name + + _raw_fork = os.fork + + def fork_gevent(): + """ + Forks the process using :func:`os.fork` and prepares the + child process to continue using gevent before returning. + + .. note:: + + The PID returned by this function may not be waitable with + either the original :func:`os.waitpid` or this module's + :func:`waitpid` and it may not generate SIGCHLD signals if + libev child watchers are or ever have been in use. For + example, the :mod:`gevent.subprocess` module uses libev + child watchers (which parts of gevent use libev child + watchers is subject to change at any time). Most + applications should use :func:`fork_and_watch`, which is + monkey-patched as the default replacement for + :func:`os.fork` and implements the ``fork`` function of + this module by default, unless the environment variable + ``GEVENT_NOWAITPID`` is defined before this module is + imported. + + .. versionadded:: 1.1b2 + """ + result = _raw_fork() + if not result: + reinit() + return result + + def fork(): + """ + A wrapper for :func:`fork_gevent` for non-POSIX platforms. + """ + return fork_gevent() + + if hasattr(os, 'forkpty'): + _raw_forkpty = os.forkpty + + def forkpty_gevent(): + """ + Forks the process using :func:`os.forkpty` and prepares the + child process to continue using gevent before returning. + + Returns a tuple (pid, master_fd). The `master_fd` is *not* put into + non-blocking mode. + + Availability: Some Unix systems. + + .. seealso:: This function has the same limitations as :func:`fork_gevent`. + + .. versionadded:: 1.1b5 + """ + pid, master_fd = _raw_forkpty() + if not pid: + reinit() + return pid, master_fd + + forkpty = forkpty_gevent + + __implements__.append('forkpty') + __extensions__.append("forkpty_gevent") + + if hasattr(os, 'WNOWAIT') or hasattr(os, 'WNOHANG'): + # We can only do this on POSIX + import time + + _waitpid = os.waitpid + _WNOHANG = os.WNOHANG + + # replaced by the signal module. + _on_child_hook = lambda: None + + # {pid -> watcher or tuple(pid, rstatus, timestamp)} + _watched_children = {} + + def _on_child(watcher, callback): + # XXX: Could handle tracing here by not stopping + # until the pid is terminated + watcher.stop() + try: + _watched_children[watcher.pid] = (watcher.pid, watcher.rstatus, time.time()) + if callback: + callback(watcher) + # dispatch an "event"; used by gevent.signal.signal + _on_child_hook() + # now is as good a time as any to reap children + _reap_children() + finally: + watcher.close() + + def _reap_children(timeout=60): + # Remove all the dead children that haven't been waited on + # for the *timeout* seconds. + # Some platforms queue delivery of SIGCHLD for all children that die; + # in that case, a well-behaved application should call waitpid() for each + # signal. + # Some platforms (linux) only guarantee one delivery if multiple children + # die. On that platform, the well-behave application calls waitpid() in a loop + # until it gets back -1, indicating no more dead children need to be waited for. + # In either case, waitpid should be called the same number of times as dead children, + # thus removing all the watchers when a SIGCHLD arrives. The (generous) timeout + # is to work with applications that neglect to call waitpid and prevent "unlimited" + # growth. + # Note that we don't watch for the case of pid wraparound. That is, we fork a new + # child with the same pid as an existing watcher, but the child is already dead, + # just not waited on yet. + now = time.time() + oldest_allowed = now - timeout + dead = [pid for pid, val + in _watched_children.items() + if isinstance(val, tuple) and val[2] < oldest_allowed] + for pid in dead: + del _watched_children[pid] + + def waitpid(pid, options): + """ + Wait for a child process to finish. + + If the child process was spawned using + :func:`fork_and_watch`, then this function behaves + cooperatively. If not, it *may* have race conditions; see + :func:`fork_gevent` for more information. + + The arguments are as for the underlying + :func:`os.waitpid`. Some combinations of *options* may not + be supported cooperatively (as of 1.1 that includes + WUNTRACED). Using a *pid* of 0 to request waiting on only processes + from the current process group is not cooperative. + + Availability: POSIX. + + .. versionadded:: 1.1b1 + .. versionchanged:: 1.2a1 + More cases are handled in a cooperative manner. + """ + # pylint: disable=too-many-return-statements + # XXX Does not handle tracing children + + # So long as libev's loop doesn't run, it's OK to add + # child watchers. The SIGCHLD handler only feeds events + # for the next iteration of the loop to handle. (And the + # signal handler itself is only called from the next loop + # iteration.) + + if pid <= 0: + # magic functions for multiple children. + if pid == -1: + # Any child. If we have one that we're watching and that finished, + # we will use that one. Otherwise, let the OS take care of it. + for k, v in _watched_children.items(): + if isinstance(v, tuple): + pid = k + break + if pid <= 0: + # We didn't have one that was ready. If there are + # no funky options set, and the pid was -1 + # (meaning any process, not 0, which means process + # group--- libev doesn't know about process + # groups) then we can use a child watcher of pid 0; otherwise, + # pass through to the OS. + if pid == -1 and options == 0: + hub = get_hub() + with hub.loop.child(0, False) as watcher: + hub.wait(watcher) + return watcher.rpid, watcher.rstatus + # There were funky options/pid, so we must go to the OS. + return _waitpid(pid, options) + + if pid in _watched_children: + # yes, we're watching it + + # Note that the remainder of this code must be careful to NOT + # yield to the event loop except at well known times, or + # we have a race condition between the _on_child callback and the + # code here that could lead to a process to hang. + if options & _WNOHANG or isinstance(_watched_children[pid], tuple): + # We're either asked not to block, or it already finished, in which + # case blocking doesn't matter + result = _watched_children[pid] + if isinstance(result, tuple): + # it finished. libev child watchers + # are one-shot + del _watched_children[pid] + return result[:2] + # it's not finished + return (0, 0) + + # Ok, we need to "block". Do so via a watcher so that we're + # cooperative. We know it's our child, etc, so this should work. + watcher = _watched_children[pid] + # We can't start a watcher that's already started, + # so we can't reuse the existing watcher. Notice that the + # old watcher must not have fired already, or during this time, but + # only after we successfully `start()` the watcher. So this must + # not yield to the event loop. + with watcher.loop.child(pid, False) as new_watcher: + get_hub().wait(new_watcher) + # Ok, so now the new watcher is done. That means + # the old watcher's callback (_on_child) should + # have fired, potentially taking this child out of + # _watched_children (but that could depend on how + # many callbacks there were to run, so use the + # watcher object directly; libev sets all the + # watchers at the same time). + return watcher.rpid, watcher.rstatus + + # we're not watching it and it may not even be our child, + # so we must go to the OS to be sure to get the right semantics (exception) + # XXX + # libuv has a race condition because the signal + # handler is a Python function, so the InterruptedError + # is raised before the signal handler runs and calls the + # child watcher + # we're not watching it + return _waitpid(pid, options) + + def fork_and_watch(callback=None, loop=None, ref=False, fork=fork_gevent): + """ + Fork a child process and start a child watcher for it in the parent process. + + This call cooperates with :func:`waitpid` to enable cooperatively waiting + for children to finish. When monkey-patching, these functions are patched in as + :func:`os.fork` and :func:`os.waitpid`, respectively. + + In the child process, this function calls :func:`gevent.hub.reinit` before returning. + + Availability: POSIX. + + :keyword callback: If given, a callable that will be called with the child watcher + when the child finishes. + :keyword loop: The loop to start the watcher in. Defaults to the + loop of the current hub. + :keyword fork: The fork function. Defaults to :func:`the one defined in this + module ` (which automatically calls :func:`gevent.hub.reinit`). + Pass the builtin :func:`os.fork` function if you do not need to + initialize gevent in the child process. + + .. versionadded:: 1.1b1 + .. seealso:: + :func:`gevent.monkey.get_original` To access the builtin :func:`os.fork`. + """ + pid = fork() + if pid: + # parent + loop = loop or get_hub().loop + watcher = loop.child(pid, ref=ref) + _watched_children[pid] = watcher + watcher.start(_on_child, watcher, callback) + return pid + + __extensions__.append('fork_and_watch') + __extensions__.append('fork_gevent') + + if 'forkpty' in __implements__: + def forkpty_and_watch(callback=None, loop=None, ref=False, forkpty=forkpty_gevent): + """ + Like :func:`fork_and_watch`, except using :func:`forkpty_gevent`. + + Availability: Some Unix systems. + + .. versionadded:: 1.1b5 + """ + result = [] + + def _fork(): + pid_and_fd = forkpty() + result.append(pid_and_fd) + return pid_and_fd[0] + fork_and_watch(callback, loop, ref, _fork) + return result[0] + + __extensions__.append('forkpty_and_watch') + + # Watch children by default + if not config.disable_watch_children: + # Broken out into separate functions instead of simple name aliases + # for documentation purposes. + def fork(*args, **kwargs): + """ + Forks a child process and starts a child watcher for it in the + parent process so that ``waitpid`` and SIGCHLD work as expected. + + This implementation of ``fork`` is a wrapper for :func:`fork_and_watch` + when the environment variable ``GEVENT_NOWAITPID`` is *not* defined. + This is the default and should be used by most applications. + + .. versionchanged:: 1.1b2 + """ + # take any args to match fork_and_watch + return fork_and_watch(*args, **kwargs) + + if 'forkpty' in __implements__: + def forkpty(*args, **kwargs): + """ + Like :func:`fork`, but using :func:`forkpty_gevent`. + + This implementation of ``forkpty`` is a wrapper for :func:`forkpty_and_watch` + when the environment variable ``GEVENT_NOWAITPID`` is *not* defined. + This is the default and should be used by most applications. + + .. versionadded:: 1.1b5 + """ + # take any args to match fork_and_watch + return forkpty_and_watch(*args, **kwargs) + __implements__.append("waitpid") + else: + def fork(): + """ + Forks a child process, initializes gevent in the child, + but *does not* prepare the parent to wait for the child or receive SIGCHLD. + + This implementation of ``fork`` is a wrapper for :func:`fork_gevent` + when the environment variable ``GEVENT_NOWAITPID`` *is* defined. + This is not recommended for most applications. + """ + return fork_gevent() + + if 'forkpty' in __implements__: + def forkpty(): + """ + Like :func:`fork`, but using :func:`os.forkpty` + + This implementation of ``forkpty`` is a wrapper for :func:`forkpty_gevent` + when the environment variable ``GEVENT_NOWAITPID`` *is* defined. + This is not recommended for most applications. + + .. versionadded:: 1.1b5 + """ + return forkpty_gevent() + __extensions__.append("waitpid") + +else: + __implements__.remove('fork') + +__imports__ = copy_globals(os, globals(), + names_to_ignore=__implements__ + __extensions__, + dunder_names_to_keep=()) + +__all__ = list(set(__implements__ + __extensions__)) diff --git a/src/gevent/pool.py b/src/gevent/pool.py new file mode 100644 index 0000000..bed3ff4 --- /dev/null +++ b/src/gevent/pool.py @@ -0,0 +1,675 @@ +# Copyright (c) 2009-2011 Denis Bilenko. See LICENSE for details. +""" +Managing greenlets in a group. + +The :class:`Group` class in this module abstracts a group of running +greenlets. When a greenlet dies, it's automatically removed from the +group. All running greenlets in a group can be waited on with +:meth:`Group.join`, or all running greenlets can be killed with +:meth:`Group.kill`. + +The :class:`Pool` class, which is a subclass of :class:`Group`, +provides a way to limit concurrency: its :meth:`spawn ` +method blocks if the number of greenlets in the pool has already +reached the limit, until there is a free slot. +""" +from __future__ import print_function, absolute_import, division + + +from gevent.hub import GreenletExit, getcurrent, kill as _kill +from gevent.greenlet import joinall, Greenlet +from gevent.queue import Full as QueueFull +from gevent.timeout import Timeout +from gevent.event import Event +from gevent.lock import Semaphore, DummySemaphore + +from gevent._compat import izip +from gevent._imap import IMap +from gevent._imap import IMapUnordered + +__all__ = [ + 'Group', + 'Pool', + 'PoolFull', +] + + + + +class GroupMappingMixin(object): + # Internal, non-public API class. + # Provides mixin methods for implementing mapping pools. Subclasses must define: + + def spawn(self, func, *args, **kwargs): + """ + A function that runs *func* with *args* and *kwargs*, potentially + asynchronously. Return a value with a ``get`` method that blocks + until the results of func are available, and a ``rawlink`` method + that calls a callback when the results are available. + + If this object has an upper bound on how many asyncronously executing + tasks can exist, this method may block until a slot becomes available. + """ + raise NotImplementedError() + + def _apply_immediately(self): + """ + should the function passed to apply be called immediately, + synchronously? + """ + raise NotImplementedError() + + def _apply_async_use_greenlet(self): + """ + Should apply_async directly call Greenlet.spawn(), bypassing + `spawn`? + + Return true when self.spawn would block. + """ + raise NotImplementedError() + + def _apply_async_cb_spawn(self, callback, result): + """ + Run the given callback function, possibly + asynchronously, possibly synchronously. + """ + raise NotImplementedError() + + def apply_cb(self, func, args=None, kwds=None, callback=None): + """ + :meth:`apply` the given *func(\\*args, \\*\\*kwds)*, and, if a *callback* is given, run it with the + results of *func* (unless an exception was raised.) + + The *callback* may be called synchronously or asynchronously. If called + asynchronously, it will not be tracked by this group. (:class:`Group` and :class:`Pool` + call it asynchronously in a new greenlet; :class:`~gevent.threadpool.ThreadPool` calls + it synchronously in the current greenlet.) + """ + result = self.apply(func, args, kwds) + if callback is not None: + self._apply_async_cb_spawn(callback, result) + return result + + def apply_async(self, func, args=None, kwds=None, callback=None): + """ + A variant of the :meth:`apply` method which returns a :class:`~.Greenlet` object. + + When the returned greenlet gets to run, it *will* call :meth:`apply`, + passing in *func*, *args* and *kwds*. + + If *callback* is specified, then it should be a callable which + accepts a single argument. When the result becomes ready + callback is applied to it (unless the call failed). + + This method will never block, even if this group is full (that is, + even if :meth:`spawn` would block, this method will not). + + .. caution:: The returned greenlet may or may not be tracked + as part of this group, so :meth:`joining ` this group is + not a reliable way to wait for the results to be available or + for the returned greenlet to run; instead, join the returned + greenlet. + + .. tip:: Because :class:`~.ThreadPool` objects do not track greenlets, the returned + greenlet will never be a part of it. To reduce overhead and improve performance, + :class:`Group` and :class:`Pool` may choose to track the returned + greenlet. These are implementation details that may change. + """ + if args is None: + args = () + if kwds is None: + kwds = {} + if self._apply_async_use_greenlet(): + # cannot call self.spawn() directly because it will block + # XXX: This is always the case for ThreadPool, but for Group/Pool + # of greenlets, this is only the case when they are full...hence + # the weasely language about "may or may not be tracked". Should we make + # Group/Pool always return true as well so it's never tracked by any + # implementation? That would simplify that logic, but could increase + # the total number of greenlets in the system and add a layer of + # overhead for the simple cases when the pool isn't full. + return Greenlet.spawn(self.apply_cb, func, args, kwds, callback) + + greenlet = self.spawn(func, *args, **kwds) + if callback is not None: + greenlet.link(pass_value(callback)) + return greenlet + + def apply(self, func, args=None, kwds=None): + """ + Rough quivalent of the :func:`apply()` builtin function blocking until + the result is ready and returning it. + + The ``func`` will *usually*, but not *always*, be run in a way + that allows the current greenlet to switch out (for example, + in a new greenlet or thread, depending on implementation). But + if the current greenlet or thread is already one that was + spawned by this pool, the pool may choose to immediately run + the `func` synchronously. + + Any exception ``func`` raises will be propagated to the caller of ``apply`` (that is, + this method will raise the exception that ``func`` raised). + """ + if args is None: + args = () + if kwds is None: + kwds = {} + if self._apply_immediately(): + return func(*args, **kwds) + return self.spawn(func, *args, **kwds).get() + + def __map(self, func, iterable): + return [g.get() for g in + [self.spawn(func, i) for i in iterable]] + + def map(self, func, iterable): + """Return a list made by applying the *func* to each element of + the iterable. + + .. seealso:: :meth:`imap` + """ + # We can't return until they're all done and in order. It + # wouldn't seem to much matter what order we wait on them in, + # so the simple, fast (50% faster than imap) solution would be: + + # return [g.get() for g in + # [self.spawn(func, i) for i in iterable]] + + # If the pool size is unlimited (or more than the len(iterable)), this + # is equivalent to imap (spawn() will never block, all of them run concurrently, + # we call get() in the order the iterable was given). + + # Now lets imagine the pool if is limited size. Suppose the + # func is time.sleep, our pool is limited to 3 threads, and + # our input is [10, 1, 10, 1, 1] We would start three threads, + # one to sleep for 10, one to sleep for 1, and the last to + # sleep for 10. We would block starting the fourth thread. At + # time 1, we would finish the second thread and start another + # one for time 1. At time 2, we would finish that one and + # start the last thread, and then begin executing get() on the first + # thread. + + # Because it's spawn that blocks, this is *also* equivalent to what + # imap would do. + + # The one remaining difference is that imap runs in its own + # greenlet, potentially changing the way the event loop runs. + # That's easy enough to do. + + g = Greenlet.spawn(self.__map, func, iterable) + return g.get() + + def map_cb(self, func, iterable, callback=None): + result = self.map(func, iterable) + if callback is not None: + callback(result) + return result + + def map_async(self, func, iterable, callback=None): + """ + A variant of the map() method which returns a Greenlet object that is executing + the map function. + + If callback is specified then it should be a callable which accepts a + single argument. + """ + return Greenlet.spawn(self.map_cb, func, iterable, callback) + + def __imap(self, cls, func, *iterables, **kwargs): + # Python 2 doesn't support the syntax that lets us mix varargs and + # a named kwarg, so we have to unpack manually + maxsize = kwargs.pop('maxsize', None) + if kwargs: + raise TypeError("Unsupported keyword arguments") + return cls.spawn(func, izip(*iterables), spawn=self.spawn, + _zipped=True, maxsize=maxsize) + + def imap(self, func, *iterables, **kwargs): + """ + imap(func, *iterables, maxsize=None) -> iterable + + An equivalent of :func:`itertools.imap`, operating in parallel. + The *func* is applied to each element yielded from each + iterable in *iterables* in turn, collecting the result. + + If this object has a bound on the number of active greenlets it can + contain (such as :class:`Pool`), then at most that number of tasks will operate + in parallel. + + :keyword int maxsize: If given and not-None, specifies the maximum number of + finished results that will be allowed to accumulate awaiting the reader; + more than that number of results will cause map function greenlets to begin + to block. This is most useful if there is a great disparity in the speed of + the mapping code and the consumer and the results consume a great deal of resources. + + .. note:: This is separate from any bound on the number of active parallel + tasks, though they may have some interaction (for example, limiting the + number of parallel tasks to the smallest bound). + + .. note:: Using a bound is slightly more computationally expensive than not using a bound. + + .. tip:: The :meth:`imap_unordered` method makes much better + use of this parameter. Some additional, unspecified, + number of objects may be required to be kept in memory + to maintain order by this function. + + :return: An iterable object. + + .. versionchanged:: 1.1b3 + Added the *maxsize* keyword parameter. + .. versionchanged:: 1.1a1 + Accept multiple *iterables* to iterate in parallel. + """ + return self.__imap(IMap, func, *iterables, **kwargs) + + def imap_unordered(self, func, *iterables, **kwargs): + """ + imap_unordered(func, *iterables, maxsize=None) -> iterable + + The same as :meth:`imap` except that the ordering of the results + from the returned iterator should be considered in arbitrary + order. + + This is lighter weight than :meth:`imap` and should be preferred if order + doesn't matter. + + .. seealso:: :meth:`imap` for more details. + """ + return self.__imap(IMapUnordered, func, *iterables, **kwargs) + + +class Group(GroupMappingMixin): + """ + Maintain a group of greenlets that are still running, without + limiting their number. + + Links to each item and removes it upon notification. + + Groups can be iterated to discover what greenlets they are tracking, + they can be tested to see if they contain a greenlet, and they know the + number (len) of greenlets they are tracking. If they are not tracking any + greenlets, they are False in a boolean context. + + .. attribute:: greenlet_class + + Either :class:`gevent.Greenlet` (the default) or a subclass. + These are the type of + object we will :meth:`spawn`. This can be + changed on an instance or in a subclass. + """ + + greenlet_class = Greenlet + + def __init__(self, *args): + assert len(args) <= 1, args + self.greenlets = set(*args) + if args: + for greenlet in args[0]: + greenlet.rawlink(self._discard) + # each item we kill we place in dying, to avoid killing the same greenlet twice + self.dying = set() + self._empty_event = Event() + self._empty_event.set() + + def __repr__(self): + return '<%s at 0x%x %s>' % (self.__class__.__name__, id(self), self.greenlets) + + def __len__(self): + """ + Answer how many greenlets we are tracking. Note that if we are empty, + we are False in a boolean context. + """ + return len(self.greenlets) + + def __contains__(self, item): + """ + Answer if we are tracking the given greenlet. + """ + return item in self.greenlets + + def __iter__(self): + """ + Iterate across all the greenlets we are tracking, in no particular order. + """ + return iter(self.greenlets) + + def add(self, greenlet): + """ + Begin tracking the *greenlet*. + + If this group is :meth:`full`, then this method may block + until it is possible to track the greenlet. + + Typically the *greenlet* should **not** be started when + it is added because if this object blocks in this method, + then the *greenlet* may run to completion before it is tracked. + """ + try: + rawlink = greenlet.rawlink + except AttributeError: + pass # non-Greenlet greenlet, like MAIN + else: + rawlink(self._discard) + self.greenlets.add(greenlet) + self._empty_event.clear() + + def _discard(self, greenlet): + self.greenlets.discard(greenlet) + self.dying.discard(greenlet) + if not self.greenlets: + self._empty_event.set() + + def discard(self, greenlet): + """ + Stop tracking the greenlet. + """ + self._discard(greenlet) + try: + unlink = greenlet.unlink + except AttributeError: + pass # non-Greenlet greenlet, like MAIN + else: + unlink(self._discard) + + def start(self, greenlet): + """ + Add the **unstarted** *greenlet* to the collection of greenlets + this group is monitoring, and then start it. + """ + self.add(greenlet) + greenlet.start() + + def spawn(self, *args, **kwargs): # pylint:disable=arguments-differ + """ + Begin a new greenlet with the given arguments (which are passed + to the greenlet constructor) and add it to the collection of greenlets + this group is monitoring. + + :return: The newly started greenlet. + """ + greenlet = self.greenlet_class(*args, **kwargs) + self.start(greenlet) + return greenlet + +# def close(self): +# """Prevents any more tasks from being submitted to the pool""" +# self.add = RaiseException("This %s has been closed" % self.__class__.__name__) + + def join(self, timeout=None, raise_error=False): + """ + Wait for this group to become empty *at least once*. + + If there are no greenlets in the group, returns immediately. + + .. note:: By the time the waiting code (the caller of this + method) regains control, a greenlet may have been added to + this group, and so this object may no longer be empty. (That + is, ``group.join(); assert len(group) == 0`` is not + guaranteed to hold.) This method only guarantees that the group + reached a ``len`` of 0 at some point. + + :keyword bool raise_error: If True (*not* the default), if any + greenlet that finished while the join was in progress raised + an exception, that exception will be raised to the caller of + this method. If multiple greenlets raised exceptions, which + one gets re-raised is not determined. Only greenlets currently + in the group when this method is called are guaranteed to + be checked for exceptions. + + :return bool: A value indicating whether this group became empty. + If the timeout is specified and the group did not become empty + during that timeout, then this will be a false value. Otherwise + it will be a true value. + + .. versionchanged:: 1.2a1 + Add the return value. + """ + greenlets = list(self.greenlets) if raise_error else () + result = self._empty_event.wait(timeout=timeout) + + for greenlet in greenlets: + if greenlet.exception is not None: + if hasattr(greenlet, '_raise_exception'): + greenlet._raise_exception() + raise greenlet.exception + + return result + + def kill(self, exception=GreenletExit, block=True, timeout=None): + """ + Kill all greenlets being tracked by this group. + """ + timer = Timeout._start_new_or_dummy(timeout) + try: + while self.greenlets: + for greenlet in list(self.greenlets): + if greenlet in self.dying: + continue + try: + kill = greenlet.kill + except AttributeError: + _kill(greenlet, exception) + else: + kill(exception, block=False) + self.dying.add(greenlet) + if not block: + break + joinall(self.greenlets) + except Timeout as ex: + if ex is not timer: + raise + finally: + timer.cancel() + + def killone(self, greenlet, exception=GreenletExit, block=True, timeout=None): + """ + If the given *greenlet* is running and being tracked by this group, + kill it. + """ + if greenlet not in self.dying and greenlet in self.greenlets: + greenlet.kill(exception, block=False) + self.dying.add(greenlet) + if block: + greenlet.join(timeout) + + def full(self): + """ + Return a value indicating whether this group can track more greenlets. + + In this implementation, because there are no limits on the number of + tracked greenlets, this will always return a ``False`` value. + """ + return False + + def wait_available(self, timeout=None): + """ + Block until it is possible to :meth:`spawn` a new greenlet. + + In this implementation, because there are no limits on the number + of tracked greenlets, this will always return immediately. + """ + + # MappingMixin methods + + def _apply_immediately(self): + # If apply() is called from one of our own + # worker greenlets, don't spawn a new one---if we're full, that + # could deadlock. + return getcurrent() in self + + def _apply_async_cb_spawn(self, callback, result): + Greenlet.spawn(callback, result) + + def _apply_async_use_greenlet(self): + # cannot call self.spawn() because it will block, so + # use a fresh, untracked greenlet that when run will + # (indirectly) call self.spawn() for us. + return self.full() + + + +class PoolFull(QueueFull): + """ + Raised when a Pool is full and an attempt was made to + add a new greenlet to it in non-blocking mode. + """ + + +class Pool(Group): + + def __init__(self, size=None, greenlet_class=None): + """ + Create a new pool. + + A pool is like a group, but the maximum number of members + is governed by the *size* parameter. + + :keyword int size: If given, this non-negative integer is the + maximum count of active greenlets that will be allowed in + this pool. A few values have special significance: + + * `None` (the default) places no limit on the number of + greenlets. This is useful when you want to track, but not limit, + greenlets. In general, a :class:`Group` + may be a more efficient way to achieve the same effect, but some things + need the additional abilities of this class (one example being the *spawn* + parameter of :class:`gevent.baseserver.BaseServer` and + its subclass :class:`gevent.pywsgi.WSGIServer`). + + * ``0`` creates a pool that can never have any active greenlets. Attempting + to spawn in this pool will block forever. This is only useful + if an application uses :meth:`wait_available` with a timeout and checks + :meth:`free_count` before attempting to spawn. + """ + if size is not None and size < 0: + raise ValueError('size must not be negative: %r' % (size, )) + Group.__init__(self) + self.size = size + if greenlet_class is not None: + self.greenlet_class = greenlet_class + if size is None: + factory = DummySemaphore + else: + factory = Semaphore + self._semaphore = factory(size) + + def wait_available(self, timeout=None): + """ + Wait until it's possible to spawn a greenlet in this pool. + + :param float timeout: If given, only wait the specified number + of seconds. + + .. warning:: If the pool was initialized with a size of 0, this + method will block forever unless a timeout is given. + + :return: A number indicating how many new greenlets can be put into + the pool without blocking. + + .. versionchanged:: 1.1a3 + Added the ``timeout`` parameter. + """ + return self._semaphore.wait(timeout=timeout) + + def full(self): + """ + Return a boolean indicating whether this pool is full, e.g. if + :meth:`add` would block. + + :return: False if there is room for new members, True if there isn't. + """ + return self.free_count() <= 0 + + def free_count(self): + """ + Return a number indicating *approximately* how many more members + can be added to this pool. + """ + if self.size is None: + return 1 + return max(0, self.size - len(self)) + + def start(self, greenlet, *args, **kwargs): # pylint:disable=arguments-differ + """ + start(greenlet, blocking=True, timeout=None) -> None + + Add the **unstarted** *greenlet* to the collection of greenlets + this group is monitoring and then start it. + + Parameters are as for :meth:`add`. + """ + self.add(greenlet, *args, **kwargs) + greenlet.start() + + def add(self, greenlet, blocking=True, timeout=None): # pylint:disable=arguments-differ + """ + Begin tracking the given **unstarted** greenlet, possibly blocking + until space is available. + + Usually you should call :meth:`start` to track and start the greenlet + instead of using this lower-level method, or :meth:`spawn` to + also create the greenlet. + + :keyword bool blocking: If True (the default), this function + will block until the pool has space or a timeout occurs. If + False, this function will immediately raise a Timeout if the + pool is currently full. + :keyword float timeout: The maximum number of seconds this + method will block, if ``blocking`` is True. (Ignored if + ``blocking`` is False.) + :raises PoolFull: if either ``blocking`` is False and the pool + was full, or if ``blocking`` is True and ``timeout`` was + exceeded. + + .. caution:: If the *greenlet* has already been started and + *blocking* is true, then the greenlet may run to completion + while the current greenlet blocks waiting to track it. This would + enable higher concurrency than desired. + + .. seealso:: :meth:`Group.add` + + .. versionchanged:: 1.3.0 Added the ``blocking`` and + ``timeout`` parameters. + """ + if not self._semaphore.acquire(blocking=blocking, timeout=timeout): + # We failed to acquire the semaphore. + # If blocking was True, then there was a timeout. If blocking was + # False, then there was no capacity. Either way, raise PoolFull. + raise PoolFull() + + try: + Group.add(self, greenlet) + except: + self._semaphore.release() + raise + + def _discard(self, greenlet): + Group._discard(self, greenlet) + self._semaphore.release() + + +class pass_value(object): + __slots__ = ['callback'] + + def __init__(self, callback): + self.callback = callback + + def __call__(self, source): + if source.successful(): + self.callback(source.value) + + def __hash__(self): + return hash(self.callback) + + def __eq__(self, other): + return self.callback == getattr(other, 'callback', other) + + def __str__(self): + return str(self.callback) + + def __repr__(self): + return repr(self.callback) + + def __getattr__(self, item): + assert item != 'callback' + return getattr(self.callback, item) diff --git a/src/gevent/pywsgi.py b/src/gevent/pywsgi.py new file mode 100644 index 0000000..5276eb5 --- /dev/null +++ b/src/gevent/pywsgi.py @@ -0,0 +1,1549 @@ +# Copyright (c) 2005-2009, eventlet contributors +# Copyright (c) 2009-2018, gevent contributors +""" +A pure-Python, gevent-friendly WSGI server. + +The server is provided in :class:`WSGIServer`, but most of the actual +WSGI work is handled by :class:`WSGIHandler` --- a new instance is +created for each request. The server can be customized to use +different subclasses of :class:`WSGIHandler`. + +""" +from __future__ import absolute_import + +# FIXME: Can we refactor to make smallor? +# pylint:disable=too-many-lines + +import errno +from io import BytesIO +import string +import sys +import time +import traceback +from datetime import datetime + +try: + from urllib import unquote +except ImportError: + from urllib.parse import unquote # python 2 pylint:disable=import-error,no-name-in-module + +from gevent import socket +import gevent +from gevent.server import StreamServer +from gevent.hub import GreenletExit +from gevent._compat import PY3, reraise + +from functools import partial +if PY3: + unquote_latin1 = partial(unquote, encoding='latin-1') +else: + unquote_latin1 = unquote + +_no_undoc_members = True # Don't put undocumented things into sphinx + +__all__ = [ + 'WSGIServer', + 'WSGIHandler', + 'LoggingLogAdapter', + 'Environ', + 'SecureEnviron', + 'WSGISecureEnviron', +] + + +MAX_REQUEST_LINE = 8192 +# Weekday and month names for HTTP date/time formatting; always English! +_WEEKDAYNAME = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] +_MONTHNAME = [None, # Dummy so we can use 1-based month numbers + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"] + +# The contents of the "HEX" grammar rule for HTTP, upper and lowercase A-F plus digits, +# in byte form for comparing to the network. +_HEX = string.hexdigits.encode('ascii') + +# Errors +_ERRORS = dict() +_INTERNAL_ERROR_STATUS = '500 Internal Server Error' +_INTERNAL_ERROR_BODY = b'Internal Server Error' +_INTERNAL_ERROR_HEADERS = [('Content-Type', 'text/plain'), + ('Connection', 'close'), + ('Content-Length', str(len(_INTERNAL_ERROR_BODY)))] +_ERRORS[500] = (_INTERNAL_ERROR_STATUS, _INTERNAL_ERROR_HEADERS, _INTERNAL_ERROR_BODY) + +_BAD_REQUEST_STATUS = '400 Bad Request' +_BAD_REQUEST_BODY = '' +_BAD_REQUEST_HEADERS = [('Content-Type', 'text/plain'), + ('Connection', 'close'), + ('Content-Length', str(len(_BAD_REQUEST_BODY)))] +_ERRORS[400] = (_BAD_REQUEST_STATUS, _BAD_REQUEST_HEADERS, _BAD_REQUEST_BODY) + +_REQUEST_TOO_LONG_RESPONSE = b"HTTP/1.1 414 Request URI Too Long\r\nConnection: close\r\nContent-length: 0\r\n\r\n" +_BAD_REQUEST_RESPONSE = b"HTTP/1.1 400 Bad Request\r\nConnection: close\r\nContent-length: 0\r\n\r\n" +_CONTINUE_RESPONSE = b"HTTP/1.1 100 Continue\r\n\r\n" + + +def format_date_time(timestamp): + # Return a byte-string of the date and time in HTTP format + # .. versionchanged:: 1.1b5 + # Return a byte string, not a native string + year, month, day, hh, mm, ss, wd, _y, _z = time.gmtime(timestamp) + value = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (_WEEKDAYNAME[wd], day, _MONTHNAME[month], year, hh, mm, ss) + if PY3: + value = value.encode("latin-1") + return value + + +class _InvalidClientInput(IOError): + # Internal exception raised by Input indicating that the client + # sent invalid data at the lowest level of the stream. The result + # *should* be a HTTP 400 error. + pass + + +class _InvalidClientRequest(ValueError): + # Internal exception raised by WSGIHandler.read_request + # indicating that the client sent an HTTP request that cannot + # be parsed (e.g., invalid grammar). The result *should* be an + # HTTP 400 error + pass + + +class Input(object): + + __slots__ = ('rfile', 'content_length', 'socket', 'position', + 'chunked_input', 'chunk_length', '_chunked_input_error') + + def __init__(self, rfile, content_length, socket=None, chunked_input=False): + # pylint:disable=redefined-outer-name + self.rfile = rfile + self.content_length = content_length + self.socket = socket + self.position = 0 + self.chunked_input = chunked_input + self.chunk_length = -1 + self._chunked_input_error = False + + def _discard(self): + if self._chunked_input_error: + # We are in an unknown state, so we can't necessarily discard + # the body (e.g., if the client keeps the socket open, we could hang + # here forever). + # In this case, we've raised an exception and the user of this object + # is going to close the socket, so we don't have to discard + return + + if self.socket is None and (self.position < (self.content_length or 0) or self.chunked_input): + # ## Read and discard body + while 1: + d = self.read(16384) + if not d: + break + + def _send_100_continue(self): + if self.socket is not None: + self.socket.sendall(_CONTINUE_RESPONSE) + self.socket = None + + def _do_read(self, length=None, use_readline=False): + if use_readline: + reader = self.rfile.readline + else: + reader = self.rfile.read + content_length = self.content_length + if content_length is None: + # Either Content-Length or "Transfer-Encoding: chunked" must be present in a request with a body + # if it was chunked, then this function would have not been called + return b'' + + self._send_100_continue() + left = content_length - self.position + if length is None: + length = left + elif length > left: + length = left + if not length: + return b'' + + # On Python 2, self.rfile is usually socket.makefile(), which + # uses cStringIO.StringIO. If *length* is greater than the C + # sizeof(int) (typically 32 bits signed), parsing the argument to + # readline raises OverflowError. StringIO.read(), OTOH, uses + # PySize_t, typically a long (64 bits). In a bare readline() + # case, because the header lines we're trying to read with + # readline are typically expected to be small, we can correct + # that failure by simply doing a smaller call to readline and + # appending; failures in read we let propagate. + try: + read = reader(length) + except OverflowError: + if not use_readline: + # Expecting to read more than 64 bits of data. Ouch! + raise + # We could loop on calls to smaller readline(), appending them + # until we actually get a newline. For uses in this module, + # we expect the actual length to be small, but WSGI applications + # are allowed to pass in an arbitrary length. (This loop isn't optimal, + # but even client applications *probably* have short lines.) + read = b'' + while len(read) < length and not read.endswith(b'\n'): + read += reader(MAX_REQUEST_LINE) + + self.position += len(read) + if len(read) < length: + if (use_readline and not read.endswith(b"\n")) or not use_readline: + raise IOError("unexpected end of file while reading request at position %s" % (self.position,)) + + return read + + def __read_chunk_length(self, rfile): + # Read and return the next integer chunk length. If no + # chunk length can be read, raises _InvalidClientInput. + + # Here's the production for a chunk: + # (http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html) + # chunk = chunk-size [ chunk-extension ] CRLF + # chunk-data CRLF + # chunk-size = 1*HEX + # chunk-extension= *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) + # chunk-ext-name = token + # chunk-ext-val = token | quoted-string + + # To cope with malicious or broken clients that fail to send valid + # chunk lines, the strategy is to read character by character until we either reach + # a ; or newline. If at any time we read a non-HEX digit, we bail. If we hit a + # ;, indicating an chunk-extension, we'll read up to the next + # MAX_REQUEST_LINE characters + # looking for the CRLF, and if we don't find it, we bail. If we read more than 16 hex characters, + # (the number needed to represent a 64-bit chunk size), we bail (this protects us from + # a client that sends an infinite stream of `F`, for example). + + buf = BytesIO() + while 1: + char = rfile.read(1) + if not char: + self._chunked_input_error = True + raise _InvalidClientInput("EOF before chunk end reached") + if char == b'\r': + break + if char == b';': + break + + if char not in _HEX: + self._chunked_input_error = True + raise _InvalidClientInput("Non-hex data", char) + buf.write(char) + if buf.tell() > 16: + self._chunked_input_error = True + raise _InvalidClientInput("Chunk-size too large.") + + if char == b';': + i = 0 + while i < MAX_REQUEST_LINE: + char = rfile.read(1) + if char == b'\r': + break + i += 1 + else: + # we read more than MAX_REQUEST_LINE without + # hitting CR + self._chunked_input_error = True + raise _InvalidClientInput("Too large chunk extension") + + if char == b'\r': + # We either got here from the main loop or from the + # end of an extension + char = rfile.read(1) + if char != b'\n': + self._chunked_input_error = True + raise _InvalidClientInput("Line didn't end in CRLF") + return int(buf.getvalue(), 16) + + def _chunked_read(self, length=None, use_readline=False): + # pylint:disable=too-many-branches + rfile = self.rfile + self._send_100_continue() + + if length == 0: + return b"" + + if use_readline: + reader = self.rfile.readline + else: + reader = self.rfile.read + + response = [] + while self.chunk_length != 0: + maxreadlen = self.chunk_length - self.position + if length is not None and length < maxreadlen: + maxreadlen = length + + if maxreadlen > 0: + data = reader(maxreadlen) + if not data: + self.chunk_length = 0 + self._chunked_input_error = True + raise IOError("unexpected end of file while parsing chunked data") + + datalen = len(data) + response.append(data) + + self.position += datalen + if self.chunk_length == self.position: + rfile.readline() + + if length is not None: + length -= datalen + if length == 0: + break + if use_readline and data[-1] == b"\n"[0]: + break + else: + # We're at the beginning of a chunk, so we need to + # determine the next size to read + self.chunk_length = self.__read_chunk_length(rfile) + self.position = 0 + if self.chunk_length == 0: + # Last chunk. Terminates with a CRLF. + rfile.readline() + return b''.join(response) + + def read(self, length=None): + if length is not None and length < 0: + length = None + if self.chunked_input: + return self._chunked_read(length) + return self._do_read(length) + + def readline(self, size=None): + if size is not None and size < 0: + size = None + if self.chunked_input: + return self._chunked_read(size, True) + return self._do_read(size, use_readline=True) + + def readlines(self, hint=None): + # pylint:disable=unused-argument + return list(self) + + def __iter__(self): + return self + + def next(self): + line = self.readline() + if not line: + raise StopIteration + return line + __next__ = next + + +try: + import mimetools + headers_factory = mimetools.Message +except ImportError: + # adapt Python 3 HTTP headers to old API + from http import client # pylint:disable=import-error + + class OldMessage(client.HTTPMessage): + def __init__(self, **kwargs): + super(client.HTTPMessage, self).__init__(**kwargs) # pylint:disable=bad-super-call + self.status = '' + + def getheader(self, name, default=None): + return self.get(name, default) + + @property + def headers(self): + for key, value in self._headers: + yield '%s: %s\r\n' % (key, value) + + @property + def typeheader(self): + return self.get('content-type') + + def headers_factory(fp, *args): # pylint:disable=unused-argument + try: + ret = client.parse_headers(fp, _class=OldMessage) + except client.LineTooLong: + ret = OldMessage() + ret.status = 'Line too long' + return ret + + +class WSGIHandler(object): + """ + Handles HTTP requests from a socket, creates the WSGI environment, and + interacts with the WSGI application. + + This is the default value of :attr:`WSGIServer.handler_class`. + This class may be subclassed carefully, and that class set on a + :class:`WSGIServer` instance through a keyword argument at + construction time. + + Instances are constructed with the same arguments as passed to the + server's :meth:`WSGIServer.handle` method followed by the server + itself. The application and environment are obtained from the server. + + """ + # pylint:disable=too-many-instance-attributes + + protocol_version = 'HTTP/1.1' + if PY3: + # if we do like Py2, then headers_factory unconditionally + # becomes a bound method, meaning the fp argument becomes WSGIHandler + def MessageClass(self, *args): + return headers_factory(*args) + else: + MessageClass = headers_factory + + # Attributes reset at various times for each request; not public + # documented. Class attributes to keep the constructor fast + # (but not make lint tools complain) + + status = None # byte string: b'200 OK' + _orig_status = None # native string: '200 OK' + response_headers = None # list of tuples (b'name', b'value') + code = None # Integer parsed from status + provided_date = None + provided_content_length = None + close_connection = False + time_start = 0 # time.time() when begin handling request + time_finish = 0 # time.time() when done handling request + headers_sent = False # Have we already sent headers? + response_use_chunked = False # Write with transfer-encoding chunked + environ = None # Dict from self.get_environ + application = None # application callable from self.server.application + requestline = None # native str 'GET / HTTP/1.1' + response_length = 0 # How much data we sent + result = None # The return value of the WSGI application + wsgi_input = None # Instance of Input() + content_length = 0 # From application-provided headers Incoming + # request headers, instance of MessageClass (gunicorn uses hasattr + # on this so the default value needs to be compatible with the + # API) + headers = headers_factory(BytesIO()) + request_version = None # str: 'HTTP 1.1' + command = None # str: 'GET' + path = None # str: '/' + + def __init__(self, sock, address, server, rfile=None): + # Deprecation: The rfile kwarg was introduced in 1.0a1 as part + # of a refactoring. It was never documented or used. It is + # considered DEPRECATED and may be removed in the future. Its + # use is not supported. + + self.socket = sock + self.client_address = address + self.server = server + if rfile is None: + self.rfile = sock.makefile('rb', -1) + else: + self.rfile = rfile + + def handle(self): + """ + The main request handling method, called by the server. + + This method runs a request handling loop, calling + :meth:`handle_one_request` until all requests on the + connection have been handled (that is, it implements + keep-alive). + """ + try: + while self.socket is not None: + self.time_start = time.time() + self.time_finish = 0 + + result = self.handle_one_request() + if result is None: + break + if result is True: + continue + + self.status, response_body = result + self.socket.sendall(response_body) + if self.time_finish == 0: + self.time_finish = time.time() + self.log_request() + break + finally: + if self.socket is not None: + _sock = getattr(self.socket, '_sock', None) # Python 3 + try: + # read out request data to prevent error: [Errno 104] Connection reset by peer + if _sock: + try: + # socket.recv would hang + _sock.recv(16384) + finally: + _sock.close() + self.socket.close() + except socket.error: + pass + self.__dict__.pop('socket', None) + self.__dict__.pop('rfile', None) + + def _check_http_version(self): + version_str = self.request_version + if not version_str.startswith("HTTP/"): + return False + version = tuple(int(x) for x in version_str[5:].split(".")) # "HTTP/" + if version[1] < 0 or version < (0, 9) or version >= (2, 0): + return False + return True + + def read_request(self, raw_requestline): + """ + Parse the incoming request. + + Parses various headers into ``self.headers`` using + :attr:`MessageClass`. Other attributes that are set upon a successful + return of this method include ``self.content_length`` and ``self.close_connection``. + + :param str raw_requestline: A native :class:`str` representing + the request line. A processed version of this will be stored + into ``self.requestline``. + + :raises ValueError: If the request is invalid. This error will + not be logged as a traceback (because it's a client issue, not a server problem). + :return: A boolean value indicating whether the request was successfully parsed. + This method should either return a true value or have raised a ValueError + with details about the parsing error. + + .. versionchanged:: 1.1b6 + Raise the previously documented :exc:`ValueError` in more cases instead of returning a + false value; this allows subclasses more opportunity to customize behaviour. + """ + # pylint:disable=too-many-branches + self.requestline = raw_requestline.rstrip() + words = self.requestline.split() + if len(words) == 3: + self.command, self.path, self.request_version = words + if not self._check_http_version(): + raise _InvalidClientRequest('Invalid http version: %r' % (raw_requestline,)) + elif len(words) == 2: + self.command, self.path = words + if self.command != "GET": + raise _InvalidClientRequest('Expected GET method: %r' % (raw_requestline,)) + self.request_version = "HTTP/0.9" + # QQQ I'm pretty sure we can drop support for HTTP/0.9 + else: + raise _InvalidClientRequest('Invalid HTTP method: %r' % (raw_requestline,)) + + self.headers = self.MessageClass(self.rfile, 0) + + if self.headers.status: + raise _InvalidClientRequest('Invalid headers status: %r' % (self.headers.status,)) + + if self.headers.get("transfer-encoding", "").lower() == "chunked": + try: + del self.headers["content-length"] + except KeyError: + pass + + content_length = self.headers.get("content-length") + if content_length is not None: + content_length = int(content_length) + if content_length < 0: + raise _InvalidClientRequest('Invalid Content-Length: %r' % (content_length,)) + + if content_length and self.command in ('HEAD', ): + raise _InvalidClientRequest('Unexpected Content-Length') + + self.content_length = content_length + + if self.request_version == "HTTP/1.1": + conntype = self.headers.get("Connection", "").lower() + self.close_connection = (conntype == 'close') + else: + self.close_connection = True + + return True + + def log_error(self, msg, *args): + try: + message = msg % args + except Exception: # pylint:disable=broad-except + traceback.print_exc() + message = '%r %r' % (msg, args) + try: + message = '%s: %s' % (self.socket, message) + except Exception: # pylint:disable=broad-except + pass + + try: + self.server.error_log.write(message + '\n') + except Exception: # pylint:disable=broad-except + traceback.print_exc() + + def read_requestline(self): + """ + Read and return the HTTP request line. + + Under both Python 2 and 3, this should return the native + ``str`` type; under Python 3, this probably means the bytes read + from the network need to be decoded (using the ISO-8859-1 charset, aka + latin-1). + """ + line = self.rfile.readline(MAX_REQUEST_LINE) + if PY3: + line = line.decode('latin-1') + return line + + def handle_one_request(self): + """ + Handles one HTTP request using ``self.socket`` and ``self.rfile``. + + Each invocation of this method will do several things, including (but not limited to): + + - Read the request line using :meth:`read_requestline`; + - Read the rest of the request, including headers, with :meth:`read_request`; + - Construct a new WSGI environment in ``self.environ`` using :meth:`get_environ`; + - Store the application in ``self.application``, retrieving it from the server; + - Handle the remainder of the request, including invoking the application, + with :meth:`handle_one_response` + + There are several possible return values to indicate the state + of the client connection: + + - ``None`` + The client connection is already closed or should + be closed because the WSGI application or client set the + ``Connection: close`` header. The request handling + loop should terminate and perform cleanup steps. + - (status, body) + An HTTP status and body tuple. The request was in error, + as detailed by the status and body. The request handling + loop should terminate, close the connection, and perform + cleanup steps. Note that the ``body`` is the complete contents + to send to the client, including all headers and the initial + status line. + - ``True`` + The literal ``True`` value. The request was successfully handled + and the response sent to the client by :meth:`handle_one_response`. + The connection remains open to process more requests and the connection + handling loop should call this method again. This is the typical return + value. + + .. seealso:: :meth:`handle` + + .. versionchanged:: 1.1b6 + Funnel exceptions having to do with invalid HTTP requests through + :meth:`_handle_client_error` to allow subclasses to customize. Note that + this is experimental and may change in the future. + """ + # pylint:disable=too-many-return-statements + if self.rfile.closed: + return + + try: + self.requestline = self.read_requestline() + # Account for old subclasses that haven't done this + if PY3 and isinstance(self.requestline, bytes): + self.requestline = self.requestline.decode('latin-1') + except socket.error: + # "Connection reset by peer" or other socket errors aren't interesting here + return + + if not self.requestline: + return + + self.response_length = 0 + + if len(self.requestline) >= MAX_REQUEST_LINE: + return ('414', _REQUEST_TOO_LONG_RESPONSE) + + try: + # for compatibility with older versions of pywsgi, we pass self.requestline as an argument there + # NOTE: read_request is supposed to raise ValueError on invalid input; allow old + # subclasses that return a False value instead. + # NOTE: This can mutate the value of self.headers, so self.get_environ() must not be + # called until AFTER this call is done. + if not self.read_request(self.requestline): + return ('400', _BAD_REQUEST_RESPONSE) + except Exception as ex: # pylint:disable=broad-except + # Notice we don't use self.handle_error because it reports + # a 500 error to the client, and this is almost certainly + # a client error. + # Provide a hook for subclasses. + return self._handle_client_error(ex) + + self.environ = self.get_environ() + self.application = self.server.application + + self.handle_one_response() + + if self.close_connection: + return + + if self.rfile.closed: + return + + return True # read more requests + + def finalize_headers(self): + if self.provided_date is None: + self.response_headers.append((b'Date', format_date_time(time.time()))) + + if self.code not in (304, 204): + # the reply will include message-body; make sure we have either Content-Length or chunked + if self.provided_content_length is None: + if hasattr(self.result, '__len__'): + total_len = sum(len(chunk) for chunk in self.result) + total_len_str = str(total_len) + if PY3: + total_len_str = total_len_str.encode("latin-1") + self.response_headers.append((b'Content-Length', total_len_str)) + else: + if self.request_version != 'HTTP/1.0': + self.response_use_chunked = True + self.response_headers.append((b'Transfer-Encoding', b'chunked')) + + def _sendall(self, data): + try: + self.socket.sendall(data) + except socket.error as ex: + self.status = 'socket error: %s' % ex + if self.code > 0: + self.code = -self.code + raise + self.response_length += len(data) + + def _write(self, data, + _PY34_EXACTLY=(sys.version_info[:2] == (3, 4)), + _bytearray=bytearray): + if not data: + # The application/middleware are allowed to yield + # empty bytestrings. + return + + if self.response_use_chunked: + ## Write the chunked encoding + # header + if _PY34_EXACTLY: + # This is the only version we support that doesn't + # allow % to be used with bytes. Passing a bytestring + # directly in to bytearray() is faster than passing a + # (unicode) str with encoding, which naturally is faster still + # than encoding first. Interestingly, byte formatting on Python 3 + # is faster than str formatting. + header_str = '%x\r\n' % len(data) + towrite = _bytearray(header_str, 'ascii') + else: + header_str = b'%x\r\n' % len(data) + towrite = _bytearray(header_str) + + # data + towrite += data + # trailer + towrite += b'\r\n' + self._sendall(towrite) + else: + self._sendall(data) + + def write(self, data): + # The write() callable we return from start_response. + # https://www.python.org/dev/peps/pep-3333/#the-write-callable + # Supposed to do pretty much the same thing as yielding values + # from the application's return. + if self.code in (304, 204) and data: + raise AssertionError('The %s response must have no body' % self.code) + + if self.headers_sent: + self._write(data) + else: + if not self.status: + raise AssertionError("The application did not call start_response()") + self._write_with_headers(data) + + def _write_with_headers(self, data): + self.headers_sent = True + self.finalize_headers() + + # self.response_headers and self.status are already in latin-1, as encoded by self.start_response + towrite = bytearray(b'HTTP/1.1 ') + towrite += self.status + towrite += b'\r\n' + for header, value in self.response_headers: + towrite += header + towrite += b': ' + towrite += value + towrite += b"\r\n" + + towrite += b'\r\n' + self._sendall(towrite) + # No need to copy the data into towrite; we may make an extra syscall + # but the copy time could be substantial too, and it reduces the chances + # of sendall being able to send everything in one go + self._write(data) + + def start_response(self, status, headers, exc_info=None): + """ + .. versionchanged:: 1.2a1 + Avoid HTTP header injection by raising a :exc:`ValueError` + if *status* or any *header* name or value contains a carriage + return or newline. + .. versionchanged:: 1.1b5 + Pro-actively handle checking the encoding of the status line + and headers during this method. On Python 2, avoid some + extra encodings. + """ + # pylint:disable=too-many-branches,too-many-statements + if exc_info: + try: + if self.headers_sent: + # Re-raise original exception if headers sent + reraise(*exc_info) + finally: + # Avoid dangling circular ref + exc_info = None + + # Pep 3333, "The start_response callable": + # https://www.python.org/dev/peps/pep-3333/#the-start-response-callable + # "Servers should check for errors in the headers at the time + # start_response is called, so that an error can be raised + # while the application is still running." Here, we check the encoding. + # This aids debugging: headers especially are generated programmatically + # and an encoding error in a loop or list comprehension yields an opaque + # UnicodeError without any clue which header was wrong. + # Note that this results in copying the header list at this point, not modifying it, + # although we are allowed to do so if needed. This slightly increases memory usage. + # We also check for HTTP Response Splitting vulnerabilities + response_headers = [] + header = None + value = None + try: + for header, value in headers: + if not isinstance(header, str): + raise UnicodeError("The header must be a native string", header, value) + if not isinstance(value, str): + raise UnicodeError("The value must be a native string", header, value) + if '\r' in header or '\n' in header: + raise ValueError('carriage return or newline in header name', header) + if '\r' in value or '\n' in value: + raise ValueError('carriage return or newline in header value', value) + # Either we're on Python 2, in which case bytes is correct, or + # we're on Python 3 and the user screwed up (because it should be a native + # string). In either case, make sure that this is latin-1 compatible. Under + # Python 2, bytes.encode() will take a round-trip through the system encoding, + # which may be ascii, which is not really what we want. However, the latin-1 encoding + # can encode everything except control characters and the block from 0x7F to 0x9F, so + # explicitly round-tripping bytes through the encoding is unlikely to be of much + # benefit, so we go for speed (the WSGI spec specifically calls out allowing the range + # from 0x00 to 0xFF, although the HTTP spec forbids the control characters). + # Note: Some Python 2 implementations, like Jython, may allow non-octet (above 255) values + # in their str implementation; this is mentioned in the WSGI spec, but we don't + # run on any platform like that so we can assume that a str value is pure bytes. + response_headers.append((header if not PY3 else header.encode("latin-1"), + value if not PY3 else value.encode("latin-1"))) + except UnicodeEncodeError: + # If we get here, we're guaranteed to have a header and value + raise UnicodeError("Non-latin1 header", repr(header), repr(value)) + + # Same as above + if not isinstance(status, str): + raise UnicodeError("The status string must be a native string") + if '\r' in status or '\n' in status: + raise ValueError("carriage return or newline in status", status) + # don't assign to anything until the validation is complete, including parsing the + # code + code = int(status.split(' ', 1)[0]) + + self.status = status if not PY3 else status.encode("latin-1") + self._orig_status = status # Preserve the native string for logging + self.response_headers = response_headers + self.code = code + + provided_connection = None + self.provided_date = None + self.provided_content_length = None + + for header, value in headers: + header = header.lower() + if header == 'connection': + provided_connection = value + elif header == 'date': + self.provided_date = value + elif header == 'content-length': + self.provided_content_length = value + + if self.request_version == 'HTTP/1.0' and provided_connection is None: + response_headers.append((b'Connection', b'close')) + self.close_connection = True + elif provided_connection == 'close': + self.close_connection = True + + if self.code in (304, 204): + if self.provided_content_length is not None and self.provided_content_length != '0': + msg = 'Invalid Content-Length for %s response: %r (must be absent or zero)' % (self.code, self.provided_content_length) + if PY3: + msg = msg.encode('latin-1') + raise AssertionError(msg) + + return self.write + + def log_request(self): + self.server.log.write(self.format_request() + '\n') + + def format_request(self): + now = datetime.now().replace(microsecond=0) + length = self.response_length or '-' + if self.time_finish: + delta = '%.6f' % (self.time_finish - self.time_start) + else: + delta = '-' + client_address = self.client_address[0] if isinstance(self.client_address, tuple) else self.client_address + return '%s - - [%s] "%s" %s %s %s' % ( + client_address or '-', + now, + self.requestline or '', + # Use the native string version of the status, saved so we don't have to + # decode. But fallback to the encoded 'status' in case of subclasses + # (Is that really necessary? At least there's no overhead.) + (self._orig_status or self.status or '000').split()[0], + length, + delta) + + def process_result(self): + for data in self.result: + if data: + self.write(data) + if self.status and not self.headers_sent: + # In other words, the application returned an empty + # result iterable (and did not use the write callable) + # Trigger the flush of the headers. + self.write(b'') + if self.response_use_chunked: + self._sendall(b'0\r\n\r\n') + + + def run_application(self): + assert self.result is None + try: + self.result = self.application(self.environ, self.start_response) + self.process_result() + finally: + close = getattr(self.result, 'close', None) + try: + if close is not None: + close() + finally: + # Discard the result. If it's a generator this can + # free a lot of hidden resources (if we failed to iterate + # all the way through it---the frames are automatically + # cleaned up when StopIteration is raised); but other cases + # could still free up resources sooner than otherwise. + close = None + self.result = None + + #: These errors are silently ignored by :meth:`handle_one_response` to avoid producing + #: excess log entries on normal operating conditions. They indicate + #: a remote client has disconnected and there is little or nothing + #: this process can be expected to do about it. You may change this + #: value in a subclass. + #: + #: The default value includes :data:`errno.EPIPE` and :data:`errno.ECONNRESET`. + #: On Windows this also includes :data:`errno.WSAECONNABORTED`. + #: + #: This is a provisional API, subject to change. See :pr:`377`, :pr:`999` + #: and :issue:`136`. + #: + #: .. versionadded:: 1.3 + ignored_socket_errors = (errno.EPIPE, errno.ECONNRESET) + try: + ignored_socket_errors += (errno.WSAECONNABORTED,) + except AttributeError: + pass # Not windows + + def handle_one_response(self): + """ + Invoke the application to produce one response. + + This is called by :meth:`handle_one_request` after all the + state for the request has been established. It is responsible + for error handling. + """ + self.time_start = time.time() + self.status = None + self.headers_sent = False + + self.result = None + self.response_use_chunked = False + self.response_length = 0 + + try: + try: + self.run_application() + finally: + try: + self.wsgi_input._discard() + except (socket.error, IOError): + # Don't let exceptions during discarding + # input override any exception that may have been + # raised by the application, such as our own _InvalidClientInput. + # In the general case, these aren't even worth logging (see the comment + # just below) + pass + except _InvalidClientInput: + self._send_error_response_if_possible(400) + except socket.error as ex: + if ex.args[0] in self.ignored_socket_errors: + # See description of self.ignored_socket_errors. + if not PY3: + sys.exc_clear() + self.close_connection = True + else: + self.handle_error(*sys.exc_info()) + except: # pylint:disable=bare-except + self.handle_error(*sys.exc_info()) + finally: + self.time_finish = time.time() + self.log_request() + + def _send_error_response_if_possible(self, error_code): + if self.response_length: + self.close_connection = True + else: + status, headers, body = _ERRORS[error_code] + try: + self.start_response(status, headers[:]) + self.write(body) + except socket.error: + if not PY3: + sys.exc_clear() + self.close_connection = True + + def _log_error(self, t, v, tb): + # TODO: Shouldn't we dump this to wsgi.errors? If we did that now, it would + # wind up getting logged twice + if not issubclass(t, GreenletExit): + context = self.environ + if not isinstance(context, self.server.secure_environ_class): + context = self.server.secure_environ_class(context) + self.server.loop.handle_error(context, t, v, tb) + + def handle_error(self, t, v, tb): + # Called for internal, unexpected errors, NOT invalid client input + self._log_error(t, v, tb) + del tb + self._send_error_response_if_possible(500) + + def _handle_client_error(self, ex): + # Called for invalid client input + # Returns the appropriate error response. + if not isinstance(ex, ValueError): + # XXX: Why not self._log_error to send it through the loop's + # handle_error method? + traceback.print_exc() + if isinstance(ex, _InvalidClientRequest): + # These come with good error messages, and we want to let + # log_error deal with the formatting, especially to handle encoding + self.log_error(*ex.args) + else: + self.log_error('Invalid request: %s', str(ex) or ex.__class__.__name__) + return ('400', _BAD_REQUEST_RESPONSE) + + def _headers(self): + key = None + value = None + IGNORED_KEYS = (None, 'CONTENT_TYPE', 'CONTENT_LENGTH') + for header in self.headers.headers: + if key is not None and header[:1] in " \t": + value += header + continue + + if key not in IGNORED_KEYS: + yield 'HTTP_' + key, value.strip() + + key, value = header.split(':', 1) + if '_' in key: + # strip incoming bad veaders + key = None + else: + key = key.replace('-', '_').upper() + + if key not in IGNORED_KEYS: + yield 'HTTP_' + key, value.strip() + + def get_environ(self): + """ + Construct and return a new WSGI environment dictionary for a specific request. + + This should begin with asking the server for the base environment + using :meth:`WSGIServer.get_environ`, and then proceed to add the + request specific values. + + By the time this method is invoked the request line and request shall have + been parsed and ``self.headers`` shall be populated. + """ + env = self.server.get_environ() + env['REQUEST_METHOD'] = self.command + env['SCRIPT_NAME'] = '' + + if '?' in self.path: + path, query = self.path.split('?', 1) + else: + path, query = self.path, '' + # Note that self.path contains the original str object; if it contains + # encoded escapes, it will NOT match PATH_INFO. + env['PATH_INFO'] = unquote_latin1(path) + env['QUERY_STRING'] = query + + if self.headers.typeheader is not None: + env['CONTENT_TYPE'] = self.headers.typeheader + + length = self.headers.getheader('content-length') + if length: + env['CONTENT_LENGTH'] = length + env['SERVER_PROTOCOL'] = self.request_version + + client_address = self.client_address + if isinstance(client_address, tuple): + env['REMOTE_ADDR'] = str(client_address[0]) + env['REMOTE_PORT'] = str(client_address[1]) + + for key, value in self._headers(): + if key in env: + if 'COOKIE' in key: + env[key] += '; ' + value + else: + env[key] += ',' + value + else: + env[key] = value + + if env.get('HTTP_EXPECT') == '100-continue': + sock = self.socket + else: + sock = None + + chunked = env.get('HTTP_TRANSFER_ENCODING', '').lower() == 'chunked' + self.wsgi_input = Input(self.rfile, self.content_length, socket=sock, chunked_input=chunked) + env['wsgi.input'] = self.wsgi_input + # This is a non-standard flag indicating that our input stream is + # self-terminated (returns EOF when consumed). + # See https://github.com/gevent/gevent/issues/1308 + env['wsgi.input_terminated'] = True + return env + + +class _NoopLog(object): + # Does nothing; implements just enough file-like methods + # to pass the WSGI validator + + def write(self, *args, **kwargs): + # pylint:disable=unused-argument + return + + def flush(self): + pass + + def writelines(self, *args, **kwargs): + pass + + +class LoggingLogAdapter(object): + """ + An adapter for :class:`logging.Logger` instances + to let them be used with :class:`WSGIServer`. + + .. warning:: Unless the entire process is monkey-patched at a very + early part of the lifecycle (before logging is configured), + loggers are likely to not be gevent-cooperative. For example, + the socket and syslog handlers use the socket module in a way + that can block, and most handlers acquire threading locks. + + .. warning:: It *may* be possible for the logging functions to be + called in the :class:`gevent.Hub` greenlet. Code running in the + hub greenlet cannot use any gevent blocking functions without triggering + a ``LoopExit``. + + .. versionadded:: 1.1a3 + + .. versionchanged:: 1.1b6 + Attributes not present on this object are proxied to the underlying + logger instance. This permits using custom :class:`~logging.Logger` + subclasses (or indeed, even duck-typed objects). + + .. versionchanged:: 1.1 + Strip trailing newline characters on the message passed to :meth:`write` + because log handlers will usually add one themselves. + """ + + # gevent avoids importing and using logging because importing it and + # creating loggers creates native locks unless monkey-patched. + + __slots__ = ('_logger', '_level') + + def __init__(self, logger, level=20): + """ + Write information to the *logger* at the given *level* (default to INFO). + """ + self._logger = logger + self._level = level + + def write(self, msg): + if msg and msg.endswith('\n'): + msg = msg[:-1] + self._logger.log(self._level, msg) + + def flush(self): + "No-op; required to be a file-like object" + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __getattr__(self, name): + return getattr(self._logger, name) + + def __setattr__(self, name, value): + if name not in LoggingLogAdapter.__slots__: + setattr(self._logger, name, value) + else: + object.__setattr__(self, name, value) + + def __delattr__(self, name): + delattr(self._logger, name) + +#### +## Environ classes. +# These subclass dict. They could subclass collections.UserDict on +# 3.3+ and proxy to the underlying real dict to avoid a copy if we +# have to print them (on 2.7 it's slightly more complicated to be an +# instance of collections.MutableMapping; UserDict.UserDict isn't.) +# Then we could have either the WSGIHandler.get_environ or the +# WSGIServer.get_environ return one of these proxies, and +# WSGIHandler.run_application would know to access the `environ.data` +# attribute to be able to pass the *real* dict to the application +# (because PEP3333 requires no subclasses, only actual dict objects; +# wsgiref.validator and webob.Request both enforce this). This has the +# advantage of not being fragile if anybody else tries to print/log +# self.environ (and not requiring a copy). However, if there are any +# subclasses of Handler or Server, this could break if they don't know +# to return this type. +#### + +class Environ(dict): + """ + A base class that can be used for WSGI environment objects. + + Provisional API. + + .. versionadded:: 1.2a1 + """ + + __slots__ = () # add no ivars or weakref ability + + def copy(self): + return self.__class__(self) + + if not hasattr(dict, 'iteritems'): + # Python 3 + def iteritems(self): + return self.items() + + def __reduce_ex__(self, proto): + return (dict, (), None, None, iter(self.iteritems())) + +class SecureEnviron(Environ): + """ + An environment that does not print its keys and values + by default. + + Provisional API. + + This is intended to keep potentially sensitive information like + HTTP authorization and cookies from being inadvertently printed + or logged. + + For debugging, each instance can have its *secure_repr* attribute + set to ``False``, which will cause it to print like a normal dict. + + When *secure_repr* is ``True`` (the default), then the value of + the *whitelist_keys* attribute is consulted; if this value is + true-ish, it should be a container (something that responds to + ``in``) of key names (typically a list or set). Keys and values in + this dictionary that are in *whitelist_keys* will then be printed, + while all other values will be masked. These values may be + customized on the class by setting the *default_secure_repr* and + *default_whitelist_keys*, respectively:: + + >>> environ = SecureEnviron(key='value') + >>> environ # doctest: +ELLIPSIS + >> environ.whitelist_keys = {'key'} + >>> environ + {'key': 'value'} + + A non-whitelisted key (*only*, to avoid doctest issues) is masked:: + + >>> environ['secure'] = 'secret'; del environ['key'] + >>> environ + {'secure': ''} + + We can turn it off entirely for the instance:: + + >>> environ.secure_repr = False + >>> environ + {'secure': 'secret'} + + We can also customize it at the class level (here we use a new + class to be explicit and to avoid polluting the true default + values; we would set this class to be the ``environ_class`` of the + server):: + + >>> class MyEnviron(SecureEnviron): + ... default_whitelist_keys = ('key',) + ... + >>> environ = MyEnviron({'key': 'value'}) + >>> environ + {'key': 'value'} + + .. versionadded:: 1.2a1 + """ + + default_secure_repr = True + default_whitelist_keys = () + default_print_masked_keys = True + + # Allow instances to override the class values, + # but inherit from the class if not present. Keeps instances + # small since we can't combine __slots__ with class attributes + # of the same name. + __slots__ = ('secure_repr', 'whitelist_keys', 'print_masked_keys') + + def __getattr__(self, name): + if name in SecureEnviron.__slots__: + return getattr(type(self), 'default_' + name) + raise AttributeError(name) + + def __repr__(self): + if self.secure_repr: + whitelist = self.whitelist_keys + print_masked = self.print_masked_keys + if whitelist: + safe = {k: self[k] if k in whitelist else "" + for k in self + if k in whitelist or print_masked} + safe_repr = repr(safe) + if not print_masked and len(safe) != len(self): + safe_repr = safe_repr[:-1] + ", (hidden keys: %d)}" % (len(self) - len(safe)) + return safe_repr + return "" % (len(self), id(self)) + return Environ.__repr__(self) + __str__ = __repr__ + + +class WSGISecureEnviron(SecureEnviron): + """ + Specializes the default list of whitelisted keys to a few + common WSGI variables. + + Example:: + + >>> environ = WSGISecureEnviron(REMOTE_ADDR='::1', HTTP_AUTHORIZATION='secret') + >>> environ + {'REMOTE_ADDR': '::1', (hidden keys: 1)} + >>> import pprint + >>> pprint.pprint(environ) + {'REMOTE_ADDR': '::1', (hidden keys: 1)} + >>> print(pprint.pformat(environ)) + {'REMOTE_ADDR': '::1', (hidden keys: 1)} + """ + default_whitelist_keys = ('REMOTE_ADDR', 'REMOTE_PORT', 'HTTP_HOST') + default_print_masked_keys = False + + +class WSGIServer(StreamServer): + """ + A WSGI server based on :class:`StreamServer` that supports HTTPS. + + + :keyword log: If given, an object with a ``write`` method to which + request (access) logs will be written. If not given, defaults + to :obj:`sys.stderr`. You may pass ``None`` to disable request + logging. You may use a wrapper, around e.g., :mod:`logging`, + to support objects that don't implement a ``write`` method. + (If you pass a :class:`~logging.Logger` instance, or in + general something that provides a ``log`` method but not a + ``write`` method, such a wrapper will automatically be created + and it will be logged to at the :data:`~logging.INFO` level.) + + :keyword error_log: If given, a file-like object with ``write``, + ``writelines`` and ``flush`` methods to which error logs will + be written. If not given, defaults to :obj:`sys.stderr`. You + may pass ``None`` to disable error logging (not recommended). + You may use a wrapper, around e.g., :mod:`logging`, to support + objects that don't implement the proper methods. This + parameter will become the value for ``wsgi.errors`` in the + WSGI environment (if not already set). (As with *log*, + wrappers for :class:`~logging.Logger` instances and the like + will be created automatically and logged to at the :data:`~logging.ERROR` + level.) + + .. seealso:: + + :class:`LoggingLogAdapter` + See important warnings before attempting to use :mod:`logging`. + + .. versionchanged:: 1.1a3 + Added the ``error_log`` parameter, and set ``wsgi.errors`` in the WSGI + environment to this value. + .. versionchanged:: 1.1a3 + Add support for passing :class:`logging.Logger` objects to the ``log`` and + ``error_log`` arguments. + """ + + #: A callable taking three arguments: (socket, address, server) and returning + #: an object with a ``handle()`` method. The callable is called once for + #: each incoming socket request, as is its handle method. The handle method should not + #: return until all use of the socket is complete. + #: + #: This class uses the :class:`WSGIHandler` object as the default value. You may + #: subclass this class and set a different default value, or you may pass + #: a value to use in the ``handler_class`` keyword constructor argument. + handler_class = WSGIHandler + + #: The object to which request logs will be written. + #: It must never be None. Initialized from the ``log`` constructor + #: parameter. + log = None + + #: The object to which error logs will be written. + #: It must never be None. Initialized from the ``error_log`` constructor + #: parameter. + error_log = None + + #: The class of environ objects passed to the handlers. + #: Must be a dict subclass. For compliance with :pep:`3333` + #: and libraries like WebOb, this is simply :class:`dict` + #: but this can be customized in a subclass or per-instance + #: (probably to :class:`WSGISecureEnviron`). + #: + #: .. versionadded:: 1.2a1 + environ_class = dict + + # Undocumented internal detail: the class that WSGIHandler._log_error + # will cast to before passing to the loop. + secure_environ_class = WSGISecureEnviron + + base_env = {'GATEWAY_INTERFACE': 'CGI/1.1', + 'SERVER_SOFTWARE': 'gevent/%d.%d Python/%d.%d' % (gevent.version_info[:2] + sys.version_info[:2]), + 'SCRIPT_NAME': '', + 'wsgi.version': (1, 0), + 'wsgi.multithread': False, # XXX: Aren't we really, though? + 'wsgi.multiprocess': False, + 'wsgi.run_once': False} + + def __init__(self, listener, application=None, backlog=None, spawn='default', + log='default', error_log='default', + handler_class=None, + environ=None, **ssl_args): + StreamServer.__init__(self, listener, backlog=backlog, spawn=spawn, **ssl_args) + if application is not None: + self.application = application + if handler_class is not None: + self.handler_class = handler_class + + # Note that we can't initialize these as class variables: + # sys.stderr might get monkey patched at runtime. + def _make_log(l, level=20): + if l == 'default': + return sys.stderr + if l is None: + return _NoopLog() + if not hasattr(l, 'write') and hasattr(l, 'log'): + return LoggingLogAdapter(l, level) + return l + self.log = _make_log(log) + self.error_log = _make_log(error_log, 40) # logging.ERROR + + self.set_environ(environ) + self.set_max_accept() + + def set_environ(self, environ=None): + if environ is not None: + self.environ = environ + environ_update = getattr(self, 'environ', None) + + self.environ = self.environ_class(self.base_env) + if self.ssl_enabled: + self.environ['wsgi.url_scheme'] = 'https' + else: + self.environ['wsgi.url_scheme'] = 'http' + if environ_update is not None: + self.environ.update(environ_update) + if self.environ.get('wsgi.errors') is None: + self.environ['wsgi.errors'] = self.error_log + + def set_max_accept(self): + if self.environ.get('wsgi.multiprocess'): + self.max_accept = 1 + + def get_environ(self): + return self.environ_class(self.environ) + + def init_socket(self): + StreamServer.init_socket(self) + self.update_environ() + + def update_environ(self): + """ + Called before the first request is handled to fill in WSGI environment values. + + This includes getting the correct server name and port. + """ + address = self.address + if isinstance(address, tuple): + if 'SERVER_NAME' not in self.environ: + try: + name = socket.getfqdn(address[0]) + except socket.error: + name = str(address[0]) + if PY3 and not isinstance(name, str): + name = name.decode('ascii') + self.environ['SERVER_NAME'] = name + self.environ.setdefault('SERVER_PORT', str(address[1])) + else: + self.environ.setdefault('SERVER_NAME', '') + self.environ.setdefault('SERVER_PORT', '') + + def handle(self, sock, address): + """ + Create an instance of :attr:`handler_class` to handle the request. + + This method blocks until the handler returns. + """ + # pylint:disable=method-hidden + handler = self.handler_class(sock, address, self) + handler.handle() + +def _main(): + # Provisional main handler, for quick tests, not production + # usage. + from gevent import monkey; monkey.patch_all() + + import argparse + import importlib + + parser = argparse.ArgumentParser() + parser.add_argument("app", help="dotted name of WSGI app callable [module:callable]") + parser.add_argument("-b", "--bind", + help="The socket to bind", + default=":8080") + + args = parser.parse_args() + + module_name, app_name = args.app.split(':') + module = importlib.import_module(module_name) + app = getattr(module, app_name) + bind = args.bind + + server = WSGIServer(bind, app) + server.serve_forever() + +if __name__ == '__main__': + _main() diff --git a/src/gevent/queue.py b/src/gevent/queue.py new file mode 100644 index 0000000..57b937b --- /dev/null +++ b/src/gevent/queue.py @@ -0,0 +1,685 @@ +# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details. +# copyright (c) 2018 gevent +# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False +""" +Synchronized queues. + +The :mod:`gevent.queue` module implements multi-producer, multi-consumer queues +that work across greenlets, with the API similar to the classes found in the +standard :mod:`Queue` and :class:`multiprocessing ` modules. + +The classes in this module implement the iterator protocol. Iterating +over a queue means repeatedly calling :meth:`get ` until +:meth:`get ` returns ``StopIteration`` (specifically that +class, not an instance or subclass). + + >>> queue = gevent.queue.Queue() + >>> queue.put(1) + >>> queue.put(2) + >>> queue.put(StopIteration) + >>> for item in queue: + ... print(item) + 1 + 2 + +.. versionchanged:: 1.0 + ``Queue(0)`` now means queue of infinite size, not a channel. A :exc:`DeprecationWarning` + will be issued with this argument. +""" + +from __future__ import absolute_import +import sys +from heapq import heappush as _heappush +from heapq import heappop as _heappop +from heapq import heapify as _heapify +import collections + +if sys.version_info[0] == 2: + import Queue as __queue__ # python 3: pylint:disable=import-error +else: + import queue as __queue__ # python 2: pylint:disable=import-error +Full = __queue__.Full +Empty = __queue__.Empty + +from gevent.timeout import Timeout +from gevent._hub_local import get_hub_noargs as get_hub +from greenlet import getcurrent +from gevent.exceptions import InvalidSwitchError + +__all__ = [] +__implements__ = ['Queue', 'PriorityQueue', 'LifoQueue'] +__extensions__ = ['JoinableQueue', 'Channel'] +__imports__ = ['Empty', 'Full'] +if hasattr(__queue__, 'SimpleQueue'): + __all__.append('SimpleQueue') # New in 3.7 + # SimpleQueue is implemented in C and directly allocates locks + # unaffected by monkey patching. We need the Python version. + SimpleQueue = __queue__._PySimpleQueue # pylint:disable=no-member +__all__ += (__implements__ + __extensions__ + __imports__) + + +# pylint 2.0.dev2 things collections.dequeue.popleft() doesn't return +# pylint:disable=assignment-from-no-return + +def _safe_remove(deq, item): + # For when the item may have been removed by + # Queue._unlock + try: + deq.remove(item) + except ValueError: + pass + +import gevent._waiter +locals()['Waiter'] = gevent._waiter.Waiter + +class ItemWaiter(Waiter): # pylint:disable=undefined-variable + # pylint:disable=assigning-non-slot + __slots__ = ( + 'item', + 'queue', + ) + + def __init__(self, item, queue): + Waiter.__init__(self) # pylint:disable=undefined-variable + self.item = item + self.queue = queue + + def put_and_switch(self): + self.queue._put(self.item) + self.queue = None + self.item = None + return self.switch(self) + +class Queue(object): + """ + Create a queue object with a given maximum size. + + If *maxsize* is less than or equal to zero or ``None``, the queue + size is infinite. + + Queues have a ``len`` equal to the number of items in them (the :meth:`qsize`), + but in a boolean context they are always True. + + .. versionchanged:: 1.1b3 + Queues now support :func:`len`; it behaves the same as :meth:`qsize`. + .. versionchanged:: 1.1b3 + Multiple greenlets that block on a call to :meth:`put` for a full queue + will now be awakened to put their items into the queue in the order in which + they arrived. Likewise, multiple greenlets that block on a call to :meth:`get` for + an empty queue will now receive items in the order in which they blocked. An + implementation quirk under CPython *usually* ensured this was roughly the case + previously anyway, but that wasn't the case for PyPy. + """ + + __slots__ = ( + '_maxsize', + 'getters', + 'putters', + 'hub', + '_event_unlock', + 'queue', + '__weakref__', + ) + + def __init__(self, maxsize=None, items=(), _warn_depth=2): + if maxsize is not None and maxsize <= 0: + if maxsize == 0: + import warnings + warnings.warn( + 'Queue(0) now equivalent to Queue(None); if you want a channel, use Channel', + DeprecationWarning, + stacklevel=_warn_depth) + maxsize = None + + self._maxsize = maxsize if maxsize is not None else -1 + # Explicitly maintain order for getters and putters that block + # so that callers can consistently rely on getting things out + # in the apparent order they went in. This was once required by + # imap_unordered. Previously these were set() objects, and the + # items put in the set have default hash() and eq() methods; + # under CPython, since new objects tend to have increasing + # hash values, this tended to roughly maintain order anyway, + # but that's not true under PyPy. An alternative to a deque + # (to avoid the linear scan of remove()) might be an + # OrderedDict, but it's 2.7 only; we don't expect to have so + # many waiters that removing an arbitrary element is a + # bottleneck, though. + self.getters = collections.deque() + self.putters = collections.deque() + self.hub = get_hub() + self._event_unlock = None + self.queue = self._create_queue(items) + + @property + def maxsize(self): + return self._maxsize if self._maxsize > 0 else None + + @maxsize.setter + def maxsize(self, nv): + # QQQ make maxsize into a property with setter that schedules unlock if necessary + if nv is None or nv <= 0: + self._maxsize = -1 + else: + self._maxsize = nv + + def copy(self): + return type(self)(self.maxsize, self.queue) + + def _create_queue(self, items=()): + return collections.deque(items) + + def _get(self): + return self.queue.popleft() + + def _peek(self): + return self.queue[0] + + def _put(self, item): + self.queue.append(item) + + def __repr__(self): + return '<%s at %s%s>' % (type(self).__name__, hex(id(self)), self._format()) + + def __str__(self): + return '<%s%s>' % (type(self).__name__, self._format()) + + def _format(self): + result = [] + if self.maxsize is not None: + result.append('maxsize=%r' % (self.maxsize, )) + if getattr(self, 'queue', None): + result.append('queue=%r' % (self.queue, )) + if self.getters: + result.append('getters[%s]' % len(self.getters)) + if self.putters: + result.append('putters[%s]' % len(self.putters)) + if result: + return ' ' + ' '.join(result) + return '' + + def qsize(self): + """Return the size of the queue.""" + return len(self.queue) + + def __len__(self): + """ + Return the size of the queue. This is the same as :meth:`qsize`. + + .. versionadded: 1.1b3 + + Previously, getting len() of a queue would raise a TypeError. + """ + + return self.qsize() + + def __bool__(self): + """ + A queue object is always True. + + .. versionadded: 1.1b3 + + Now that queues support len(), they need to implement ``__bool__`` + to return True for backwards compatibility. + """ + return True + + def __nonzero__(self): + # Py2. + # For Cython; __bool__ becomes a special method that we can't + # get by name. + return True + + def empty(self): + """Return ``True`` if the queue is empty, ``False`` otherwise.""" + return not self.qsize() + + def full(self): + """Return ``True`` if the queue is full, ``False`` otherwise. + + ``Queue(None)`` is never full. + """ + return self._maxsize > 0 and self.qsize() >= self._maxsize + + def put(self, item, block=True, timeout=None): + """Put an item into the queue. + + If optional arg *block* is true and *timeout* is ``None`` (the default), + block if necessary until a free slot is available. If *timeout* is + a positive number, it blocks at most *timeout* seconds and raises + the :class:`Full` exception if no free slot was available within that time. + Otherwise (*block* is false), put an item on the queue if a free slot + is immediately available, else raise the :class:`Full` exception (*timeout* + is ignored in that case). + """ + if self._maxsize == -1 or self.qsize() < self._maxsize: + # there's a free slot, put an item right away + self._put(item) + if self.getters: + self._schedule_unlock() + elif self.hub is getcurrent(): + # We're in the mainloop, so we cannot wait; we can switch to other greenlets though. + # Check if possible to get a free slot in the queue. + while self.getters and self.qsize() and self.qsize() >= self._maxsize: + getter = self.getters.popleft() + getter.switch(getter) + if self.qsize() < self._maxsize: + self._put(item) + return + raise Full + elif block: + waiter = ItemWaiter(item, self) + self.putters.append(waiter) + timeout = Timeout._start_new_or_dummy(timeout, Full) + try: + if self.getters: + self._schedule_unlock() + result = waiter.get() + if result is not waiter: + raise InvalidSwitchError("Invalid switch into Queue.put: %r" % (result, )) + finally: + timeout.cancel() + _safe_remove(self.putters, waiter) + else: + raise Full + + def put_nowait(self, item): + """Put an item into the queue without blocking. + + Only enqueue the item if a free slot is immediately available. + Otherwise raise the :class:`Full` exception. + """ + self.put(item, False) + + def __get_or_peek(self, method, block, timeout): + # Internal helper method. The `method` should be either + # self._get when called from self.get() or self._peek when + # called from self.peek(). Call this after the initial check + # to see if there are items in the queue. + + if self.hub is getcurrent(): + # special case to make get_nowait() or peek_nowait() runnable in the mainloop greenlet + # there are no items in the queue; try to fix the situation by unlocking putters + while self.putters: + # Note: get() used popleft(), peek used pop(); popleft + # is almost certainly correct. + self.putters.popleft().put_and_switch() + if self.qsize(): + return method() + raise Empty() + + if not block: + # We can't block, we're not the hub, and we have nothing + # to return. No choice... + raise Empty() + + waiter = Waiter() # pylint:disable=undefined-variable + timeout = Timeout._start_new_or_dummy(timeout, Empty) + try: + self.getters.append(waiter) + if self.putters: + self._schedule_unlock() + result = waiter.get() + if result is not waiter: + raise InvalidSwitchError('Invalid switch into Queue.get: %r' % (result, )) + return method() + finally: + timeout.cancel() + _safe_remove(self.getters, waiter) + + def get(self, block=True, timeout=None): + """Remove and return an item from the queue. + + If optional args *block* is true and *timeout* is ``None`` (the default), + block if necessary until an item is available. If *timeout* is a positive number, + it blocks at most *timeout* seconds and raises the :class:`Empty` exception + if no item was available within that time. Otherwise (*block* is false), return + an item if one is immediately available, else raise the :class:`Empty` exception + (*timeout* is ignored in that case). + """ + if self.qsize(): + if self.putters: + self._schedule_unlock() + return self._get() + + return self.__get_or_peek(self._get, block, timeout) + + def get_nowait(self): + """Remove and return an item from the queue without blocking. + + Only get an item if one is immediately available. Otherwise + raise the :class:`Empty` exception. + """ + return self.get(False) + + def peek(self, block=True, timeout=None): + """Return an item from the queue without removing it. + + If optional args *block* is true and *timeout* is ``None`` (the default), + block if necessary until an item is available. If *timeout* is a positive number, + it blocks at most *timeout* seconds and raises the :class:`Empty` exception + if no item was available within that time. Otherwise (*block* is false), return + an item if one is immediately available, else raise the :class:`Empty` exception + (*timeout* is ignored in that case). + """ + if self.qsize(): + # XXX: Why doesn't this schedule an unlock like get() does? + return self._peek() + + return self.__get_or_peek(self._peek, block, timeout) + + def peek_nowait(self): + """Return an item from the queue without blocking. + + Only return an item if one is immediately available. Otherwise + raise the :class:`Empty` exception. + """ + return self.peek(False) + + def _unlock(self): + while True: + repeat = False + if self.putters and (self._maxsize == -1 or self.qsize() < self._maxsize): + repeat = True + try: + putter = self.putters.popleft() + self._put(putter.item) + except: # pylint:disable=bare-except + putter.throw(*sys.exc_info()) + else: + putter.switch(putter) + if self.getters and self.qsize(): + repeat = True + getter = self.getters.popleft() + getter.switch(getter) + if not repeat: + return + + def _schedule_unlock(self): + if not self._event_unlock: + self._event_unlock = self.hub.loop.run_callback(self._unlock) + + def __iter__(self): + return self + + def __next__(self): + result = self.get() + if result is StopIteration: + raise result + return result + + next = __next__ # Py2 + + +class UnboundQueue(Queue): + # A specialization of Queue that knows it can never + # be bound. Changing its maxsize has no effect. + + __slots__ = () + + def __init__(self, maxsize=None, items=()): + if maxsize is not None: + raise ValueError("UnboundQueue has no maxsize") + Queue.__init__(self, maxsize, items) + self.putters = None # Will never be used. + + def put(self, item, block=True, timeout=None): + self._put(item) + if self.getters: + self._schedule_unlock() + + +class PriorityQueue(Queue): + '''A subclass of :class:`Queue` that retrieves entries in priority order (lowest first). + + Entries are typically tuples of the form: ``(priority number, data)``. + + .. versionchanged:: 1.2a1 + Any *items* given to the constructor will now be passed through + :func:`heapq.heapify` to ensure the invariants of this class hold. + Previously it was just assumed that they were already a heap. + ''' + + __slots__ = () + + def _create_queue(self, items=()): + q = list(items) + _heapify(q) + return q + + def _put(self, item): + _heappush(self.queue, item) + + def _get(self): + return _heappop(self.queue) + + +class LifoQueue(Queue): + '''A subclass of :class:`Queue` that retrieves most recently added entries first.''' + + __slots__ = () + + def _create_queue(self, items=()): + return list(items) + + def _put(self, item): + self.queue.append(item) + + def _get(self): + return self.queue.pop() + + def _peek(self): + return self.queue[-1] + + +class JoinableQueue(Queue): + """ + A subclass of :class:`Queue` that additionally has + :meth:`task_done` and :meth:`join` methods. + """ + + __slots__ = ( + '_cond', + 'unfinished_tasks', + ) + + def __init__(self, maxsize=None, items=(), unfinished_tasks=None): + """ + + .. versionchanged:: 1.1a1 + If *unfinished_tasks* is not given, then all the given *items* + (if any) will be considered unfinished. + + """ + Queue.__init__(self, maxsize, items, _warn_depth=3) + + from gevent.event import Event + self._cond = Event() + self._cond.set() + + if unfinished_tasks: + self.unfinished_tasks = unfinished_tasks + elif items: + self.unfinished_tasks = len(items) + else: + self.unfinished_tasks = 0 + + if self.unfinished_tasks: + self._cond.clear() + + def copy(self): + return type(self)(self.maxsize, self.queue, self.unfinished_tasks) + + def _format(self): + result = Queue._format(self) + if self.unfinished_tasks: + result += ' tasks=%s _cond=%s' % (self.unfinished_tasks, self._cond) + return result + + def _put(self, item): + Queue._put(self, item) + self.unfinished_tasks += 1 + self._cond.clear() + + def task_done(self): + '''Indicate that a formerly enqueued task is complete. Used by queue consumer threads. + For each :meth:`get ` used to fetch a task, a subsequent call to :meth:`task_done` tells the queue + that the processing on the task is complete. + + If a :meth:`join` is currently blocking, it will resume when all items have been processed + (meaning that a :meth:`task_done` call was received for every item that had been + :meth:`put ` into the queue). + + Raises a :exc:`ValueError` if called more times than there were items placed in the queue. + ''' + if self.unfinished_tasks <= 0: + raise ValueError('task_done() called too many times') + self.unfinished_tasks -= 1 + if self.unfinished_tasks == 0: + self._cond.set() + + def join(self, timeout=None): + ''' + Block until all items in the queue have been gotten and processed. + + The count of unfinished tasks goes up whenever an item is added to the queue. + The count goes down whenever a consumer thread calls :meth:`task_done` to indicate + that the item was retrieved and all work on it is complete. When the count of + unfinished tasks drops to zero, :meth:`join` unblocks. + + :param float timeout: If not ``None``, then wait no more than this time in seconds + for all tasks to finish. + :return: ``True`` if all tasks have finished; if ``timeout`` was given and expired before + all tasks finished, ``False``. + + .. versionchanged:: 1.1a1 + Add the *timeout* parameter. + ''' + return self._cond.wait(timeout=timeout) + + +class Channel(object): + + __slots__ = ( + 'getters', + 'putters', + 'hub', + '_event_unlock', + '__weakref__', + ) + + def __init__(self, maxsize=1): + # We take maxsize to simplify certain kinds of code + if maxsize != 1: + raise ValueError("Channels have a maxsize of 1") + self.getters = collections.deque() + self.putters = collections.deque() + self.hub = get_hub() + self._event_unlock = None + + def __repr__(self): + return '<%s at %s %s>' % (type(self).__name__, hex(id(self)), self._format()) + + def __str__(self): + return '<%s %s>' % (type(self).__name__, self._format()) + + def _format(self): + result = '' + if self.getters: + result += ' getters[%s]' % len(self.getters) + if self.putters: + result += ' putters[%s]' % len(self.putters) + return result + + @property + def balance(self): + return len(self.putters) - len(self.getters) + + def qsize(self): + return 0 + + def empty(self): + return True + + def full(self): + return True + + def put(self, item, block=True, timeout=None): + if self.hub is getcurrent(): + if self.getters: + getter = self.getters.popleft() + getter.switch(item) + return + raise Full + + if not block: + timeout = 0 + + waiter = Waiter() # pylint:disable=undefined-variable + item = (item, waiter) + self.putters.append(item) + timeout = Timeout._start_new_or_dummy(timeout, Full) + try: + if self.getters: + self._schedule_unlock() + result = waiter.get() + if result is not waiter: + raise InvalidSwitchError("Invalid switch into Channel.put: %r" % (result, )) + except: + _safe_remove(self.putters, item) + raise + finally: + timeout.cancel() + + def put_nowait(self, item): + self.put(item, False) + + def get(self, block=True, timeout=None): + if self.hub is getcurrent(): + if self.putters: + item, putter = self.putters.popleft() + self.hub.loop.run_callback(putter.switch, putter) + return item + + if not block: + timeout = 0 + + waiter = Waiter() # pylint:disable=undefined-variable + timeout = Timeout._start_new_or_dummy(timeout, Empty) + try: + self.getters.append(waiter) + if self.putters: + self._schedule_unlock() + return waiter.get() + except: + self.getters.remove(waiter) + raise + finally: + timeout.close() + + def get_nowait(self): + return self.get(False) + + def _unlock(self): + while self.putters and self.getters: + getter = self.getters.popleft() + item, putter = self.putters.popleft() + getter.switch(item) + putter.switch(putter) + + def _schedule_unlock(self): + if not self._event_unlock: + self._event_unlock = self.hub.loop.run_callback(self._unlock) + + def __iter__(self): + return self + + def __next__(self): + result = self.get() + if result is StopIteration: + raise result + return result + + next = __next__ # Py2 + +from gevent._util import import_c_accel +import_c_accel(globals(), 'gevent._queue') diff --git a/src/gevent/resolver/__init__.py b/src/gevent/resolver/__init__.py new file mode 100644 index 0000000..bffa292 --- /dev/null +++ b/src/gevent/resolver/__init__.py @@ -0,0 +1,105 @@ +# Copyright (c) 2018 gevent contributors. See LICENSE for details. + +from _socket import gaierror +from _socket import error +from _socket import getservbyname +from _socket import getaddrinfo +from _socket import SOCK_STREAM +from _socket import SOCK_DGRAM +from _socket import SOL_TCP +from _socket import AI_CANONNAME +from _socket import EAI_SERVICE +from _socket import AF_INET +from _socket import AI_PASSIVE + + +from gevent._compat import string_types +from gevent._compat import integer_types + +# Nothing public here. +__all__ = [] + +def _lookup_port(port, socktype): + # pylint:disable=too-many-branches + socktypes = [] + if isinstance(port, string_types): + try: + port = int(port) + except ValueError: + try: + if socktype == 0: + origport = port + try: + port = getservbyname(port, 'tcp') + socktypes.append(SOCK_STREAM) + except error: + port = getservbyname(port, 'udp') + socktypes.append(SOCK_DGRAM) + else: + try: + if port == getservbyname(origport, 'udp'): + socktypes.append(SOCK_DGRAM) + except error: + pass + elif socktype == SOCK_STREAM: + port = getservbyname(port, 'tcp') + elif socktype == SOCK_DGRAM: + port = getservbyname(port, 'udp') + else: + raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype') + except error as ex: + if 'not found' in str(ex): + raise gaierror(EAI_SERVICE, 'Servname not supported for ai_socktype') + else: + raise gaierror(str(ex)) + except UnicodeEncodeError: + raise error('Int or String expected', port) + elif port is None: + port = 0 + elif isinstance(port, integer_types): + pass + else: + raise error('Int or String expected', port, type(port)) + port = int(port % 65536) + if not socktypes and socktype: + socktypes.append(socktype) + return port, socktypes + +hostname_types = tuple(set(string_types + (bytearray, bytes))) + +def _resolve_special(hostname, family): + if not isinstance(hostname, hostname_types): + raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(hostname),)) + + if hostname == '': + result = getaddrinfo(None, 0, family, SOCK_DGRAM, 0, AI_PASSIVE) + if len(result) != 1: + raise error('wildcard resolved to multiple address') + return result[0][4][0] + return hostname + + +class AbstractResolver(object): + + def gethostbyname(self, hostname, family=AF_INET): + hostname = _resolve_special(hostname, family) + return self.gethostbyname_ex(hostname, family)[-1][0] + + def gethostbyname_ex(self, hostname, family=AF_INET): + aliases = self._getaliases(hostname, family) + addresses = [] + tuples = self.getaddrinfo(hostname, 0, family, + SOCK_STREAM, + SOL_TCP, AI_CANONNAME) + canonical = tuples[0][3] + for item in tuples: + addresses.append(item[4][0]) + # XXX we just ignore aliases + return (canonical, aliases, addresses) + + def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): + raise NotImplementedError() + + def _getaliases(self, hostname, family): + # pylint:disable=unused-argument + return [] diff --git a/src/gevent/resolver/ares.py b/src/gevent/resolver/ares.py new file mode 100644 index 0000000..ea6e919 --- /dev/null +++ b/src/gevent/resolver/ares.py @@ -0,0 +1,357 @@ +# Copyright (c) 2011-2015 Denis Bilenko. See LICENSE for details. +""" +c-ares based hostname resolver. +""" +from __future__ import absolute_import, print_function, division +import os +import sys + +from _socket import getaddrinfo +from _socket import gaierror +from _socket import error + +from gevent._compat import string_types +from gevent._compat import text_type + +from gevent._compat import reraise +from gevent._compat import PY3 + +from gevent.hub import Waiter +from gevent.hub import get_hub + +from gevent.socket import AF_UNSPEC +from gevent.socket import AF_INET +from gevent.socket import AF_INET6 +from gevent.socket import SOCK_STREAM +from gevent.socket import SOCK_DGRAM +from gevent.socket import SOCK_RAW +from gevent.socket import AI_NUMERICHOST + +from gevent._config import config +from gevent._config import AresSettingMixin + +from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module +from . import _lookup_port as lookup_port +from . import _resolve_special +from . import AbstractResolver + +__all__ = ['Resolver'] + + +class Resolver(AbstractResolver): + """ + Implementation of the resolver API using the `c-ares`_ library. + + This implementation uses the c-ares library to handle name + resolution. c-ares is natively asynchronous at the socket level + and so integrates well into gevent's event loop. + + In comparison to :class:`gevent.resolver_thread.Resolver` (which + delegates to the native system resolver), the implementation is + much more complex. In addition, there have been reports of it not + properly honoring certain system configurations (for example, the + order in which IPv4 and IPv6 results are returned may not match + the threaded resolver). However, because it does not use threads, + it may scale better for applications that make many lookups. + + There are some known differences from the system resolver: + + - ``gethostbyname_ex`` and ``gethostbyaddr`` may return different + for the ``aliaslist`` tuple member. (Sometimes the same, + sometimes in a different order, sometimes a different alias + altogether.) + - ``gethostbyname_ex`` may return the ``ipaddrlist`` in a different order. + - ``getaddrinfo`` does not return ``SOCK_RAW`` results. + - ``getaddrinfo`` may return results in a different order. + - Handling of ``.local`` (mDNS) names may be different, even if they are listed in + the hosts file. + - c-ares will not resolve ``broadcasthost``, even if listed in the hosts file. + - This implementation may raise ``gaierror(4)`` where the system implementation would raise + ``herror(1)``. + - The results for ``localhost`` may be different. In particular, some system + resolvers will return more results from ``getaddrinfo`` than c-ares does, + such as SOCK_DGRAM results, and c-ares may report more ips on a multi-homed + host. + + .. caution:: This module is considered extremely experimental on PyPy, and + due to its implementation in cython, it may be slower. It may also lead to + interpreter crashes. + + .. _c-ares: http://c-ares.haxx.se + """ + + ares_class = channel + + def __init__(self, hub=None, use_environ=True, **kwargs): + if hub is None: + hub = get_hub() + self.hub = hub + if use_environ: + for setting in config.settings.values(): + if isinstance(setting, AresSettingMixin): + value = setting.get() + if value is not None: + kwargs.setdefault(setting.kwarg_name, value) + self.ares = self.ares_class(hub.loop, **kwargs) + self.pid = os.getpid() + self.params = kwargs + self.fork_watcher = hub.loop.fork(ref=False) + self.fork_watcher.start(self._on_fork) + + def __repr__(self): + return '' % (id(self), self.ares) + + def _on_fork(self): + # NOTE: See comment in gevent.hub.reinit. + pid = os.getpid() + if pid != self.pid: + self.hub.loop.run_callback(self.ares.destroy) + self.ares = self.ares_class(self.hub.loop, **self.params) + self.pid = pid + + def close(self): + if self.ares is not None: + self.hub.loop.run_callback(self.ares.destroy) + self.ares = None + self.fork_watcher.stop() + + def gethostbyname(self, hostname, family=AF_INET): + hostname = _resolve_special(hostname, family) + return self.gethostbyname_ex(hostname, family)[-1][0] + + def gethostbyname_ex(self, hostname, family=AF_INET): + if PY3: + if isinstance(hostname, str): + hostname = hostname.encode('idna') + elif not isinstance(hostname, (bytes, bytearray)): + raise TypeError('Expected es(idna), not %s' % type(hostname).__name__) + else: + if isinstance(hostname, text_type): + hostname = hostname.encode('ascii') + elif not isinstance(hostname, str): + raise TypeError('Expected string, not %s' % type(hostname).__name__) + + while True: + ares = self.ares + try: + waiter = Waiter(self.hub) + ares.gethostbyname(waiter, hostname, family) + result = waiter.get() + if not result[-1]: + raise gaierror(-5, 'No address associated with hostname') + return result + except gaierror: + if ares is self.ares: + if hostname == b'255.255.255.255': + # The stdlib handles this case in 2.7 and 3.x, but ares does not. + # It is tested by test_socket.py in 3.4. + # HACK: So hardcode the expected return. + return ('255.255.255.255', [], ['255.255.255.255']) + raise + # "self.ares is not ares" means channel was destroyed (because we were forked) + + def _lookup_port(self, port, socktype): + return lookup_port(port, socktype) + + def _getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): + # pylint:disable=too-many-locals,too-many-branches + if isinstance(host, text_type): + host = host.encode('idna') + elif not isinstance(host, str) or (flags & AI_NUMERICHOST): + # this handles cases which do not require network access + # 1) host is None + # 2) host is of an invalid type + # 3) AI_NUMERICHOST flag is set + return getaddrinfo(host, port, family, socktype, proto, flags) + # we also call _socket.getaddrinfo below if family is not one of AF_* + + port, socktypes = self._lookup_port(port, socktype) + + socktype_proto = [(SOCK_STREAM, 6), (SOCK_DGRAM, 17), (SOCK_RAW, 0)] + if socktypes: + socktype_proto = [(x, y) for (x, y) in socktype_proto if x in socktypes] + if proto: + socktype_proto = [(x, y) for (x, y) in socktype_proto if proto == y] + + ares = self.ares + + if family == AF_UNSPEC: + ares_values = Values(self.hub, 2) + ares.gethostbyname(ares_values, host, AF_INET) + ares.gethostbyname(ares_values, host, AF_INET6) + elif family == AF_INET: + ares_values = Values(self.hub, 1) + ares.gethostbyname(ares_values, host, AF_INET) + elif family == AF_INET6: + ares_values = Values(self.hub, 1) + ares.gethostbyname(ares_values, host, AF_INET6) + else: + raise gaierror(5, 'ai_family not supported: %r' % (family, )) + + values = ares_values.get() + if len(values) == 2 and values[0] == values[1]: + values.pop() + + result = [] + result4 = [] + result6 = [] + + for addrs in values: + if addrs.family == AF_INET: + for addr in addrs[-1]: + sockaddr = (addr, port) + for socktype4, proto4 in socktype_proto: + result4.append((AF_INET, socktype4, proto4, '', sockaddr)) + elif addrs.family == AF_INET6: + for addr in addrs[-1]: + if addr == '::1': + dest = result + else: + dest = result6 + sockaddr = (addr, port, 0, 0) + for socktype6, proto6 in socktype_proto: + dest.append((AF_INET6, socktype6, proto6, '', sockaddr)) + + # As of 2016, some platforms return IPV6 first and some do IPV4 first, + # and some might even allow configuration of which is which. For backwards + # compatibility with earlier releases (but not necessarily resolver_thread!) + # we return 4 first. See https://github.com/gevent/gevent/issues/815 for more. + result += result4 + result6 + + if not result: + raise gaierror(-5, 'No address associated with hostname') + + return result + + def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): + while True: + ares = self.ares + try: + return self._getaddrinfo(host, port, family, socktype, proto, flags) + except gaierror: + if ares is self.ares: + raise + + def _gethostbyaddr(self, ip_address): + if PY3: + if isinstance(ip_address, str): + ip_address = ip_address.encode('idna') + elif not isinstance(ip_address, (bytes, bytearray)): + raise TypeError('Expected es(idna), not %s' % type(ip_address).__name__) + else: + if isinstance(ip_address, text_type): + ip_address = ip_address.encode('ascii') + elif not isinstance(ip_address, str): + raise TypeError('Expected string, not %s' % type(ip_address).__name__) + + waiter = Waiter(self.hub) + try: + self.ares.gethostbyaddr(waiter, ip_address) + return waiter.get() + except InvalidIP: + result = self._getaddrinfo(ip_address, None, family=AF_UNSPEC, socktype=SOCK_DGRAM) + if not result: + raise + _ip_address = result[0][-1][0] + if isinstance(_ip_address, text_type): + _ip_address = _ip_address.encode('ascii') + if _ip_address == ip_address: + raise + waiter.clear() + self.ares.gethostbyaddr(waiter, _ip_address) + return waiter.get() + + def gethostbyaddr(self, ip_address): + ip_address = _resolve_special(ip_address, AF_UNSPEC) + while True: + ares = self.ares + try: + return self._gethostbyaddr(ip_address) + except gaierror: + if ares is self.ares: + raise + + def _getnameinfo(self, sockaddr, flags): + if not isinstance(flags, int): + raise TypeError('an integer is required') + if not isinstance(sockaddr, tuple): + raise TypeError('getnameinfo() argument 1 must be a tuple') + + address = sockaddr[0] + if not PY3 and isinstance(address, text_type): + address = address.encode('ascii') + + if not isinstance(address, string_types): + raise TypeError('sockaddr[0] must be a string, not %s' % type(address).__name__) + + port = sockaddr[1] + if not isinstance(port, int): + raise TypeError('port must be an integer, not %s' % type(port)) + + waiter = Waiter(self.hub) + result = self._getaddrinfo(address, str(sockaddr[1]), family=AF_UNSPEC, socktype=SOCK_DGRAM) + if not result: + reraise(*sys.exc_info()) + elif len(result) != 1: + raise error('sockaddr resolved to multiple addresses') + family, _socktype, _proto, _name, address = result[0] + + if family == AF_INET: + if len(sockaddr) != 2: + raise error("IPv4 sockaddr must be 2 tuple") + elif family == AF_INET6: + address = address[:2] + sockaddr[2:] + + self.ares.getnameinfo(waiter, address, flags) + node, service = waiter.get() + + if service is None: + if PY3: + # ares docs: "If the query did not complete + # successfully, or one of the values was not + # requested, node or service will be NULL ". Python 2 + # allows that for the service, but Python 3 raises + # an error. This is tested by test_socket in py 3.4 + err = gaierror('nodename nor servname provided, or not known') + err.errno = 8 + raise err + service = '0' + return node, service + + def getnameinfo(self, sockaddr, flags): + while True: + ares = self.ares + try: + return self._getnameinfo(sockaddr, flags) + except gaierror: + if ares is self.ares: + raise + + +class Values(object): + # helper to collect multiple values; ignore errors unless nothing has succeeded + # QQQ could probably be moved somewhere - hub.py? + + __slots__ = ['count', 'values', 'error', 'waiter'] + + def __init__(self, hub, count): + self.count = count + self.values = [] + self.error = None + self.waiter = Waiter(hub) + + def __call__(self, source): + self.count -= 1 + if source.exception is None: + self.values.append(source.value) + else: + self.error = source.exception + if self.count <= 0: + self.waiter.switch(None) + + def get(self): + self.waiter.get() + if self.values: + return self.values + assert error is not None + raise self.error # pylint:disable=raising-bad-type diff --git a/src/gevent/resolver/blocking.py b/src/gevent/resolver/blocking.py new file mode 100644 index 0000000..4a26a76 --- /dev/null +++ b/src/gevent/resolver/blocking.py @@ -0,0 +1,45 @@ +# Copyright (c) 2018 gevent contributors. See LICENSE for details. + +import _socket + +__all__ = [ + 'Resolver', +] + +class Resolver(object): + """ + A resolver that directly uses the system's resolver functions. + + .. caution:: + + This resolver is *not* cooperative. + + This resolver has the lowest overhead of any resolver and + typically approaches the speed of the unmodified :mod:`socket` + functions. However, it is not cooperative, so if name resolution + blocks, the entire thread and all its greenlets will be blocked. + + This can be useful during debugging, or it may be a good choice if + your operating system provides a good caching resolver (such as + macOS's Directory Services) that is usually very fast and + functionally non-blocking. + + .. versionchanged:: 1.3a2 + This was previously undocumented and existed in :mod:`gevent.socket`. + + """ + + def __init__(self, hub=None): + pass + + def close(self): + pass + + for method in ( + 'gethostbyname', + 'gethostbyname_ex', + 'getaddrinfo', + 'gethostbyaddr', + 'getnameinfo' + ): + locals()[method] = staticmethod(getattr(_socket, method)) diff --git a/src/gevent/resolver/cares.pyx b/src/gevent/resolver/cares.pyx new file mode 100644 index 0000000..cc43c13 --- /dev/null +++ b/src/gevent/resolver/cares.pyx @@ -0,0 +1,464 @@ +# Copyright (c) 2011-2012 Denis Bilenko. See LICENSE for details. +# Automatic pickling of cdef classes was added in 0.26. Unfortunately it +# seems to be buggy (at least for the `result` class) and produces code that +# can't compile ("local variable 'result' referenced before assignment"). +# See https://github.com/cython/cython/issues/1786 +# cython: auto_pickle=False +cimport libcares as cares +import sys + +from cpython.ref cimport Py_INCREF +from cpython.ref cimport Py_DECREF +from cpython.mem cimport PyMem_Malloc +from cpython.mem cimport PyMem_Free + +from _socket import gaierror + + +__all__ = ['channel'] + +cdef object string_types +cdef object text_type + +if sys.version_info[0] >= 3: + string_types = str, + text_type = str +else: + string_types = __builtins__.basestring, + text_type = __builtins__.unicode + +TIMEOUT = 1 + +DEF EV_READ = 1 +DEF EV_WRITE = 2 + + +cdef extern from "dnshelper.c": + int AF_INET + int AF_INET6 + + struct hostent: + char* h_name + int h_addrtype + + struct sockaddr_t "sockaddr": + pass + + struct ares_channeldata: + pass + + object parse_h_name(hostent*) + object parse_h_aliases(hostent*) + object parse_h_addr_list(hostent*) + void* create_object_from_hostent(void*) + + # this imports _socket lazily + object PyUnicode_FromString(char*) + int PyTuple_Check(object) + int PyArg_ParseTuple(object, char*, ...) except 0 + struct sockaddr_in6: + pass + int gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, sockaddr_in6* sa6) + + + void memset(void*, int, int) + + +ARES_SUCCESS = cares.ARES_SUCCESS +ARES_ENODATA = cares.ARES_ENODATA +ARES_EFORMERR = cares.ARES_EFORMERR +ARES_ESERVFAIL = cares.ARES_ESERVFAIL +ARES_ENOTFOUND = cares.ARES_ENOTFOUND +ARES_ENOTIMP = cares.ARES_ENOTIMP +ARES_EREFUSED = cares.ARES_EREFUSED +ARES_EBADQUERY = cares.ARES_EBADQUERY +ARES_EBADNAME = cares.ARES_EBADNAME +ARES_EBADFAMILY = cares.ARES_EBADFAMILY +ARES_EBADRESP = cares.ARES_EBADRESP +ARES_ECONNREFUSED = cares.ARES_ECONNREFUSED +ARES_ETIMEOUT = cares.ARES_ETIMEOUT +ARES_EOF = cares.ARES_EOF +ARES_EFILE = cares.ARES_EFILE +ARES_ENOMEM = cares.ARES_ENOMEM +ARES_EDESTRUCTION = cares.ARES_EDESTRUCTION +ARES_EBADSTR = cares.ARES_EBADSTR +ARES_EBADFLAGS = cares.ARES_EBADFLAGS +ARES_ENONAME = cares.ARES_ENONAME +ARES_EBADHINTS = cares.ARES_EBADHINTS +ARES_ENOTINITIALIZED = cares.ARES_ENOTINITIALIZED +ARES_ELOADIPHLPAPI = cares.ARES_ELOADIPHLPAPI +ARES_EADDRGETNETWORKPARAMS = cares.ARES_EADDRGETNETWORKPARAMS +ARES_ECANCELLED = cares.ARES_ECANCELLED + +ARES_FLAG_USEVC = cares.ARES_FLAG_USEVC +ARES_FLAG_PRIMARY = cares.ARES_FLAG_PRIMARY +ARES_FLAG_IGNTC = cares.ARES_FLAG_IGNTC +ARES_FLAG_NORECURSE = cares.ARES_FLAG_NORECURSE +ARES_FLAG_STAYOPEN = cares.ARES_FLAG_STAYOPEN +ARES_FLAG_NOSEARCH = cares.ARES_FLAG_NOSEARCH +ARES_FLAG_NOALIASES = cares.ARES_FLAG_NOALIASES +ARES_FLAG_NOCHECKRESP = cares.ARES_FLAG_NOCHECKRESP + + +_ares_errors = dict([ + (cares.ARES_SUCCESS, 'ARES_SUCCESS'), + (cares.ARES_ENODATA, 'ARES_ENODATA'), + (cares.ARES_EFORMERR, 'ARES_EFORMERR'), + (cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'), + (cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'), + (cares.ARES_ENOTIMP, 'ARES_ENOTIMP'), + (cares.ARES_EREFUSED, 'ARES_EREFUSED'), + (cares.ARES_EBADQUERY, 'ARES_EBADQUERY'), + (cares.ARES_EBADNAME, 'ARES_EBADNAME'), + (cares.ARES_EBADFAMILY, 'ARES_EBADFAMILY'), + (cares.ARES_EBADRESP, 'ARES_EBADRESP'), + (cares.ARES_ECONNREFUSED, 'ARES_ECONNREFUSED'), + (cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'), + (cares.ARES_EOF, 'ARES_EOF'), + (cares.ARES_EFILE, 'ARES_EFILE'), + (cares.ARES_ENOMEM, 'ARES_ENOMEM'), + (cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'), + (cares.ARES_EBADSTR, 'ARES_EBADSTR'), + (cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'), + (cares.ARES_ENONAME, 'ARES_ENONAME'), + (cares.ARES_EBADHINTS, 'ARES_EBADHINTS'), + (cares.ARES_ENOTINITIALIZED, 'ARES_ENOTINITIALIZED'), + (cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'), + (cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'), + (cares.ARES_ECANCELLED, 'ARES_ECANCELLED')]) + + +# maps c-ares flag to _socket module flag +_cares_flag_map = None + + +cdef _prepare_cares_flag_map(): + global _cares_flag_map + import _socket + _cares_flag_map = [ + (getattr(_socket, 'NI_NUMERICHOST', 1), cares.ARES_NI_NUMERICHOST), + (getattr(_socket, 'NI_NUMERICSERV', 2), cares.ARES_NI_NUMERICSERV), + (getattr(_socket, 'NI_NOFQDN', 4), cares.ARES_NI_NOFQDN), + (getattr(_socket, 'NI_NAMEREQD', 8), cares.ARES_NI_NAMEREQD), + (getattr(_socket, 'NI_DGRAM', 16), cares.ARES_NI_DGRAM)] + + +cpdef _convert_cares_flags(int flags, int default=cares.ARES_NI_LOOKUPHOST|cares.ARES_NI_LOOKUPSERVICE): + if _cares_flag_map is None: + _prepare_cares_flag_map() + for socket_flag, cares_flag in _cares_flag_map: + if socket_flag & flags: + default |= cares_flag + flags &= ~socket_flag + if not flags: + return default + raise gaierror(-1, "Bad value for ai_flags: 0x%x" % flags) + + +cpdef strerror(code): + return '%s: %s' % (_ares_errors.get(code) or code, cares.ares_strerror(code)) + + +class InvalidIP(ValueError): + pass + + +cdef void gevent_sock_state_callback(void *data, int s, int read, int write): + if not data: + return + cdef channel ch = data + ch._sock_state_callback(s, read, write) + + +cdef class result: + cdef public object value + cdef public object exception + + def __init__(self, object value=None, object exception=None): + self.value = value + self.exception = exception + + def __repr__(self): + if self.exception is None: + return '%s(%r)' % (self.__class__.__name__, self.value) + elif self.value is None: + return '%s(exception=%r)' % (self.__class__.__name__, self.exception) + else: + return '%s(value=%r, exception=%r)' % (self.__class__.__name__, self.value, self.exception) + # add repr_recursive precaution + + def successful(self): + return self.exception is None + + def get(self): + if self.exception is not None: + raise self.exception + return self.value + + +class ares_host_result(tuple): + + def __new__(cls, family, iterable): + cdef object self = tuple.__new__(cls, iterable) + self.family = family + return self + + def __getnewargs__(self): + return (self.family, tuple(self)) + + +cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent* host): + cdef channel channel + cdef object callback + channel, callback = arg + Py_DECREF(arg) + cdef object host_result + try: + if status or not host: + callback(result(None, gaierror(status, strerror(status)))) + else: + try: + host_result = ares_host_result(host.h_addrtype, (parse_h_name(host), parse_h_aliases(host), parse_h_addr_list(host))) + except: + callback(result(None, sys.exc_info()[1])) + else: + callback(result(host_result)) + except: + channel.loop.handle_error(callback, *sys.exc_info()) + + +cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, char *c_node, char *c_service): + cdef channel channel + cdef object callback + channel, callback = arg + Py_DECREF(arg) + cdef object node + cdef object service + try: + if status: + callback(result(None, gaierror(status, strerror(status)))) + else: + if c_node: + node = PyUnicode_FromString(c_node) + else: + node = None + if c_service: + service = PyUnicode_FromString(c_service) + else: + service = None + callback(result((node, service))) + except: + channel.loop.handle_error(callback, *sys.exc_info()) + + +cdef class channel: + + cdef public object loop + cdef ares_channeldata* channel + cdef public dict _watchers + cdef public object _timer + + def __init__(self, object loop, flags=None, timeout=None, tries=None, ndots=None, + udp_port=None, tcp_port=None, servers=None): + cdef ares_channeldata* channel = NULL + cdef cares.ares_options options + memset(&options, 0, sizeof(cares.ares_options)) + cdef int optmask = cares.ARES_OPT_SOCK_STATE_CB + options.sock_state_cb = gevent_sock_state_callback + options.sock_state_cb_data = self + if flags is not None: + options.flags = int(flags) + optmask |= cares.ARES_OPT_FLAGS + if timeout is not None: + options.timeout = int(float(timeout) * 1000) + optmask |= cares.ARES_OPT_TIMEOUTMS + if tries is not None: + options.tries = int(tries) + optmask |= cares.ARES_OPT_TRIES + if ndots is not None: + options.ndots = int(ndots) + optmask |= cares.ARES_OPT_NDOTS + if udp_port is not None: + options.udp_port = int(udp_port) + optmask |= cares.ARES_OPT_UDP_PORT + if tcp_port is not None: + options.tcp_port = int(tcp_port) + optmask |= cares.ARES_OPT_TCP_PORT + cdef int result = cares.ares_library_init(cares.ARES_LIB_INIT_ALL) # ARES_LIB_INIT_WIN32 -DUSE_WINSOCK? + if result: + raise gaierror(result, strerror(result)) + result = cares.ares_init_options(&channel, &options, optmask) + if result: + raise gaierror(result, strerror(result)) + self._timer = loop.timer(TIMEOUT, TIMEOUT) + self._watchers = {} + self.channel = channel + try: + if servers is not None: + self.set_servers(servers) + self.loop = loop + except: + self.destroy() + raise + + def __repr__(self): + args = (self.__class__.__name__, id(self), self._timer, len(self._watchers)) + return '<%s at 0x%x _timer=%r _watchers[%s]>' % args + + def destroy(self): + if self.channel: + # XXX ares_library_cleanup? + cares.ares_destroy(self.channel) + self.channel = NULL + self._watchers.clear() + self._timer.stop() + self.loop = None + + def __dealloc__(self): + if self.channel: + # XXX ares_library_cleanup? + cares.ares_destroy(self.channel) + self.channel = NULL + + cpdef set_servers(self, servers=None): + if not self.channel: + raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') + if not servers: + servers = [] + if isinstance(servers, string_types): + servers = servers.split(',') + cdef int length = len(servers) + cdef int result, index + cdef char* string + cdef cares.ares_addr_node* c_servers + if length <= 0: + result = cares.ares_set_servers(self.channel, NULL) + else: + c_servers = PyMem_Malloc(sizeof(cares.ares_addr_node) * length) + if not c_servers: + raise MemoryError + try: + index = 0 + for server in servers: + if isinstance(server, unicode): + server = server.encode('ascii') + string = server + if cares.ares_inet_pton(AF_INET, string, &c_servers[index].addr) > 0: + c_servers[index].family = AF_INET + elif cares.ares_inet_pton(AF_INET6, string, &c_servers[index].addr) > 0: + c_servers[index].family = AF_INET6 + else: + raise InvalidIP(repr(string)) + c_servers[index].next = &c_servers[index] + 1 + index += 1 + if index >= length: + break + c_servers[length - 1].next = NULL + index = cares.ares_set_servers(self.channel, c_servers) + if index: + raise ValueError(strerror(index)) + finally: + PyMem_Free(c_servers) + + # this crashes c-ares + #def cancel(self): + # cares.ares_cancel(self.channel) + + cdef _sock_state_callback(self, int socket, int read, int write): + if not self.channel: + return + cdef object watcher = self._watchers.get(socket) + cdef int events = 0 + if read: + events |= EV_READ + if write: + events |= EV_WRITE + if watcher is None: + if not events: + return + watcher = self.loop.io(socket, events) + self._watchers[socket] = watcher + elif events: + if watcher.events == events: + return + watcher.stop() + watcher.events = events + else: + watcher.stop() + watcher.close() + self._watchers.pop(socket, None) + if not self._watchers: + self._timer.stop() + return + watcher.start(self._process_fd, watcher, pass_events=True) + self._timer.again(self._on_timer) + + def _on_timer(self): + cares.ares_process_fd(self.channel, cares.ARES_SOCKET_BAD, cares.ARES_SOCKET_BAD) + + def _process_fd(self, int events, object watcher): + if not self.channel: + return + cdef int read_fd = watcher.fd + cdef int write_fd = read_fd + if not (events & EV_READ): + read_fd = cares.ARES_SOCKET_BAD + if not (events & EV_WRITE): + write_fd = cares.ARES_SOCKET_BAD + cares.ares_process_fd(self.channel, read_fd, write_fd) + + def gethostbyname(self, object callback, char* name, int family=AF_INET): + if not self.channel: + raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') + # note that for file lookups still AF_INET can be returned for AF_INET6 request + cdef object arg = (self, callback) + Py_INCREF(arg) + cares.ares_gethostbyname(self.channel, name, family, gevent_ares_host_callback, arg) + + def gethostbyaddr(self, object callback, char* addr): + if not self.channel: + raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') + # will guess the family + cdef char addr_packed[16] + cdef int family + cdef int length + if cares.ares_inet_pton(AF_INET, addr, addr_packed) > 0: + family = AF_INET + length = 4 + elif cares.ares_inet_pton(AF_INET6, addr, addr_packed) > 0: + family = AF_INET6 + length = 16 + else: + raise InvalidIP(repr(addr)) + cdef object arg = (self, callback) + Py_INCREF(arg) + cares.ares_gethostbyaddr(self.channel, addr_packed, length, family, gevent_ares_host_callback, arg) + + cpdef _getnameinfo(self, object callback, tuple sockaddr, int flags): + if not self.channel: + raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') + cdef char* hostp = NULL + cdef int port = 0 + cdef int flowinfo = 0 + cdef int scope_id = 0 + cdef sockaddr_in6 sa6 + if not PyTuple_Check(sockaddr): + raise TypeError('expected a tuple, got %r' % (sockaddr, )) + PyArg_ParseTuple(sockaddr, "si|ii", &hostp, &port, &flowinfo, &scope_id) + if port < 0 or port > 65535: + raise gaierror(-8, 'Invalid value for port: %r' % port) + cdef int length = gevent_make_sockaddr(hostp, port, flowinfo, scope_id, &sa6) + if length <= 0: + raise InvalidIP(repr(hostp)) + cdef object arg = (self, callback) + Py_INCREF(arg) + cdef sockaddr_t* x = &sa6 + cares.ares_getnameinfo(self.channel, x, length, flags, gevent_ares_nameinfo_callback, arg) + + def getnameinfo(self, object callback, tuple sockaddr, int flags): + try: + flags = _convert_cares_flags(flags) + except gaierror: + # The stdlib just ignores bad flags + flags = 0 + return self._getnameinfo(callback, sockaddr, flags) diff --git a/src/gevent/resolver/cares_ntop.h b/src/gevent/resolver/cares_ntop.h new file mode 100644 index 0000000..9ffc9dd --- /dev/null +++ b/src/gevent/resolver/cares_ntop.h @@ -0,0 +1,7 @@ +#ifdef CARES_EMBED +#include "ares_setup.h" +#include "ares.h" +#else +#include +#define ares_inet_ntop(w,x,y,z) inet_ntop(w,x,y,z) +#endif diff --git a/src/gevent/resolver/cares_pton.h b/src/gevent/resolver/cares_pton.h new file mode 100644 index 0000000..85af403 --- /dev/null +++ b/src/gevent/resolver/cares_pton.h @@ -0,0 +1,8 @@ +#ifdef CARES_EMBED +#include "ares_setup.h" +#include "ares_inet_net_pton.h" +#else +#include +#define ares_inet_pton(x,y,z) inet_pton(x,y,z) +#define ares_inet_net_pton(w,x,y,z) inet_net_pton(w,x,y,z) +#endif diff --git a/src/gevent/resolver/dnshelper.c b/src/gevent/resolver/dnshelper.c new file mode 100644 index 0000000..3befb69 --- /dev/null +++ b/src/gevent/resolver/dnshelper.c @@ -0,0 +1,159 @@ +/* Copyright (c) 2011 Denis Bilenko. See LICENSE for details. */ +#include "Python.h" +#ifdef CARES_EMBED +#include "ares_setup.h" +#endif + +#ifdef HAVE_NETDB_H +#include +#endif + +#include "ares.h" + +#include "cares_ntop.h" +#include "cares_pton.h" + +#if PY_VERSION_HEX < 0x02060000 + #define PyUnicode_FromString PyString_FromString +#elif PY_MAJOR_VERSION < 3 + #define PyUnicode_FromString PyBytes_FromString +#endif + + +static PyObject* _socket_error = 0; + +static PyObject* +get_socket_object(PyObject** pobject, const char* name) +{ + if (!*pobject) { + PyObject* _socket; + _socket = PyImport_ImportModule("_socket"); + if (_socket) { + *pobject = PyObject_GetAttrString(_socket, name); + if (!*pobject) { + PyErr_WriteUnraisable(Py_None); + } + Py_DECREF(_socket); + } + else { + PyErr_WriteUnraisable(Py_None); + } + if (!*pobject) { + *pobject = PyExc_IOError; + } + } + return *pobject; +} + + +static int +gevent_append_addr(PyObject* list, int family, void* src, char* tmpbuf, size_t tmpsize) { + int status = -1; + PyObject* tmp; + if (ares_inet_ntop(family, src, tmpbuf, tmpsize)) { + tmp = PyUnicode_FromString(tmpbuf); + if (tmp) { + status = PyList_Append(list, tmp); + Py_DECREF(tmp); + } + } + return status; +} + + +static PyObject* +parse_h_name(struct hostent *h) +{ + return PyUnicode_FromString(h->h_name); +} + + +static PyObject* +parse_h_aliases(struct hostent *h) +{ + char **pch; + PyObject *result = NULL; + PyObject *tmp; + + result = PyList_New(0); + + if (result && h->h_aliases) { + for (pch = h->h_aliases; *pch != NULL; pch++) { + if (*pch != h->h_name && strcmp(*pch, h->h_name)) { + int status; + tmp = PyUnicode_FromString(*pch); + if (tmp == NULL) { + break; + } + + status = PyList_Append(result, tmp); + Py_DECREF(tmp); + + if (status) { + break; + } + } + } + } + + return result; +} + + +static PyObject * +parse_h_addr_list(struct hostent *h) +{ + char **pch; + PyObject *result = NULL; + + result = PyList_New(0); + + if (result) { + switch (h->h_addrtype) { + case AF_INET: + { + char tmpbuf[sizeof "255.255.255.255"]; + for (pch = h->h_addr_list; *pch != NULL; pch++) { + if (gevent_append_addr(result, AF_INET, *pch, tmpbuf, sizeof(tmpbuf))) { + break; + } + } + break; + } + case AF_INET6: + { + char tmpbuf[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")]; + for (pch = h->h_addr_list; *pch != NULL; pch++) { + if (gevent_append_addr(result, AF_INET6, *pch, tmpbuf, sizeof(tmpbuf))) { + break; + } + } + break; + } + default: + PyErr_SetString(get_socket_object(&_socket_error, "error"), "unsupported address family"); + Py_DECREF(result); + result = NULL; + } + } + + return result; +} + + +static int +gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, struct sockaddr_in6* sa6) { + if ( ares_inet_pton(AF_INET, hostp, &((struct sockaddr_in*)sa6)->sin_addr.s_addr) > 0 ) { + ((struct sockaddr_in*)sa6)->sin_family = AF_INET; + ((struct sockaddr_in*)sa6)->sin_port = htons(port); + return sizeof(struct sockaddr_in); + } + else if ( ares_inet_pton(AF_INET6, hostp, &sa6->sin6_addr.s6_addr) > 0 ) { + sa6->sin6_family = AF_INET6; + sa6->sin6_port = htons(port); + sa6->sin6_flowinfo = flowinfo; + sa6->sin6_scope_id = scope_id; + return sizeof(struct sockaddr_in6); + } + return -1; +} diff --git a/src/gevent/resolver/dnspython.py b/src/gevent/resolver/dnspython.py new file mode 100644 index 0000000..0b9e82f --- /dev/null +++ b/src/gevent/resolver/dnspython.py @@ -0,0 +1,662 @@ +# Copyright (c) 2018 gevent contributors. See LICENSE for details. + +# Portions of this code taken from the gogreen project: +# http://github.com/slideinc/gogreen +# +# Copyright (c) 2005-2010 Slide, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of the author nor the names of other +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Portions of this code taken from the eventlet project: +# https://github.com/eventlet/eventlet/blob/master/eventlet/support/greendns.py + +# Unless otherwise noted, the files in Eventlet are under the following MIT license: + +# Copyright (c) 2005-2006, Bob Ippolito +# Copyright (c) 2007-2010, Linden Research, Inc. +# Copyright (c) 2008-2010, Eventlet Contributors (see AUTHORS) + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +import time +import re +import os +import sys + +import _socket +from _socket import AI_NUMERICHOST +from _socket import error +from _socket import NI_NUMERICSERV +from _socket import AF_INET +from _socket import AF_INET6 +from _socket import AF_UNSPEC + +import socket + +from gevent.resolver import AbstractResolver +from gevent.resolver import hostname_types + +from gevent._compat import string_types +from gevent._compat import iteritems +from gevent._patcher import import_patched +from gevent._config import config + +__all__ = [ + 'Resolver', +] + +# Import the DNS packages to use the gevent modules, +# even if the system is not monkey-patched. +def _patch_dns(): + top = import_patched('dns') + for pkg in ('dns', + 'dns.rdtypes', + 'dns.rdtypes.IN', + 'dns.rdtypes.ANY'): + mod = import_patched(pkg) + for name in mod.__all__: + setattr(mod, name, import_patched(pkg + '.' + name)) + return top + +dns = _patch_dns() + +def _dns_import_patched(name): + assert name.startswith('dns') + import_patched(name) + return dns + +# This module tries to dynamically import classes +# using __import__, and it's important that they match +# the ones we just created, otherwise exceptions won't be caught +# as expected. It uses a one-arg __import__ statement and then +# tries to walk down the sub-modules using getattr, so we can't +# directly use import_patched as-is. +dns.rdata.__import__ = _dns_import_patched + +resolver = dns.resolver +dTimeout = dns.resolver.Timeout + +_exc_clear = getattr(sys, 'exc_clear', lambda: None) + +# This is a copy of resolver._getaddrinfo with the crucial change that it +# doesn't have a bare except:, because that breaks Timeout and KeyboardInterrupt +# A secondary change is that calls to sys.exc_clear() have been inserted to avoid +# failing tests in test__refcount.py (timeouts). +# See https://github.com/rthalley/dnspython/pull/300 +def _getaddrinfo(host=None, service=None, family=AF_UNSPEC, socktype=0, + proto=0, flags=0): + # pylint:disable=too-many-locals,broad-except,too-many-statements + # pylint:disable=too-many-branches + # pylint:disable=redefined-argument-from-local + # pylint:disable=consider-using-in + if flags & (socket.AI_ADDRCONFIG | socket.AI_V4MAPPED) != 0: + raise NotImplementedError + if host is None and service is None: + raise socket.gaierror(socket.EAI_NONAME) + v6addrs = [] + v4addrs = [] + canonical_name = None + try: + # Is host None or a V6 address literal? + if host is None: + canonical_name = 'localhost' + if flags & socket.AI_PASSIVE != 0: + v6addrs.append('::') + v4addrs.append('0.0.0.0') + else: + v6addrs.append('::1') + v4addrs.append('127.0.0.1') + else: + parts = host.split('%') + if len(parts) == 2: + ahost = parts[0] + else: + ahost = host + addr = dns.ipv6.inet_aton(ahost) + v6addrs.append(host) + canonical_name = host + except Exception: + _exc_clear() + try: + # Is it a V4 address literal? + addr = dns.ipv4.inet_aton(host) + v4addrs.append(host) + canonical_name = host + except Exception: + _exc_clear() + if flags & socket.AI_NUMERICHOST == 0: + try: + if family == socket.AF_INET6 or family == socket.AF_UNSPEC: + v6 = resolver._resolver.query(host, dns.rdatatype.AAAA, + raise_on_no_answer=False) + # Note that setting host ensures we query the same name + # for A as we did for AAAA. + host = v6.qname + canonical_name = v6.canonical_name.to_text(True) + if v6.rrset is not None: + for rdata in v6.rrset: + v6addrs.append(rdata.address) + if family == socket.AF_INET or family == socket.AF_UNSPEC: + v4 = resolver._resolver.query(host, dns.rdatatype.A, + raise_on_no_answer=False) + host = v4.qname + canonical_name = v4.canonical_name.to_text(True) + if v4.rrset is not None: + for rdata in v4.rrset: + v4addrs.append(rdata.address) + except dns.resolver.NXDOMAIN: + _exc_clear() + raise socket.gaierror(socket.EAI_NONAME) + except Exception: + _exc_clear() + raise socket.gaierror(socket.EAI_SYSTEM) + port = None + try: + # Is it a port literal? + if service is None: + port = 0 + else: + port = int(service) + except Exception: + _exc_clear() + if flags & socket.AI_NUMERICSERV == 0: + try: + port = socket.getservbyname(service) + except Exception: + _exc_clear() + + if port is None: + raise socket.gaierror(socket.EAI_NONAME) + tuples = [] + if socktype == 0: + socktypes = [socket.SOCK_DGRAM, socket.SOCK_STREAM] + else: + socktypes = [socktype] + if flags & socket.AI_CANONNAME != 0: + cname = canonical_name + else: + cname = '' + if family == socket.AF_INET6 or family == socket.AF_UNSPEC: + for addr in v6addrs: + for socktype in socktypes: + for proto in resolver._protocols_for_socktype[socktype]: + tuples.append((socket.AF_INET6, socktype, proto, + cname, (addr, port, 0, 0))) # XXX: gevent: this can get the scopeid wrong + if family == socket.AF_INET or family == socket.AF_UNSPEC: + for addr in v4addrs: + for socktype in socktypes: + for proto in resolver._protocols_for_socktype[socktype]: + tuples.append((socket.AF_INET, socktype, proto, + cname, (addr, port))) + if len(tuples) == 0: # pylint:disable=len-as-condition + raise socket.gaierror(socket.EAI_NONAME) + return tuples + + +resolver._getaddrinfo = _getaddrinfo + +HOSTS_TTL = 300.0 + +def _is_addr(host, parse=dns.ipv4.inet_aton): + if not host: + return False + assert isinstance(host, hostname_types), repr(host) + try: + parse(host) + except dns.exception.SyntaxError: + return False + else: + return True + +# Return True if host is a valid IPv4 address +_is_ipv4_addr = _is_addr + + +def _is_ipv6_addr(host): + # Return True if host is a valid IPv6 address + if host: + s = '%' if isinstance(host, str) else b'%' + host = host.split(s, 1)[0] + return _is_addr(host, dns.ipv6.inet_aton) + +class HostsFile(object): + """ + A class to read the contents of a hosts file (/etc/hosts). + """ + + LINES_RE = re.compile(r""" + \s* # Leading space + ([^\r\n#]+?) # The actual match, non-greedy so as not to include trailing space + \s* # Trailing space + (?:[#][^\r\n]+)? # Comments + (?:$|[\r\n]+) # EOF or newline + """, re.VERBOSE) + + def __init__(self, fname=None): + self.v4 = {} # name -> ipv4 + self.v6 = {} # name -> ipv6 + self.aliases = {} # name -> canonical_name + self.reverse = {} # ip addr -> some name + if fname is None: + if os.name == 'posix': + fname = '/etc/hosts' + elif os.name == 'nt': # pragma: no cover + fname = os.path.expandvars( + r'%SystemRoot%\system32\drivers\etc\hosts') + self.fname = fname + assert self.fname + self._last_load = 0 + + + def _readlines(self): + # Read the contents of the hosts file. + # + # Return list of lines, comment lines and empty lines are + # excluded. Note that this performs disk I/O so can be + # blocking. + with open(self.fname, 'rb') as fp: + fdata = fp.read() + + + # XXX: Using default decoding. Is that correct? + udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata + + return self.LINES_RE.findall(udata) + + def load(self): # pylint:disable=too-many-locals + # Load hosts file + + # This will (re)load the data from the hosts + # file if it has changed. + + try: + load_time = os.stat(self.fname).st_mtime + needs_load = load_time > self._last_load + except (IOError, OSError): + from gevent import get_hub + get_hub().handle_error(self, *sys.exc_info()) + needs_load = False + + if not needs_load: + return + + v4 = {} + v6 = {} + aliases = {} + reverse = {} + + for line in self._readlines(): + parts = line.split() + if len(parts) < 2: + continue + ip = parts.pop(0) + if _is_ipv4_addr(ip): + ipmap = v4 + elif _is_ipv6_addr(ip): + if ip.startswith('fe80'): + # Do not use link-local addresses, OSX stores these here + continue + ipmap = v6 + else: + continue + cname = parts.pop(0).lower() + ipmap[cname] = ip + for alias in parts: + alias = alias.lower() + ipmap[alias] = ip + aliases[alias] = cname + + # XXX: This is wrong for ipv6 + if ipmap is v4: + ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa' + else: + ptr = ip + '.ip6.arpa.' + if ptr not in reverse: + reverse[ptr] = cname + + self._last_load = load_time + self.v4 = v4 + self.v6 = v6 + self.aliases = aliases + self.reverse = reverse + + def iter_all_host_addr_pairs(self): + self.load() + for name, addr in iteritems(self.v4): + yield name, addr + for name, addr in iteritems(self.v6): + yield name, addr + +class _HostsAnswer(dns.resolver.Answer): + # Answer class for HostsResolver object + + def __init__(self, qname, rdtype, rdclass, rrset, raise_on_no_answer=True): + self.response = None + self.qname = qname + self.rdtype = rdtype + self.rdclass = rdclass + self.canonical_name = qname + if not rrset and raise_on_no_answer: + raise dns.resolver.NoAnswer() + self.rrset = rrset + self.expiration = (time.time() + + rrset.ttl if hasattr(rrset, 'ttl') else 0) + + +class _HostsResolver(object): + """ + Class to parse the hosts file + """ + + def __init__(self, fname=None, interval=HOSTS_TTL): + self.hosts_file = HostsFile(fname) + self.interval = interval + self._last_load = 0 + + def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN, + tcp=False, source=None, raise_on_no_answer=True): # pylint:disable=unused-argument + # Query the hosts file + # + # The known rdtypes are dns.rdatatype.A, dns.rdatatype.AAAA and + # dns.rdatatype.CNAME. + # The ``rdclass`` parameter must be dns.rdataclass.IN while the + # ``tcp`` and ``source`` parameters are ignored. + # Return a HostAnswer instance or raise a dns.resolver.NoAnswer + # exception. + + now = time.time() + hosts_file = self.hosts_file + if self._last_load + self.interval < now: + self._last_load = now + hosts_file.load() + + rdclass = dns.rdataclass.IN # Always + if isinstance(qname, string_types): + name = qname + qname = dns.name.from_text(qname) + else: + name = str(qname) + + name = name.lower() + rrset = dns.rrset.RRset(qname, rdclass, rdtype) + rrset.ttl = self._last_load + self.interval - now + + if rdtype == dns.rdatatype.A: + mapping = hosts_file.v4 + kind = dns.rdtypes.IN.A.A + elif rdtype == dns.rdatatype.AAAA: + mapping = hosts_file.v6 + kind = dns.rdtypes.IN.AAAA.AAAA + elif rdtype == dns.rdatatype.CNAME: + mapping = hosts_file.aliases + kind = lambda c, t, addr: dns.rdtypes.ANY.CNAME.CNAME(c, t, dns.name.from_text(addr)) + elif rdtype == dns.rdatatype.PTR: + mapping = hosts_file.reverse + kind = lambda c, t, addr: dns.rdtypes.ANY.PTR.PTR(c, t, dns.name.from_text(addr)) + + + addr = mapping.get(name) + if not addr and qname.is_absolute(): + addr = mapping.get(name[:-1]) + if addr: + rrset.add(kind(rdclass, rdtype, addr)) + return _HostsAnswer(qname, rdtype, rdclass, rrset, raise_on_no_answer) + + def getaliases(self, hostname): + # Return a list of all the aliases of a given cname + + # Due to the way store aliases this is a bit inefficient, this + # clearly was an afterthought. But this is only used by + # gethostbyname_ex so it's probably fine. + aliases = self.hosts_file.aliases + result = [] + if hostname in aliases: + cannon = aliases[hostname] + else: + cannon = hostname + result.append(cannon) + for alias, cname in iteritems(aliases): + if cannon == cname: + result.append(alias) + result.remove(hostname) + return result + +class _DualResolver(object): + + def __init__(self): + self.hosts_resolver = _HostsResolver() + self.network_resolver = resolver.get_default_resolver() + self.network_resolver.cache = resolver.LRUCache() + + def query(self, qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN, + tcp=False, source=None, raise_on_no_answer=True, + _hosts_rdtypes=(dns.rdatatype.A, dns.rdatatype.AAAA, dns.rdatatype.PTR)): + # Query the resolver, using /etc/hosts + + # Behavior: + # 1. if hosts is enabled and contains answer, return it now + # 2. query nameservers for qname + if qname is None: + qname = '0.0.0.0' + + if not isinstance(qname, string_types): + if isinstance(qname, bytes): + qname = qname.decode("idna") + + if isinstance(qname, string_types): + qname = dns.name.from_text(qname, None) + + if isinstance(rdtype, string_types): + rdtype = dns.rdatatype.from_text(rdtype) + + if rdclass == dns.rdataclass.IN and rdtype in _hosts_rdtypes: + try: + answer = self.hosts_resolver.query(qname, rdtype, raise_on_no_answer=False) + except Exception: # pylint: disable=broad-except + from gevent import get_hub + get_hub().handle_error(self, *sys.exc_info()) + else: + if answer.rrset: + return answer + + return self.network_resolver.query(qname, rdtype, rdclass, + tcp, source, raise_on_no_answer=raise_on_no_answer) + +def _family_to_rdtype(family): + if family == socket.AF_INET: + rdtype = dns.rdatatype.A + elif family == socket.AF_INET6: + rdtype = dns.rdatatype.AAAA + else: + raise socket.gaierror(socket.EAI_FAMILY, + 'Address family not supported') + return rdtype + +class Resolver(AbstractResolver): + """ + An *experimental* resolver that uses `dnspython`_. + + This is typically slower than the default threaded resolver + (unless there's a cache hit, in which case it can be much faster). + It is usually much faster than the c-ares resolver. It tends to + scale well as more concurrent resolutions are attempted. + + Under Python 2, if the ``idna`` package is installed, this + resolver can resolve Unicode host names that the system resolver + cannot. + + .. note:: + + This **does not** use dnspython's default resolver object, or share any + classes with ``import dns``. A separate copy of the objects is imported to + be able to function in a non monkey-patched process. The documentation for the resolver + object still applies. + + The resolver that we use is available as the :attr:`resolver` attribute + of this object (typically ``gevent.get_hub().resolver.resolver``). + + .. caution:: + + Many of the same caveats about DNS results apply here as are documented + for :class:`gevent.resolver.ares.Resolver`. + + .. caution:: + + This resolver is experimental. It may be removed or modified in + the future. As always, feedback is welcome. + + .. versionadded:: 1.3a2 + + .. _dnspython: http://www.dnspython.org + """ + + def __init__(self, hub=None): # pylint: disable=unused-argument + if resolver._resolver is None: + _resolver = resolver._resolver = _DualResolver() + if config.resolver_nameservers: + _resolver.network_resolver.nameservers[:] = config.resolver_nameservers + if config.resolver_timeout: + _resolver.network_resolver.lifetime = config.resolver_timeout + # Different hubs in different threads could be sharing the same + # resolver. + assert isinstance(resolver._resolver, _DualResolver) + self._resolver = resolver._resolver + + @property + def resolver(self): + """ + The dnspython resolver object we use. + + This object has several useful attributes that can be used to + adjust the behaviour of the DNS system: + + * ``cache`` is a :class:`dns.resolver.LRUCache`. Its maximum size + can be configured by calling :meth:`resolver.cache.set_max_size` + * ``nameservers`` controls which nameservers to talk to + * ``lifetime`` configures a timeout for each individual query. + """ + return self._resolver.network_resolver + + def close(self): + pass + + def _getaliases(self, hostname, family): + if not isinstance(hostname, str): + if isinstance(hostname, bytes): + hostname = hostname.decode("idna") + aliases = self._resolver.hosts_resolver.getaliases(hostname) + net_resolver = self._resolver.network_resolver + rdtype = _family_to_rdtype(family) + while True: + try: + ans = net_resolver.query(hostname, dns.rdatatype.CNAME, rdtype) + except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.resolver.NoNameservers): + break + except dTimeout: + break + else: + aliases.extend(str(rr.target) for rr in ans.rrset) + hostname = ans[0].target + return aliases + + def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): + if ((host in (u'localhost', b'localhost') + or (_is_ipv6_addr(host) and host.startswith('fe80'))) + or not isinstance(host, str) or (flags & AI_NUMERICHOST)): + # this handles cases which do not require network access + # 1) host is None + # 2) host is of an invalid type + # 3) host is localhost or a link-local ipv6; dnspython returns the wrong + # scope-id for those. + # 3) AI_NUMERICHOST flag is set + + return _socket.getaddrinfo(host, port, family, socktype, proto, flags) + + if family == AF_UNSPEC: + # This tends to raise in the case that a v6 address did not exist + # but a v4 does. So we break it into two parts. + + # Note that if there is no ipv6 in the hosts file, but there *is* + # an ipv4, and there *is* an ipv6 in the nameservers, we will return + # both (from the first call). The system resolver on OS X only returns + # the results from the hosts file. doubleclick.com is one example. + + # See also https://github.com/gevent/gevent/issues/1012 + try: + return _getaddrinfo(host, port, family, socktype, proto, flags) + except socket.gaierror: + try: + return _getaddrinfo(host, port, AF_INET6, socktype, proto, flags) + except socket.gaierror: + return _getaddrinfo(host, port, AF_INET, socktype, proto, flags) + else: + return _getaddrinfo(host, port, family, socktype, proto, flags) + + def getnameinfo(self, sockaddr, flags): + if (sockaddr + and isinstance(sockaddr, (list, tuple)) + and sockaddr[0] in ('::1', '127.0.0.1', 'localhost')): + return _socket.getnameinfo(sockaddr, flags) + if isinstance(sockaddr, (list, tuple)) and not isinstance(sockaddr[0], hostname_types): + raise TypeError("getnameinfo(): illegal sockaddr argument") + try: + return resolver._getnameinfo(sockaddr, flags) + except error: + if not flags: + # dnspython doesn't like getting ports it can't resolve. + # We have one test, test__socket_dns.py:Test_getnameinfo_geventorg.test_port_zero + # that does this. We conservatively fix it here; this could be expanded later. + return resolver._getnameinfo(sockaddr, NI_NUMERICSERV) + + def gethostbyaddr(self, ip_address): + if ip_address in (u'127.0.0.1', u'::1', + b'127.0.0.1', b'::1', + 'localhost'): + return _socket.gethostbyaddr(ip_address) + + if not isinstance(ip_address, hostname_types): + raise TypeError("argument 1 must be str, bytes or bytearray, not %s" % (type(ip_address),)) + + return resolver._gethostbyaddr(ip_address) diff --git a/src/gevent/resolver/libcares.pxd b/src/gevent/resolver/libcares.pxd new file mode 100644 index 0000000..7b551a7 --- /dev/null +++ b/src/gevent/resolver/libcares.pxd @@ -0,0 +1,109 @@ +cdef extern from "ares.h": + struct ares_options: + int flags + void* sock_state_cb + void* sock_state_cb_data + int timeout + int tries + int ndots + unsigned short udp_port + unsigned short tcp_port + char **domains + int ndomains + char* lookups + + int ARES_OPT_FLAGS + int ARES_OPT_SOCK_STATE_CB + int ARES_OPT_TIMEOUTMS + int ARES_OPT_TRIES + int ARES_OPT_NDOTS + int ARES_OPT_TCP_PORT + int ARES_OPT_UDP_PORT + int ARES_OPT_SERVERS + int ARES_OPT_DOMAINS + int ARES_OPT_LOOKUPS + + int ARES_FLAG_USEVC + int ARES_FLAG_PRIMARY + int ARES_FLAG_IGNTC + int ARES_FLAG_NORECURSE + int ARES_FLAG_STAYOPEN + int ARES_FLAG_NOSEARCH + int ARES_FLAG_NOALIASES + int ARES_FLAG_NOCHECKRESP + + int ARES_LIB_INIT_ALL + int ARES_SOCKET_BAD + + int ARES_SUCCESS + int ARES_ENODATA + int ARES_EFORMERR + int ARES_ESERVFAIL + int ARES_ENOTFOUND + int ARES_ENOTIMP + int ARES_EREFUSED + int ARES_EBADQUERY + int ARES_EBADNAME + int ARES_EBADFAMILY + int ARES_EBADRESP + int ARES_ECONNREFUSED + int ARES_ETIMEOUT + int ARES_EOF + int ARES_EFILE + int ARES_ENOMEM + int ARES_EDESTRUCTION + int ARES_EBADSTR + int ARES_EBADFLAGS + int ARES_ENONAME + int ARES_EBADHINTS + int ARES_ENOTINITIALIZED + int ARES_ELOADIPHLPAPI + int ARES_EADDRGETNETWORKPARAMS + int ARES_ECANCELLED + + int ARES_NI_NOFQDN + int ARES_NI_NUMERICHOST + int ARES_NI_NAMEREQD + int ARES_NI_NUMERICSERV + int ARES_NI_DGRAM + int ARES_NI_TCP + int ARES_NI_UDP + int ARES_NI_SCTP + int ARES_NI_DCCP + int ARES_NI_NUMERICSCOPE + int ARES_NI_LOOKUPHOST + int ARES_NI_LOOKUPSERVICE + + + int ares_library_init(int flags) + void ares_library_cleanup() + int ares_init_options(void *channelptr, ares_options *options, int) + int ares_init(void *channelptr) + void ares_destroy(void *channelptr) + void ares_gethostbyname(void* channel, char *name, int family, void* callback, void *arg) + void ares_gethostbyaddr(void* channel, void *addr, int addrlen, int family, void* callback, void *arg) + void ares_process_fd(void* channel, int read_fd, int write_fd) + char* ares_strerror(int code) + void ares_cancel(void* channel) + void ares_getnameinfo(void* channel, void* sa, int salen, int flags, void* callback, void *arg) + + struct in_addr: + pass + + struct ares_in6_addr: + pass + + struct addr_union: + in_addr addr4 + ares_in6_addr addr6 + + struct ares_addr_node: + ares_addr_node *next + int family + addr_union addr + + int ares_set_servers(void* channel, ares_addr_node *servers) + + +cdef extern from "cares_pton.h": + int ares_inet_pton(int af, char *src, void *dst) diff --git a/src/gevent/resolver/thread.py b/src/gevent/resolver/thread.py new file mode 100644 index 0000000..df71706 --- /dev/null +++ b/src/gevent/resolver/thread.py @@ -0,0 +1,71 @@ +# Copyright (c) 2012-2015 Denis Bilenko. See LICENSE for details. +""" +Native thread-based hostname resolver. +""" +import _socket + +from gevent.hub import get_hub + + +__all__ = ['Resolver'] + + +# trigger import of encodings.idna to avoid https://github.com/gevent/gevent/issues/349 +u'foo'.encode('idna') + + +class Resolver(object): + """ + Implementation of the resolver API using native threads and native resolution + functions. + + Using the native resolution mechanisms ensures the highest + compatibility with what a non-gevent program would return + including good support for platform specific configuration + mechanisms. The use of native (non-greenlet) threads ensures that + a caller doesn't block other greenlets. + + This implementation also has the benefit of being very simple in comparison to + :class:`gevent.resolver_ares.Resolver`. + + .. tip:: + + Most users find this resolver to be quite reliable in a + properly monkey-patched environment. However, there have been + some reports of long delays, slow performance or even hangs, + particularly in long-lived programs that make many, many DNS + requests. If you suspect that may be happening to you, try the + dnspython or ares resolver (and submit a bug report). + """ + def __init__(self, hub=None): + if hub is None: + hub = get_hub() + self.pool = hub.threadpool + if _socket.gaierror not in hub.NOT_ERROR: + # Do not cause lookup failures to get printed by the default + # error handler. This can be very noisy. + hub.NOT_ERROR += (_socket.gaierror, _socket.herror) + + def __repr__(self): + return '' % (id(self), self.pool) + + def close(self): + pass + + # from briefly reading socketmodule.c, it seems that all of the functions + # below are thread-safe in Python, even if they are not thread-safe in C. + + def gethostbyname(self, *args): + return self.pool.apply(_socket.gethostbyname, args) + + def gethostbyname_ex(self, *args): + return self.pool.apply(_socket.gethostbyname_ex, args) + + def getaddrinfo(self, *args, **kwargs): + return self.pool.apply(_socket.getaddrinfo, args, kwargs) + + def gethostbyaddr(self, *args, **kwargs): + return self.pool.apply(_socket.gethostbyaddr, args, kwargs) + + def getnameinfo(self, *args, **kwargs): + return self.pool.apply(_socket.getnameinfo, args, kwargs) diff --git a/src/gevent/resolver_ares.py b/src/gevent/resolver_ares.py new file mode 100644 index 0000000..9f0c449 --- /dev/null +++ b/src/gevent/resolver_ares.py @@ -0,0 +1,17 @@ +"""Backwards compatibility alias for :mod:`gevent.resolver.ares`. + +.. deprecated:: 1.3 + Use :mod:`gevent.resolver.ares` +""" +import warnings +warnings.warn( + "gevent.resolver_ares is deprecated and will be removed in 1.5. " + "Use gevent.resolver.ares instead.", + DeprecationWarning, + stacklevel=2 +) +del warnings +from gevent.resolver.ares import * # pylint:disable=wildcard-import,unused-wildcard-import +import gevent.resolver.ares as _ares +__all__ = _ares.__all__ +del _ares diff --git a/src/gevent/resolver_thread.py b/src/gevent/resolver_thread.py new file mode 100644 index 0000000..1486e42 --- /dev/null +++ b/src/gevent/resolver_thread.py @@ -0,0 +1,17 @@ +"""Backwards compatibility alias for :mod:`gevent.resolver.thread`. + +.. deprecated:: 1.3 + Use :mod:`gevent.resolver.thread` +""" +import warnings +warnings.warn( + "gevent.resolver_thread is deprecated and will be removed in 1.5. " + "Use gevent.resolver.thread instead.", + DeprecationWarning, + stacklevel=2 +) +del warnings +from gevent.resolver.thread import * # pylint:disable=wildcard-import,unused-wildcard-import +import gevent.resolver.thread as _thread +__all__ = _thread.__all__ +del _thread diff --git a/src/gevent/select.py b/src/gevent/select.py new file mode 100644 index 0000000..a4c182c --- /dev/null +++ b/src/gevent/select.py @@ -0,0 +1,286 @@ +# Copyright (c) 2009-2011 Denis Bilenko. See LICENSE for details. +""" +Waiting for I/O completion. +""" +from __future__ import absolute_import, division, print_function + +import sys + +from gevent.event import Event +from gevent.hub import _get_hub_noargs as get_hub +from gevent.hub import sleep as _g_sleep +from gevent._compat import integer_types +from gevent._compat import iteritems +from gevent._util import copy_globals +from gevent._util import _NONE + +from errno import EINTR +from select import select as _real_original_select +if sys.platform.startswith('win32'): + def _original_select(r, w, x, t): + # windows cant handle three empty lists, but we've always + # accepted that + if not r and not w and not x: + return ((), (), ()) + return _real_original_select(r, w, x, t) +else: + _original_select = _real_original_select + + +try: + from select import poll as original_poll + from select import POLLIN, POLLOUT, POLLNVAL + __implements__ = ['select', 'poll'] +except ImportError: + original_poll = None + __implements__ = ['select'] + +__all__ = ['error'] + __implements__ + +import select as __select__ + +error = __select__.error + +__imports__ = copy_globals(__select__, globals(), + names_to_ignore=__all__, + dunder_names_to_keep=()) + +_EV_READ = 1 +_EV_WRITE = 2 + +def get_fileno(obj): + try: + fileno_f = obj.fileno + except AttributeError: + if not isinstance(obj, integer_types): + raise TypeError('argument must be an int, or have a fileno() method: %r' % (obj,)) + return obj + else: + return fileno_f() + + +class SelectResult(object): + __slots__ = ('read', 'write', 'event') + + def __init__(self): + self.read = [] + self.write = [] + self.event = Event() + + def add_read(self, socket): + self.read.append(socket) + self.event.set() + + add_read.event = _EV_READ + + def add_write(self, socket): + self.write.append(socket) + self.event.set() + + add_write.event = _EV_WRITE + + def __add_watchers(self, watchers, fdlist, callback, io, pri): + for fd in fdlist: + watcher = io(get_fileno(fd), callback.event) + watcher.priority = pri + watchers.append(watcher) + watcher.start(callback, fd) + + def _make_watchers(self, watchers, rlist, wlist): + loop = get_hub().loop + io = loop.io + MAXPRI = loop.MAXPRI + + try: + self.__add_watchers(watchers, rlist, self.add_read, io, MAXPRI) + self.__add_watchers(watchers, wlist, self.add_write, io, MAXPRI) + except IOError as ex: + raise error(*ex.args) + + def _closeall(self, watchers): + for watcher in watchers: + watcher.stop() + watcher.close() + del watchers[:] + + def select(self, rlist, wlist, timeout): + watchers = [] + try: + self._make_watchers(watchers, rlist, wlist) + self.event.wait(timeout=timeout) + return self.read, self.write, [] + finally: + self._closeall(watchers) + + +def select(rlist, wlist, xlist, timeout=None): # pylint:disable=unused-argument + """An implementation of :meth:`select.select` that blocks only the current greenlet. + + .. caution:: *xlist* is ignored. + + .. versionchanged:: 1.2a1 + Raise a :exc:`ValueError` if timeout is negative. This matches Python 3's + behaviour (Python 2 would raise a ``select.error``). Previously gevent had + undefined behaviour. + .. versionchanged:: 1.2a1 + Raise an exception if any of the file descriptors are invalid. + """ + if timeout is not None and timeout < 0: + # Raise an error like the real implementation; which error + # depends on the version. Python 3, where select.error is OSError, + # raises a ValueError (which makes sense). Older pythons raise + # the error from the select syscall...but we don't actually get there. + # We choose to just raise the ValueError as it makes more sense and is + # forward compatible + raise ValueError("timeout must be non-negative") + + # First, do a poll with the original select system call. This + # is the most efficient way to check to see if any of the file descriptors + # have previously been closed and raise the correct corresponding exception. + # (Because libev tends to just return them as ready...) + # We accept the *xlist* here even though we can't below because this is all about + # error handling. + sel_results = ((), (), ()) + try: + sel_results = _original_select(rlist, wlist, xlist, 0) + except error as e: + enumber = getattr(e, 'errno', None) or e.args[0] + if enumber != EINTR: + # Ignore interrupted syscalls + raise + + if sel_results[0] or sel_results[1] or sel_results[2] or (timeout is not None and timeout == 0): + # If we actually had stuff ready, go ahead and return it. No need + # to go through the trouble of doing our own stuff. + + # Likewise, if the timeout is 0, we already did a 0 timeout + # select and we don't need to do it again. Note that in libuv, + # zero duration timers may be called immediately, without + # cycling the event loop at all. 2.7/test_telnetlib.py "hangs" + # calling zero-duration timers if we go to the loop here. + + # However, because this is typically a place where scheduling switches + # can occur, we need to make sure that's still the case; otherwise a single + # consumer could monopolize the thread. (shows up in test_ftplib.) + _g_sleep() + return sel_results + + result = SelectResult() + return result.select(rlist, wlist, timeout) + + +if original_poll is not None: + class PollResult(object): + __slots__ = ('events', 'event') + + def __init__(self): + self.events = set() + self.event = Event() + + def add_event(self, events, fd): + if events < 0: + result_flags = POLLNVAL + else: + result_flags = 0 + if events & _EV_READ: + result_flags = POLLIN + if events & _EV_WRITE: + result_flags |= POLLOUT + + self.events.add((fd, result_flags)) + self.event.set() + + class poll(object): + """ + An implementation of :class:`select.poll` that blocks only the current greenlet. + + .. caution:: ``POLLPRI`` data is not supported. + + .. versionadded:: 1.1b1 + """ + def __init__(self): + # {int -> flags} + # We can't keep watcher objects in here because people commonly + # just drop the poll object when they're done, without calling + # unregister(). dnspython does this. + self.fds = {} + self.loop = get_hub().loop + + def register(self, fd, eventmask=_NONE): + if eventmask is _NONE: + flags = _EV_READ | _EV_WRITE + else: + flags = 0 + if eventmask & POLLIN: + flags = _EV_READ + if eventmask & POLLOUT: + flags |= _EV_WRITE + # If they ask for POLLPRI, we can't support + # that. Should we raise an error? + + fileno = get_fileno(fd) + self.fds[fileno] = flags + + def modify(self, fd, eventmask): + self.register(fd, eventmask) + + def poll(self, timeout=None): + """ + poll the registered fds. + + .. versionchanged:: 1.2a1 + File descriptors that are closed are reported with POLLNVAL. + + .. versionchanged:: 1.3a2 + Under libuv, interpret *timeout* values less than 0 the same as *None*, + i.e., block. This was always the case with libev. + """ + result = PollResult() + watchers = [] + io = self.loop.io + MAXPRI = self.loop.MAXPRI + try: + for fd, flags in iteritems(self.fds): + watcher = io(fd, flags) + watchers.append(watcher) + watcher.priority = MAXPRI + watcher.start(result.add_event, fd, pass_events=True) + if timeout is not None: + if timeout < 0: + # The docs for python say that an omitted timeout, + # a negative timeout and a timeout of None are all + # supposed to block forever. Many, but not all + # OS's accept any negative number to mean that. Some + # OS's raise errors for anything negative but not -1. + # Python 3.7 changes to always pass exactly -1 in that + # case from selectors. + + # Our Timeout class currently does not have a defined behaviour + # for negative values. On libuv, it uses a check watcher and effectively + # doesn't block. On libev, it seems to block. In either case, we + # *want* to block, so turn this into the sure fire block request. + timeout = None + elif timeout: + # The docs for poll.poll say timeout is in + # milliseconds. Our result objects work in + # seconds, so this should be *=, shouldn't it? + timeout /= 1000.0 + result.event.wait(timeout=timeout) + return list(result.events) + finally: + for awatcher in watchers: + awatcher.stop() + awatcher.close() + + def unregister(self, fd): + """ + Unregister the *fd*. + + .. versionchanged:: 1.2a1 + Raise a `KeyError` if *fd* was not registered, like the standard + library. Previously gevent did nothing. + """ + fileno = get_fileno(fd) + del self.fds[fileno] + +del original_poll diff --git a/src/gevent/server.py b/src/gevent/server.py new file mode 100644 index 0000000..4c48bd3 --- /dev/null +++ b/src/gevent/server.py @@ -0,0 +1,282 @@ +# Copyright (c) 2009-2012 Denis Bilenko. See LICENSE for details. +"""TCP/SSL server""" + +from contextlib import closing + +import sys + +from _socket import error as SocketError +from _socket import SOL_SOCKET +from _socket import SO_REUSEADDR +from _socket import AF_INET +from _socket import SOCK_DGRAM + +from gevent.baseserver import BaseServer +from gevent.socket import EWOULDBLOCK +from gevent.socket import socket as GeventSocket +from gevent._compat import PYPY, PY3 + +__all__ = ['StreamServer', 'DatagramServer'] + + +if sys.platform == 'win32': + # SO_REUSEADDR on Windows does not mean the same thing as on *nix (issue #217) + DEFAULT_REUSE_ADDR = None +else: + DEFAULT_REUSE_ADDR = 1 + + +if PY3: + # sockets and SSL sockets are context managers on Python 3 + def _closing_socket(sock): + return sock +else: + # but they are not guaranteed to be so on Python 2 + _closing_socket = closing + + +class StreamServer(BaseServer): + """ + A generic TCP server. + + Accepts connections on a listening socket and spawns user-provided + *handle* function for each connection with 2 arguments: the client + socket and the client address. + + Note that although the errors in a successfully spawned handler + will not affect the server or other connections, the errors raised + by :func:`accept` and *spawn* cause the server to stop accepting + for a short amount of time. The exact period depends on the values + of :attr:`min_delay` and :attr:`max_delay` attributes. + + The delay starts with :attr:`min_delay` and doubles with each + successive error until it reaches :attr:`max_delay`. A successful + :func:`accept` resets the delay to :attr:`min_delay` again. + + See :class:`~gevent.baseserver.BaseServer` for information on defining the *handle* + function and important restrictions on it. + + **SSL Support** + + The server can optionally work in SSL mode when given the correct + keyword arguments. (That is, the presence of any keyword arguments + will trigger SSL mode.) On Python 2.7.9 and later (any Python + version that supports the :class:`ssl.SSLContext`), this can be + done with a configured ``SSLContext``. On any Python version, it + can be done by passing the appropriate arguments for + :func:`ssl.wrap_socket`. + + The incoming socket will be wrapped into an SSL socket before + being passed to the *handle* function. + + If the *ssl_context* keyword argument is present, it should + contain an :class:`ssl.SSLContext`. The remaining keyword + arguments are passed to the :meth:`ssl.SSLContext.wrap_socket` + method of that object. Depending on the Python version, supported arguments + may include: + + - server_hostname + - suppress_ragged_eofs + - do_handshake_on_connect + + .. caution:: When using an SSLContext, it should either be + imported from :mod:`gevent.ssl`, or the process needs to be monkey-patched. + If the process is not monkey-patched and you pass the standard library + SSLContext, the resulting client sockets will not cooperate with gevent. + + Otherwise, keyword arguments are assumed to apply to :func:`ssl.wrap_socket`. + These keyword arguments may include: + + - keyfile + - certfile + - cert_reqs + - ssl_version + - ca_certs + - suppress_ragged_eofs + - do_handshake_on_connect + - ciphers + + .. versionchanged:: 1.2a2 + Add support for the *ssl_context* keyword argument. + + """ + # the default backlog to use if none was provided in __init__ + backlog = 256 + + reuse_addr = DEFAULT_REUSE_ADDR + + def __init__(self, listener, handle=None, backlog=None, spawn='default', **ssl_args): + BaseServer.__init__(self, listener, handle=handle, spawn=spawn) + try: + if ssl_args: + ssl_args.setdefault('server_side', True) + if 'ssl_context' in ssl_args: + ssl_context = ssl_args.pop('ssl_context') + self.wrap_socket = ssl_context.wrap_socket + self.ssl_args = ssl_args + else: + from gevent.ssl import wrap_socket + self.wrap_socket = wrap_socket + self.ssl_args = ssl_args + else: + self.ssl_args = None + if backlog is not None: + if hasattr(self, 'socket'): + raise TypeError('backlog must be None when a socket instance is passed') + self.backlog = backlog + except: + self.close() + raise + + @property + def ssl_enabled(self): + return self.ssl_args is not None + + def set_listener(self, listener): + BaseServer.set_listener(self, listener) + try: + self.socket = self.socket._sock + except AttributeError: + pass + + def init_socket(self): + if not hasattr(self, 'socket'): + # FIXME: clean up the socket lifetime + # pylint:disable=attribute-defined-outside-init + self.socket = self.get_listener(self.address, self.backlog, self.family) + self.address = self.socket.getsockname() + if self.ssl_args: + self._handle = self.wrap_socket_and_handle + else: + self._handle = self.handle + + @classmethod + def get_listener(cls, address, backlog=None, family=None): + if backlog is None: + backlog = cls.backlog + return _tcp_listener(address, backlog=backlog, reuse_addr=cls.reuse_addr, family=family) + + if PY3: + + def do_read(self): + sock = self.socket + try: + fd, address = sock._accept() + except BlockingIOError: # python 2: pylint: disable=undefined-variable + if not sock.timeout: + return + raise + + sock = GeventSocket(sock.family, sock.type, sock.proto, fileno=fd) + # XXX Python issue #7995? + return sock, address + + else: + + def do_read(self): + try: + client_socket, address = self.socket.accept() + except SocketError as err: + if err.args[0] == EWOULDBLOCK: + return + raise + # XXX: When would this not be the case? In Python 3 it makes sense + # because we're using the low-level _accept method, + # but not in Python 2. + if not isinstance(client_socket, GeventSocket): + # This leads to a leak of the watchers in client_socket + sockobj = GeventSocket(_sock=client_socket) + if PYPY: + client_socket._drop() + else: + sockobj = client_socket + return sockobj, address + + def do_close(self, sock, *args): + # pylint:disable=arguments-differ + sock.close() + + def wrap_socket_and_handle(self, client_socket, address): + # used in case of ssl sockets + with _closing_socket(self.wrap_socket(client_socket, **self.ssl_args)) as ssl_socket: + return self.handle(ssl_socket, address) + + +class DatagramServer(BaseServer): + """A UDP server""" + + reuse_addr = DEFAULT_REUSE_ADDR + + def __init__(self, *args, **kwargs): + # The raw (non-gevent) socket, if possible + self._socket = None + BaseServer.__init__(self, *args, **kwargs) + from gevent.lock import Semaphore + self._writelock = Semaphore() + + def init_socket(self): + if not hasattr(self, 'socket'): + # FIXME: clean up the socket lifetime + # pylint:disable=attribute-defined-outside-init + self.socket = self.get_listener(self.address, self.family) + self.address = self.socket.getsockname() + self._socket = self.socket + try: + self._socket = self._socket._sock + except AttributeError: + pass + + @classmethod + def get_listener(cls, address, family=None): + return _udp_socket(address, reuse_addr=cls.reuse_addr, family=family) + + def do_read(self): + try: + data, address = self._socket.recvfrom(8192) + except SocketError as err: + if err.args[0] == EWOULDBLOCK: + return + raise + return data, address + + def sendto(self, *args): + self._writelock.acquire() + try: + self.socket.sendto(*args) + finally: + self._writelock.release() + + +def _tcp_listener(address, backlog=50, reuse_addr=None, family=AF_INET): + """A shortcut to create a TCP socket, bind it and put it into listening state.""" + sock = GeventSocket(family=family) + if reuse_addr is not None: + sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, reuse_addr) + try: + sock.bind(address) + except SocketError as ex: + strerror = getattr(ex, 'strerror', None) + if strerror is not None: + ex.strerror = strerror + ': ' + repr(address) + raise + sock.listen(backlog) + sock.setblocking(0) + return sock + + +def _udp_socket(address, backlog=50, reuse_addr=None, family=AF_INET): + # backlog argument for compat with tcp_listener + # pylint:disable=unused-argument + + # we want gevent.socket.socket here + sock = GeventSocket(family=family, type=SOCK_DGRAM) + if reuse_addr is not None: + sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, reuse_addr) + try: + sock.bind(address) + except SocketError as ex: + strerror = getattr(ex, 'strerror', None) + if strerror is not None: + ex.strerror = strerror + ': ' + repr(address) + raise + return sock diff --git a/src/gevent/signal.py b/src/gevent/signal.py new file mode 100644 index 0000000..0954af8 --- /dev/null +++ b/src/gevent/signal.py @@ -0,0 +1,137 @@ +""" +Cooperative implementation of special cases of :func:`signal.signal`. + +This module is designed to work with libev's child watchers, as used +by default in :func:`gevent.os.fork` Note that each ``SIGCHLD`` handler +will be run in a new greenlet when the signal is delivered (just like +:class:`gevent.hub.signal`) + +The implementations in this module are only monkey patched if +:func:`gevent.os.waitpid` is being used (the default) and if +:const:`signal.SIGCHLD` is available; see :func:`gevent.os.fork` for +information on configuring this not to be the case for advanced uses. + +.. versionadded:: 1.1b4 +""" + +from __future__ import absolute_import + +from gevent._util import _NONE as _INITIAL +from gevent._util import copy_globals + +import signal as _signal + +__implements__ = [] +__extensions__ = [] + + +_child_handler = _INITIAL + +_signal_signal = _signal.signal +_signal_getsignal = _signal.getsignal + + +def getsignal(signalnum): + """ + Exactly the same as :func:`signal.getsignal` except where + :const:`signal.SIGCHLD` is concerned. + + For :const:`signal.SIGCHLD`, this cooperates with :func:`signal` + to provide consistent answers. + """ + if signalnum != _signal.SIGCHLD: + return _signal_getsignal(signalnum) + + global _child_handler + if _child_handler is _INITIAL: + _child_handler = _signal_getsignal(_signal.SIGCHLD) + + return _child_handler + + +def signal(signalnum, handler): + """ + Exactly the same as :func:`signal.signal` except where + :const:`signal.SIGCHLD` is concerned. + + .. note:: + + A :const:`signal.SIGCHLD` handler installed with this function + will only be triggered for children that are forked using + :func:`gevent.os.fork` (:func:`gevent.os.fork_and_watch`); + children forked before monkey patching, or otherwise by the raw + :func:`os.fork`, will not trigger the handler installed by this + function. (It's unlikely that a SIGCHLD handler installed with + the builtin :func:`signal.signal` would be triggered either; + libev typically overwrites such a handler at the C level. At + the very least, it's full of race conditions.) + + .. note:: + + Use of ``SIG_IGN`` and ``SIG_DFL`` may also have race conditions + with libev child watchers and the :mod:`gevent.subprocess` module. + + .. versionchanged:: 1.2a1 + If ``SIG_IGN`` or ``SIG_DFL`` are used to ignore ``SIGCHLD``, a + future use of ``gevent.subprocess`` and libev child watchers + will once again work. However, on Python 2, use of ``os.popen`` + will fail. + + .. versionchanged:: 1.1rc2 + Allow using ``SIG_IGN`` and ``SIG_DFL`` to reset and ignore ``SIGCHLD``. + However, this allows the possibility of a race condition if ``gevent.subprocess`` + had already been used. + """ + if signalnum != _signal.SIGCHLD: + return _signal_signal(signalnum, handler) + + # TODO: raise value error if not called from the main + # greenlet, just like threads + + if handler != _signal.SIG_IGN and handler != _signal.SIG_DFL and not callable(handler): + # exact same error message raised by the stdlib + raise TypeError("signal handler must be signal.SIG_IGN, signal.SIG_DFL, or a callable object") + + old_handler = getsignal(signalnum) + global _child_handler + _child_handler = handler + if handler in (_signal.SIG_IGN, _signal.SIG_DFL): + # Allow resetting/ignoring this signal at the process level. + # Note that this conflicts with gevent.subprocess and other users + # of child watchers, until the next time gevent.subprocess/loop.install_sigchld() + # is called. + from gevent.hub import get_hub # Are we always safe to import here? + _signal_signal(signalnum, handler) + get_hub().loop.reset_sigchld() + return old_handler + + +def _on_child_hook(): + # This is called in the hub greenlet. To let the function + # do more useful work, like use blocking functions, + # we run it in a new greenlet; see gevent.hub.signal + if callable(_child_handler): + # None is a valid value for the frame argument + from gevent import Greenlet + greenlet = Greenlet(_child_handler, _signal.SIGCHLD, None) + greenlet.switch() + + +import gevent.os + +if 'waitpid' in gevent.os.__implements__ and hasattr(_signal, 'SIGCHLD'): + # Tightly coupled here to gevent.os and its waitpid implementation; only use these + # if necessary. + gevent.os._on_child_hook = _on_child_hook + __implements__.append("signal") + __implements__.append("getsignal") +else: + # XXX: This breaks test__all__ on windows + __extensions__.append("signal") + __extensions__.append("getsignal") + +__imports__ = copy_globals(_signal, globals(), + names_to_ignore=__implements__ + __extensions__, + dunder_names_to_keep=()) + +__all__ = __implements__ + __extensions__ diff --git a/src/gevent/socket.py b/src/gevent/socket.py new file mode 100644 index 0000000..1bb039e --- /dev/null +++ b/src/gevent/socket.py @@ -0,0 +1,129 @@ +# Copyright (c) 2009-2014 Denis Bilenko and gevent contributors. See LICENSE for details. + +"""Cooperative low-level networking interface. + +This module provides socket operations and some related functions. +The API of the functions and classes matches the API of the corresponding +items in the standard :mod:`socket` module exactly, but the synchronous functions +in this module only block the current greenlet and let the others run. + +For convenience, exceptions (like :class:`error ` and :class:`timeout `) +as well as the constants from the :mod:`socket` module are imported into this module. +""" +# Our import magic sadly makes this warning useless +# pylint: disable=undefined-variable + +from gevent._compat import PY3 +from gevent._compat import exc_clear +from gevent._util import copy_globals + + +if PY3: + from gevent import _socket3 as _source # python 2: pylint:disable=no-name-in-module +else: + from gevent import _socket2 as _source + +# define some things we're expecting to overwrite; each module +# needs to define these +__implements__ = __dns__ = __all__ = __extensions__ = __imports__ = () + + +class error(Exception): + errno = None + + +def getfqdn(*args): + # pylint:disable=unused-argument + raise NotImplementedError() + +copy_globals(_source, globals(), + dunder_names_to_keep=('__implements__', '__dns__', '__all__', + '__extensions__', '__imports__', '__socket__'), + cleanup_globs=False) + +# The _socket2 and _socket3 don't import things defined in +# __extensions__, to help avoid confusing reference cycles in the +# documentation and to prevent importing from the wrong place, but we +# *do* need to expose them here. (NOTE: This may lead to some sphinx +# warnings like: +# WARNING: missing attribute mentioned in :members: or __all__: +# module gevent._socket2, attribute cancel_wait +# These can be ignored.) +from gevent import _socketcommon +copy_globals(_socketcommon, globals(), + only_names=_socketcommon.__extensions__) + +try: + _GLOBAL_DEFAULT_TIMEOUT = __socket__._GLOBAL_DEFAULT_TIMEOUT +except AttributeError: + _GLOBAL_DEFAULT_TIMEOUT = object() + + +def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None): + """ + create_connection(address, timeout=None, source_address=None) -> socket + + Connect to *address* and return the :class:`gevent.socket.socket` + object. + + Convenience function. Connect to *address* (a 2-tuple ``(host, + port)``) and return the socket object. Passing the optional + *timeout* parameter will set the timeout on the socket instance + before attempting to connect. If no *timeout* is supplied, the + global default timeout setting returned by + :func:`getdefaulttimeout` is used. If *source_address* is set it + must be a tuple of (host, port) for the socket to bind as a source + address before making the connection. A host of '' or port 0 tells + the OS to use the default. + """ + + host, port = address + # getaddrinfo is documented as returning a list, but our interface + # is pluggable, so be sure it does. + addrs = list(getaddrinfo(host, port, 0, SOCK_STREAM)) + if not addrs: + raise error("getaddrinfo returns an empty list") + + for res in addrs: + af, socktype, proto, _, sa = res + sock = None + try: + sock = socket(af, socktype, proto) + if timeout is not _GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(sa) + except error: + if sock is not None: + sock.close() + sock = None + if res is addrs[-1]: + raise + # without exc_clear(), if connect() fails once, the socket + # is referenced by the frame in exc_info and the next + # bind() fails (see test__socket.TestCreateConnection) + # that does not happen with regular sockets though, + # because _socket.socket.connect() is a built-in. this is + # similar to "getnameinfo loses a reference" failure in + # test_socket.py + exc_clear() + except BaseException: + # Things like GreenletExit, Timeout and KeyboardInterrupt. + # These get raised immediately, being sure to + # close the socket + if sock is not None: + sock.close() + sock = None + raise + else: + try: + return sock + finally: + sock = None + + +# This is promised to be in the __all__ of the _source, but, for circularity reasons, +# we implement it in this module. Mostly for documentation purposes, put it +# in the _source too. +_source.create_connection = create_connection diff --git a/src/gevent/ssl.py b/src/gevent/ssl.py new file mode 100644 index 0000000..2418c41 --- /dev/null +++ b/src/gevent/ssl.py @@ -0,0 +1,35 @@ +""" +Secure Sockets Layer (SSL/TLS) module. +""" +from gevent._compat import PY2 +from gevent._util import copy_globals + +# things we expect to override, here for static analysis +def wrap_socket(_sock, **_kwargs): + # pylint:disable=unused-argument + raise NotImplementedError() + +if PY2: + if hasattr(__import__('ssl'), 'SSLContext'): + # It's not sufficient to check for >= 2.7.9; some distributions + # have backported most of PEP 466. Try to accommodate them. See Issue #702. + # We're just about to import ssl anyway so it's fine to import it here, just + # don't pollute the namespace + from gevent import _sslgte279 as _source + else: # pragma: no cover + from gevent import _ssl2 as _source + import warnings + warnings.warn( + "This version of Python has an insecure SSL implementation. " + "gevent is no longer tested with it, and support will be removed " + "in gevent 1.5. Please use Python 2.7.9 or newer.", + DeprecationWarning, + stacklevel=2, + ) + del warnings +else: + # Py3 + from gevent import _ssl3 as _source # pragma: no cover + + +copy_globals(_source, globals()) diff --git a/src/gevent/subprocess.py b/src/gevent/subprocess.py new file mode 100644 index 0000000..58409c2 --- /dev/null +++ b/src/gevent/subprocess.py @@ -0,0 +1,1688 @@ +""" +Cooperative ``subprocess`` module. + +.. caution:: On POSIX platforms, this module is not usable from native + threads other than the main thread; attempting to do so will raise + a :exc:`TypeError`. This module depends on libev's fork watchers. + On POSIX systems, fork watchers are implemented using signals, and + the thread to which process-directed signals are delivered `is not + defined`_. Because each native thread has its own gevent/libev + loop, this means that a fork watcher registered with one loop + (thread) may never see the signal about a child it spawned if the + signal is sent to a different thread. + +.. note:: The interface of this module is intended to match that of + the standard library :mod:`subprocess` module (with many backwards + compatible extensions from Python 3 backported to Python 2). There + are some small differences between the Python 2 and Python 3 + versions of that module (the Python 2 ``TimeoutExpired`` exception, + notably, extends ``Timeout`` and there is no ``SubprocessError``) and between the + POSIX and Windows versions. The HTML documentation here can only + describe one version; for definitive documentation, see the + standard library or the source code. + +.. _is not defined: http://www.linuxprogrammingblog.com/all-about-linux-signals?page=11 +""" +from __future__ import absolute_import, print_function +# Can we split this up to make it cleaner? See https://github.com/gevent/gevent/issues/748 +# pylint: disable=too-many-lines +# Most of this we inherit from the standard lib +# pylint: disable=bare-except,too-many-locals,too-many-statements,attribute-defined-outside-init +# pylint: disable=too-many-branches,too-many-instance-attributes +# Most of this is cross-platform +# pylint: disable=no-member,expression-not-assigned,unused-argument,unused-variable +import errno +import gc +import os +import signal +import sys +import traceback +from gevent.event import AsyncResult +from gevent.hub import _get_hub_noargs as get_hub +from gevent.hub import linkproxy +from gevent.hub import sleep +from gevent.hub import getcurrent +from gevent._compat import integer_types, string_types, xrange +from gevent._compat import PY3 +from gevent._compat import reraise +from gevent._compat import fspath +from gevent._compat import fsencode +from gevent._util import _NONE +from gevent._util import copy_globals + +from gevent.greenlet import Greenlet, joinall +spawn = Greenlet.spawn +import subprocess as __subprocess__ + + +# Standard functions and classes that this module re-implements in a gevent-aware way. +__implements__ = [ + 'Popen', + 'call', + 'check_call', + 'check_output', +] +if PY3 and not sys.platform.startswith('win32'): + __implements__.append("_posixsubprocess") + _posixsubprocess = None + +# Some symbols we define that we expect to export; +# useful for static analysis +PIPE = "PIPE should be imported" + +# Standard functions and classes that this module re-imports. +__imports__ = [ + 'PIPE', + 'STDOUT', + 'CalledProcessError', + # Windows: + 'CREATE_NEW_CONSOLE', + 'CREATE_NEW_PROCESS_GROUP', + 'STD_INPUT_HANDLE', + 'STD_OUTPUT_HANDLE', + 'STD_ERROR_HANDLE', + 'SW_HIDE', + 'STARTF_USESTDHANDLES', + 'STARTF_USESHOWWINDOW', +] + + +__extra__ = [ + 'MAXFD', + '_eintr_retry_call', + 'STARTUPINFO', + 'pywintypes', + 'list2cmdline', + '_subprocess', + '_winapi', + # Python 2.5 does not have _subprocess, so we don't use it + # XXX We don't run on Py 2.5 anymore; can/could/should we use _subprocess? + # It's only used on mswindows + 'WAIT_OBJECT_0', + 'WaitForSingleObject', + 'GetExitCodeProcess', + 'GetStdHandle', + 'CreatePipe', + 'DuplicateHandle', + 'GetCurrentProcess', + 'DUPLICATE_SAME_ACCESS', + 'GetModuleFileName', + 'GetVersion', + 'CreateProcess', + 'INFINITE', + 'TerminateProcess', + 'STILL_ACTIVE', + + # These were added for 3.5, but we make them available everywhere. + 'run', + 'CompletedProcess', +] + +if sys.version_info[:2] >= (3, 3): + __imports__ += [ + 'DEVNULL', + 'getstatusoutput', + 'getoutput', + 'SubprocessError', + 'TimeoutExpired', + ] +else: + __extra__.append("TimeoutExpired") + + +if sys.version_info[:2] >= (3, 5): + __extra__.remove('run') + __extra__.remove('CompletedProcess') + __implements__.append('run') + __implements__.append('CompletedProcess') + + # Removed in Python 3.5; this is the exact code that was removed: + # https://hg.python.org/cpython/rev/f98b0a5e5ef5 + __extra__.remove('MAXFD') + try: + MAXFD = os.sysconf("SC_OPEN_MAX") + except: + MAXFD = 256 + +if sys.version_info[:2] >= (3, 6): + # This was added to __all__ for windows in 3.6 + __extra__.remove('STARTUPINFO') + __imports__.append('STARTUPINFO') + +if sys.version_info[:2] >= (3, 7): + __imports__.extend([ + 'ABOVE_NORMAL_PRIORITY_CLASS', 'BELOW_NORMAL_PRIORITY_CLASS', + 'HIGH_PRIORITY_CLASS', 'IDLE_PRIORITY_CLASS', + 'NORMAL_PRIORITY_CLASS', + 'REALTIME_PRIORITY_CLASS', + 'CREATE_NO_WINDOW', 'DETACHED_PROCESS', + 'CREATE_DEFAULT_ERROR_MODE', + 'CREATE_BREAKAWAY_FROM_JOB' + ]) + +actually_imported = copy_globals(__subprocess__, globals(), + only_names=__imports__, + ignore_missing_names=True) +# anything we couldn't import from here we may need to find +# elsewhere +__extra__.extend(set(__imports__).difference(set(actually_imported))) +__imports__ = actually_imported +del actually_imported + + +# In Python 3 on Windows, a lot of the functions previously +# in _subprocess moved to _winapi +_subprocess = getattr(__subprocess__, '_subprocess', _NONE) +_winapi = getattr(__subprocess__, '_winapi', _NONE) + +_attr_resolution_order = [__subprocess__, _subprocess, _winapi] + +for name in list(__extra__): + if name in globals(): + continue + value = _NONE + for place in _attr_resolution_order: + value = getattr(place, name, _NONE) + if value is not _NONE: + break + + if value is _NONE: + __extra__.remove(name) + else: + globals()[name] = value + +del _attr_resolution_order +__all__ = __implements__ + __imports__ +# Some other things we want to document +for _x in ('run', 'CompletedProcess', 'TimeoutExpired'): + if _x not in __all__: + __all__.append(_x) + + +mswindows = sys.platform == 'win32' +if mswindows: + import msvcrt # pylint: disable=import-error + if PY3: + class Handle(int): + closed = False + + def Close(self): + if not self.closed: + self.closed = True + _winapi.CloseHandle(self) + + def Detach(self): + if not self.closed: + self.closed = True + return int(self) + raise ValueError("already closed") + + def __repr__(self): + return "Handle(%d)" % int(self) + + __del__ = Close + __str__ = __repr__ +else: + import fcntl + import pickle + from gevent import monkey + fork = monkey.get_original('os', 'fork') + from gevent.os import fork_and_watch + +def call(*popenargs, **kwargs): + """ + call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) -> returncode + + Run command with arguments. Wait for command to complete or + timeout, then return the returncode attribute. + + The arguments are the same as for the Popen constructor. Example:: + + retcode = call(["ls", "-l"]) + + .. versionchanged:: 1.2a1 + The ``timeout`` keyword argument is now accepted on all supported + versions of Python (not just Python 3) and if it expires will raise a + :exc:`TimeoutExpired` exception (under Python 2 this is a subclass of :exc:`~.Timeout`). + """ + timeout = kwargs.pop('timeout', None) + with Popen(*popenargs, **kwargs) as p: + try: + return p.wait(timeout=timeout, _raise_exc=True) + except: + p.kill() + p.wait() + raise + +def check_call(*popenargs, **kwargs): + """ + check_call(args, *, stdin=None, stdout=None, stderr=None, shell=False, timeout=None) -> 0 + + Run command with arguments. Wait for command to complete. If + the exit code was zero then return, otherwise raise + :exc:`CalledProcessError`. The ``CalledProcessError`` object will have the + return code in the returncode attribute. + + The arguments are the same as for the Popen constructor. Example:: + + retcode = check_call(["ls", "-l"]) + """ + retcode = call(*popenargs, **kwargs) + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd) # pylint:disable=undefined-variable + return 0 + +def check_output(*popenargs, **kwargs): + r""" + check_output(args, *, input=None, stdin=None, stderr=None, shell=False, universal_newlines=False, timeout=None) -> output + + Run command with arguments and return its output. + + If the exit code was non-zero it raises a :exc:`CalledProcessError`. The + ``CalledProcessError`` object will have the return code in the returncode + attribute and output in the output attribute. + + + The arguments are the same as for the Popen constructor. Example:: + + >>> check_output(["ls", "-1", "/dev/null"]) + '/dev/null\n' + + The ``stdout`` argument is not allowed as it is used internally. + + To capture standard error in the result, use ``stderr=STDOUT``:: + + >>> print(check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=STDOUT).decode('ascii').strip()) + ls: non_existent_file: No such file or directory + + There is an additional optional argument, "input", allowing you to + pass a string to the subprocess's stdin. If you use this argument + you may not also use the Popen constructor's "stdin" argument, as + it too will be used internally. Example:: + + >>> check_output(["sed", "-e", "s/foo/bar/"], + ... input=b"when in the course of fooman events\n") + 'when in the course of barman events\n' + + If ``universal_newlines=True`` is passed, the return value will be a + string rather than bytes. + + .. versionchanged:: 1.2a1 + The ``timeout`` keyword argument is now accepted on all supported + versions of Python (not just Python 3) and if it expires will raise a + :exc:`TimeoutExpired` exception (under Python 2 this is a subclass of :exc:`~.Timeout`). + .. versionchanged:: 1.2a1 + The ``input`` keyword argument is now accepted on all supported + versions of Python, not just Python 3 + """ + timeout = kwargs.pop('timeout', None) + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + if 'input' in kwargs: + if 'stdin' in kwargs: + raise ValueError('stdin and input arguments may not both be used.') + inputdata = kwargs['input'] + del kwargs['input'] + kwargs['stdin'] = PIPE + else: + inputdata = None + with Popen(*popenargs, stdout=PIPE, **kwargs) as process: + try: + output, unused_err = process.communicate(inputdata, timeout=timeout) + except TimeoutExpired: + process.kill() + output, unused_err = process.communicate() + raise TimeoutExpired(process.args, timeout, output=output) + except: + process.kill() + process.wait() + raise + retcode = process.poll() + if retcode: + # pylint:disable=undefined-variable + raise CalledProcessError(retcode, process.args, output=output) + return output + +_PLATFORM_DEFAULT_CLOSE_FDS = object() + +if 'TimeoutExpired' not in globals(): + # Python 2 + + # Make TimeoutExpired inherit from _Timeout so it can be caught + # the way we used to throw things (except Timeout), but make sure it doesn't + # init a timer. Note that we can't have a fake 'SubprocessError' that inherits + # from exception, because we need TimeoutExpired to just be a BaseException for + # bwc. + from gevent.timeout import Timeout as _Timeout + + class TimeoutExpired(_Timeout): + """ + This exception is raised when the timeout expires while waiting for + a child process in `communicate`. + + Under Python 2, this is a gevent extension with the same name as the + Python 3 class for source-code forward compatibility. However, it extends + :class:`gevent.timeout.Timeout` for backwards compatibility (because + we used to just raise a plain ``Timeout``); note that ``Timeout`` is a + ``BaseException``, *not* an ``Exception``. + + .. versionadded:: 1.2a1 + """ + + def __init__(self, cmd, timeout, output=None): + _Timeout.__init__(self, None) + self.cmd = cmd + self.seconds = timeout + self.output = output + + @property + def timeout(self): + return self.seconds + + def __str__(self): + return ("Command '%s' timed out after %s seconds" % + (self.cmd, self.timeout)) + + +if hasattr(os, 'set_inheritable'): + _set_inheritable = os.set_inheritable +else: + _set_inheritable = lambda i, v: True + + +def FileObject(*args): + # Defer importing FileObject until we need it + # to allow it to be configured more easily. + from gevent.fileobject import FileObject as _FileObject + globals()['FileObject'] = _FileObject + return _FileObject(*args) + +class Popen(object): + """ + The underlying process creation and management in this module is + handled by the Popen class. It offers a lot of flexibility so that + developers are able to handle the less common cases not covered by + the convenience functions. + + .. seealso:: :class:`subprocess.Popen` + This class should have the same interface as the standard library class. + + .. versionchanged:: 1.2a1 + Instances can now be used as context managers under Python 2.7. Previously + this was restricted to Python 3. + + .. versionchanged:: 1.2a1 + Instances now save the ``args`` attribute under Python 2.7. Previously this was + restricted to Python 3. + + .. versionchanged:: 1.2b1 + Add the ``encoding`` and ``errors`` parameters for Python 3. + + .. versionchanged:: 1.3a1 + Accept "path-like" objects for the *cwd* parameter on all platforms. + This was added to Python 3.6. Previously with gevent, it only worked + on POSIX platforms on 3.6. + + .. versionchanged:: 1.3a1 + Add the ``text`` argument as a synonym for ``universal_newlines``, + as added on Python 3.7. + + .. versionchanged:: 1.3a2 + Allow the same keyword arguments under Python 2 as Python 3: + ``pass_fds``, ``start_new_session``, ``restore_signals``, ``encoding`` + and ``errors``. Under Python 2, ``encoding`` and ``errors`` are ignored + because native handling of universal newlines is used. + + .. versionchanged:: 1.3a2 + Under Python 2, ``restore_signals`` defaults to ``False``. Previously it + defaulted to ``True``, the same as it did in Python 3. + """ + + # The value returned from communicate() when there was nothing to read. + # Changes if we're in text mode or universal newlines mode. + _communicate_empty_value = b'' + + def __init__(self, args, + bufsize=-1 if PY3 else 0, + executable=None, + stdin=None, stdout=None, stderr=None, + preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, shell=False, + cwd=None, env=None, universal_newlines=None, + startupinfo=None, creationflags=0, + restore_signals=PY3, start_new_session=False, + pass_fds=(), + # Added in 3.6. These are kept as ivars + encoding=None, errors=None, + # Added in 3.7. Not an ivar directly. + text=None, + # gevent additions + threadpool=None): + + self.encoding = encoding + self.errors = errors + + hub = get_hub() + + if bufsize is None: + # Python 2 doesn't allow None at all, but Python 3 treats + # it the same as the default. We do as well. + bufsize = -1 if PY3 else 0 + if not isinstance(bufsize, integer_types): + raise TypeError("bufsize must be an integer") + + if mswindows: + if preexec_fn is not None: + raise ValueError("preexec_fn is not supported on Windows " + "platforms") + if sys.version_info[:2] >= (3, 7): + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + close_fds = True + else: + any_stdio_set = (stdin is not None or stdout is not None or + stderr is not None) + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + if any_stdio_set: + close_fds = False + else: + close_fds = True + elif close_fds and any_stdio_set: + raise ValueError("close_fds is not supported on Windows " + "platforms if you redirect stdin/stdout/stderr") + if threadpool is None: + threadpool = hub.threadpool + self.threadpool = threadpool + self._waiting = False + else: + # POSIX + if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: + # close_fds has different defaults on Py3/Py2 + if PY3: # pylint: disable=simplifiable-if-statement + close_fds = True + else: + close_fds = False + + if pass_fds and not close_fds: + import warnings + warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) + close_fds = True + if startupinfo is not None: + raise ValueError("startupinfo is only supported on Windows " + "platforms") + if creationflags != 0: + raise ValueError("creationflags is only supported on Windows " + "platforms") + assert threadpool is None + self._loop = hub.loop + + # Validate the combinations of text and universal_newlines + if (text is not None and universal_newlines is not None + and bool(universal_newlines) != bool(text)): + # pylint:disable=undefined-variable + raise SubprocessError('Cannot disambiguate when both text ' + 'and universal_newlines are supplied but ' + 'different. Pass one or the other.') + + self.args = args # Previously this was Py3 only. + self.stdin = None + self.stdout = None + self.stderr = None + self.pid = None + self.returncode = None + self.universal_newlines = universal_newlines + self.result = AsyncResult() + + # Input and output objects. The general principle is like + # this: + # + # Parent Child + # ------ ----- + # p2cwrite ---stdin---> p2cread + # c2pread <--stdout--- c2pwrite + # errread <--stderr--- errwrite + # + # On POSIX, the child objects are file descriptors. On + # Windows, these are Windows file handles. The parent objects + # are file descriptors on both platforms. The parent objects + # are -1 when not using PIPEs. The child objects are -1 + # when not redirecting. + + (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) = self._get_handles(stdin, stdout, stderr) + + # We wrap OS handles *before* launching the child, otherwise a + # quickly terminating child could make our fds unwrappable + # (see #8458). + if mswindows: + if p2cwrite != -1: + p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) + if c2pread != -1: + c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) + if errread != -1: + errread = msvcrt.open_osfhandle(errread.Detach(), 0) + + text_mode = PY3 and (self.encoding or self.errors or universal_newlines or text) + if text_mode or universal_newlines: + # Always a native str in universal_newlines mode, even when that + # str type is bytes. Additionally, text_mode is only true under + # Python 3, so it's actually a unicode str + self._communicate_empty_value = '' + + + if p2cwrite != -1: + if PY3 and text_mode: + # Under Python 3, if we left on the 'b' we'd get different results + # depending on whether we used FileObjectPosix or FileObjectThread + self.stdin = FileObject(p2cwrite, 'wb', bufsize) + self.stdin.translate_newlines(None, + write_through=True, + line_buffering=(bufsize == 1), + encoding=self.encoding, errors=self.errors) + else: + self.stdin = FileObject(p2cwrite, 'wb', bufsize) + if c2pread != -1: + if universal_newlines or text_mode: + if PY3: + # FileObjectThread doesn't support the 'U' qualifier + # with a bufsize of 0 + self.stdout = FileObject(c2pread, 'rb', bufsize) + # NOTE: Universal Newlines are broken on Windows/Py3, at least + # in some cases. This is true in the stdlib subprocess module + # as well; the following line would fix the test cases in + # test__subprocess.py that depend on python_universal_newlines, + # but would be inconsistent with the stdlib: + #msvcrt.setmode(self.stdout.fileno(), os.O_TEXT) + self.stdout.translate_newlines('r', encoding=self.encoding, errors=self.errors) + else: + self.stdout = FileObject(c2pread, 'rU', bufsize) + else: + self.stdout = FileObject(c2pread, 'rb', bufsize) + if errread != -1: + if universal_newlines or text_mode: + if PY3: + self.stderr = FileObject(errread, 'rb', bufsize) + self.stderr.translate_newlines(None, encoding=encoding, errors=errors) + else: + self.stderr = FileObject(errread, 'rU', bufsize) + else: + self.stderr = FileObject(errread, 'rb', bufsize) + + self._closed_child_pipe_fds = False + # Convert here for the sake of all platforms. os.chdir accepts + # path-like objects natively under 3.6, but CreateProcess + # doesn't. + cwd = fspath(cwd) if cwd is not None else None + try: + self._execute_child(args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, universal_newlines, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, + restore_signals, start_new_session) + except: + # Cleanup if the child failed starting. + # (gevent: New in python3, but reported as gevent bug in #347. + # Note that under Py2, any error raised below will replace the + # original error so we have to use reraise) + if not PY3: + exc_info = sys.exc_info() + for f in filter(None, (self.stdin, self.stdout, self.stderr)): + try: + f.close() + except (OSError, IOError): + pass # Ignore EBADF or other errors. + + if not self._closed_child_pipe_fds: + to_close = [] + if stdin == PIPE: + to_close.append(p2cread) + if stdout == PIPE: + to_close.append(c2pwrite) + if stderr == PIPE: + to_close.append(errwrite) + if hasattr(self, '_devnull'): + to_close.append(self._devnull) + for fd in to_close: + try: + os.close(fd) + except (OSError, IOError): + pass + if not PY3: + try: + reraise(*exc_info) + finally: + del exc_info + raise + + def __repr__(self): + return '<%s at 0x%x pid=%r returncode=%r>' % (self.__class__.__name__, id(self), self.pid, self.returncode) + + def _on_child(self, watcher): + watcher.stop() + status = watcher.rstatus + if os.WIFSIGNALED(status): + self.returncode = -os.WTERMSIG(status) + else: + self.returncode = os.WEXITSTATUS(status) + self.result.set(self.returncode) + + def _get_devnull(self): + if not hasattr(self, '_devnull'): + self._devnull = os.open(os.devnull, os.O_RDWR) + return self._devnull + + _stdout_buffer = None + _stderr_buffer = None + + def communicate(self, input=None, timeout=None): + """Interact with process: Send data to stdin. Read data from + stdout and stderr, until end-of-file is reached. Wait for + process to terminate. The optional input argument should be a + string to be sent to the child process, or None, if no data + should be sent to the child. + + communicate() returns a tuple (stdout, stderr). + + :keyword timeout: Under Python 2, this is a gevent extension; if + given and it expires, we will raise :exc:`TimeoutExpired`, which + extends :exc:`gevent.timeout.Timeout` (note that this only extends :exc:`BaseException`, + *not* :exc:`Exception`) + Under Python 3, this raises the standard :exc:`TimeoutExpired` exception. + + .. versionchanged:: 1.1a2 + Under Python 2, if the *timeout* elapses, raise the :exc:`gevent.timeout.Timeout` + exception. Previously, we silently returned. + .. versionchanged:: 1.1b5 + Honor a *timeout* even if there's no way to communicate with the child + (stdin, stdout, and stderr are not pipes). + """ + greenlets = [] + if self.stdin: + greenlets.append(spawn(write_and_close, self.stdin, input)) + + # If the timeout parameter is used, and the caller calls back after + # getting a TimeoutExpired exception, we can wind up with multiple + # greenlets trying to run and read from and close stdout/stderr. + # That's bad because it can lead to 'RuntimeError: reentrant call in io.BufferedReader'. + # We can't just kill the previous greenlets when a timeout happens, + # though, because we risk losing the output collected by that greenlet + # (and Python 3, where timeout is an official parameter, explicitly says + # that no output should be lost in the event of a timeout.) Instead, we're + # watching for the exception and ignoring it. It's not elegant, + # but it works + def _make_pipe_reader(pipe_name): + pipe = getattr(self, pipe_name) + buf_name = '_' + pipe_name + '_buffer' + + def _read(): + try: + data = pipe.read() + except RuntimeError: + return + if not data: + return + the_buffer = getattr(self, buf_name) + if the_buffer: + the_buffer.append(data) + else: + setattr(self, buf_name, [data]) + return _read + + if self.stdout: + _read_out = _make_pipe_reader('stdout') + stdout = spawn(_read_out) + greenlets.append(stdout) + else: + stdout = None + + if self.stderr: + _read_err = _make_pipe_reader('stderr') + stderr = spawn(_read_err) + greenlets.append(stderr) + else: + stderr = None + + # If we were given stdin=stdout=stderr=None, we have no way to + # communicate with the child, and thus no greenlets to wait + # on. This is a nonsense case, but it comes up in the test + # case for Python 3.5 (test_subprocess.py + # RunFuncTestCase.test_timeout). Instead, we go directly to + # self.wait + if not greenlets and timeout is not None: + self.wait(timeout=timeout, _raise_exc=True) + + done = joinall(greenlets, timeout=timeout) + if timeout is not None and len(done) != len(greenlets): + raise TimeoutExpired(self.args, timeout) + + for pipe in (self.stdout, self.stderr): + if pipe: + try: + pipe.close() + except RuntimeError: + pass + + self.wait() + + def _get_output_value(pipe_name): + buf_name = '_' + pipe_name + '_buffer' + buf_value = getattr(self, buf_name) + setattr(self, buf_name, None) + if buf_value: + buf_value = self._communicate_empty_value.join(buf_value) + else: + buf_value = self._communicate_empty_value + return buf_value + + stdout_value = _get_output_value('stdout') + stderr_value = _get_output_value('stderr') + + return (None if stdout is None else stdout_value, + None if stderr is None else stderr_value) + + def poll(self): + """Check if child process has terminated. Set and return :attr:`returncode` attribute.""" + return self._internal_poll() + + def __enter__(self): + return self + + def __exit__(self, t, v, tb): + if self.stdout: + self.stdout.close() + if self.stderr: + self.stderr.close() + try: # Flushing a BufferedWriter may raise an error + if self.stdin: + self.stdin.close() + finally: + # Wait for the process to terminate, to avoid zombies. + # JAM: gevent: If the process never terminates, this + # blocks forever. + self.wait() + + def _gevent_result_wait(self, timeout=None, raise_exc=PY3): + result = self.result.wait(timeout=timeout) + if raise_exc and timeout is not None and not self.result.ready(): + raise TimeoutExpired(self.args, timeout) + return result + + + if mswindows: + # + # Windows methods + # + def _get_handles(self, stdin, stdout, stderr): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ + # pylint:disable=undefined-variable + if stdin is None and stdout is None and stderr is None: + return (-1, -1, -1, -1, -1, -1) + + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 + + try: + DEVNULL + except NameError: + _devnull = object() + else: + _devnull = DEVNULL + + if stdin is None: + p2cread = GetStdHandle(STD_INPUT_HANDLE) + if p2cread is None: + p2cread, _ = CreatePipe(None, 0) + if PY3: + p2cread = Handle(p2cread) + _winapi.CloseHandle(_) + elif stdin == PIPE: + p2cread, p2cwrite = CreatePipe(None, 0) + if PY3: + p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite) + elif stdin == _devnull: + p2cread = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stdin, int): + p2cread = msvcrt.get_osfhandle(stdin) + else: + # Assuming file-like object + p2cread = msvcrt.get_osfhandle(stdin.fileno()) + p2cread = self._make_inheritable(p2cread) + + if stdout is None: + c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) + if c2pwrite is None: + _, c2pwrite = CreatePipe(None, 0) + if PY3: + c2pwrite = Handle(c2pwrite) + _winapi.CloseHandle(_) + elif stdout == PIPE: + c2pread, c2pwrite = CreatePipe(None, 0) + if PY3: + c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite) + elif stdout == _devnull: + c2pwrite = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stdout, int): + c2pwrite = msvcrt.get_osfhandle(stdout) + else: + # Assuming file-like object + c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) + c2pwrite = self._make_inheritable(c2pwrite) + + if stderr is None: + errwrite = GetStdHandle(STD_ERROR_HANDLE) + if errwrite is None: + _, errwrite = CreatePipe(None, 0) + if PY3: + errwrite = Handle(errwrite) + _winapi.CloseHandle(_) + elif stderr == PIPE: + errread, errwrite = CreatePipe(None, 0) + if PY3: + errread, errwrite = Handle(errread), Handle(errwrite) + elif stderr == STDOUT: + errwrite = c2pwrite + elif stderr == _devnull: + errwrite = msvcrt.get_osfhandle(self._get_devnull()) + elif isinstance(stderr, int): + errwrite = msvcrt.get_osfhandle(stderr) + else: + # Assuming file-like object + errwrite = msvcrt.get_osfhandle(stderr.fileno()) + errwrite = self._make_inheritable(errwrite) + + return (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + def _make_inheritable(self, handle): + """Return a duplicate of handle, which is inheritable""" + # pylint:disable=undefined-variable + return DuplicateHandle(GetCurrentProcess(), + handle, GetCurrentProcess(), 0, 1, + DUPLICATE_SAME_ACCESS) + + def _find_w9xpopen(self): + """Find and return absolute path to w9xpopen.exe""" + # pylint:disable=undefined-variable + w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), + "w9xpopen.exe") + if not os.path.exists(w9xpopen): + # Eeek - file-not-found - possibly an embedding + # situation - see if we can locate it in sys.exec_prefix + w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), + "w9xpopen.exe") + if not os.path.exists(w9xpopen): + raise RuntimeError("Cannot locate w9xpopen.exe, which is " + "needed for Popen to work with your " + "shell or platform.") + return w9xpopen + + + def _filter_handle_list(self, handle_list): + """Filter out console handles that can't be used + in lpAttributeList["handle_list"] and make sure the list + isn't empty. This also removes duplicate handles.""" + # An handle with it's lowest two bits set might be a special console + # handle that if passed in lpAttributeList["handle_list"], will + # cause it to fail. + # Only works on 3.7+ + return list({handle for handle in handle_list + if handle & 0x3 != 0x3 + or _winapi.GetFileType(handle) != + _winapi.FILE_TYPE_CHAR}) + + + def _execute_child(self, args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, universal_newlines, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, + unused_restore_signals, unused_start_new_session): + """Execute program (MS Windows version)""" + # pylint:disable=undefined-variable + assert not pass_fds, "pass_fds not supported on Windows." + + if not isinstance(args, string_types): + args = list2cmdline(args) + + # Process startup details + if startupinfo is None: + startupinfo = STARTUPINFO() + use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) + if use_std_handles: + startupinfo.dwFlags |= STARTF_USESTDHANDLES + startupinfo.hStdInput = p2cread + startupinfo.hStdOutput = c2pwrite + startupinfo.hStdError = errwrite + + if hasattr(startupinfo, 'lpAttributeList'): + # Support for Python >= 3.7 + + attribute_list = startupinfo.lpAttributeList + have_handle_list = bool(attribute_list and + "handle_list" in attribute_list and + attribute_list["handle_list"]) + + # If we were given an handle_list or need to create one + if have_handle_list or (use_std_handles and close_fds): + if attribute_list is None: + attribute_list = startupinfo.lpAttributeList = {} + handle_list = attribute_list["handle_list"] = \ + list(attribute_list.get("handle_list", [])) + + if use_std_handles: + handle_list += [int(p2cread), int(c2pwrite), int(errwrite)] + + handle_list[:] = self._filter_handle_list(handle_list) + + if handle_list: + if not close_fds: + import warnings + warnings.warn("startupinfo.lpAttributeList['handle_list'] " + "overriding close_fds", RuntimeWarning) + + # When using the handle_list we always request to inherit + # handles but the only handles that will be inherited are + # the ones in the handle_list + close_fds = False + + if shell: + startupinfo.dwFlags |= STARTF_USESHOWWINDOW + startupinfo.wShowWindow = SW_HIDE + comspec = os.environ.get("COMSPEC", "cmd.exe") + args = '{} /c "{}"'.format(comspec, args) + if GetVersion() >= 0x80000000 or os.path.basename(comspec).lower() == "command.com": + # Win9x, or using command.com on NT. We need to + # use the w9xpopen intermediate program. For more + # information, see KB Q150956 + # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) + w9xpopen = self._find_w9xpopen() + args = '"%s" %s' % (w9xpopen, args) + # Not passing CREATE_NEW_CONSOLE has been known to + # cause random failures on win9x. Specifically a + # dialog: "Your program accessed mem currently in + # use at xxx" and a hopeful warning about the + # stability of your system. Cost is Ctrl+C wont + # kill children. + creationflags |= CREATE_NEW_CONSOLE + + # Start the process + try: + hp, ht, pid, tid = CreateProcess(executable, args, + # no special security + None, None, + int(not close_fds), + creationflags, + env, + cwd, + startupinfo) + except IOError as e: # From 2.6 on, pywintypes.error was defined as IOError + # Translate pywintypes.error to WindowsError, which is + # a subclass of OSError. FIXME: We should really + # translate errno using _sys_errlist (or similar), but + # how can this be done from Python? + if PY3: + raise # don't remap here + raise WindowsError(*e.args) + finally: + # Child is launched. Close the parent's copy of those pipe + # handles that only the child should have open. You need + # to make sure that no handles to the write end of the + # output pipe are maintained in this process or else the + # pipe will not close when the child process exits and the + # ReadFile will hang. + def _close(x): + if x is not None and x != -1: + if hasattr(x, 'Close'): + x.Close() + else: + _winapi.CloseHandle(x) + + _close(p2cread) + _close(c2pwrite) + _close(errwrite) + if hasattr(self, '_devnull'): + os.close(self._devnull) + + # Retain the process handle, but close the thread handle + self._child_created = True + self._handle = Handle(hp) if not hasattr(hp, 'Close') else hp + self.pid = pid + _winapi.CloseHandle(ht) if not hasattr(ht, 'Close') else ht.Close() + + def _internal_poll(self): + """Check if child process has terminated. Returns returncode + attribute. + """ + # pylint:disable=undefined-variable + if self.returncode is None: + if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: + self.returncode = GetExitCodeProcess(self._handle) + self.result.set(self.returncode) + return self.returncode + + def rawlink(self, callback): + if not self.result.ready() and not self._waiting: + self._waiting = True + Greenlet.spawn(self._wait) + self.result.rawlink(linkproxy(callback, self)) + # XXX unlink + + def _blocking_wait(self): + # pylint:disable=undefined-variable + WaitForSingleObject(self._handle, INFINITE) + self.returncode = GetExitCodeProcess(self._handle) + return self.returncode + + def _wait(self): + self.threadpool.spawn(self._blocking_wait).rawlink(self.result) + + def wait(self, timeout=None, _raise_exc=PY3): + """Wait for child process to terminate. Returns returncode + attribute.""" + if self.returncode is None: + if not self._waiting: + self._waiting = True + self._wait() + return self._gevent_result_wait(timeout, _raise_exc) + + def send_signal(self, sig): + """Send a signal to the process + """ + if sig == signal.SIGTERM: + self.terminate() + elif sig == signal.CTRL_C_EVENT: + os.kill(self.pid, signal.CTRL_C_EVENT) + elif sig == signal.CTRL_BREAK_EVENT: + os.kill(self.pid, signal.CTRL_BREAK_EVENT) + else: + raise ValueError("Unsupported signal: {}".format(sig)) + + def terminate(self): + """Terminates the process + """ + # pylint:disable=undefined-variable + # Don't terminate a process that we know has already died. + if self.returncode is not None: + return + try: + TerminateProcess(self._handle, 1) + except OSError as e: + # ERROR_ACCESS_DENIED (winerror 5) is received when the + # process already died. + if e.winerror != 5: + raise + rc = GetExitCodeProcess(self._handle) + if rc == STILL_ACTIVE: + raise + self.returncode = rc + self.result.set(self.returncode) + + kill = terminate + + else: + # + # POSIX methods + # + + def rawlink(self, callback): + # Not public documented, part of the link protocol + self.result.rawlink(linkproxy(callback, self)) + # XXX unlink + + def _get_handles(self, stdin, stdout, stderr): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ + p2cread, p2cwrite = -1, -1 + c2pread, c2pwrite = -1, -1 + errread, errwrite = -1, -1 + + try: + DEVNULL + except NameError: + _devnull = object() + else: + _devnull = DEVNULL + + if stdin is None: + pass + elif stdin == PIPE: + p2cread, p2cwrite = self.pipe_cloexec() + elif stdin == _devnull: + p2cread = self._get_devnull() + elif isinstance(stdin, int): + p2cread = stdin + else: + # Assuming file-like object + p2cread = stdin.fileno() + + if stdout is None: + pass + elif stdout == PIPE: + c2pread, c2pwrite = self.pipe_cloexec() + elif stdout == _devnull: + c2pwrite = self._get_devnull() + elif isinstance(stdout, int): + c2pwrite = stdout + else: + # Assuming file-like object + c2pwrite = stdout.fileno() + + if stderr is None: + pass + elif stderr == PIPE: + errread, errwrite = self.pipe_cloexec() + elif stderr == STDOUT: # pylint:disable=undefined-variable + if c2pwrite != -1: + errwrite = c2pwrite + else: # child's stdout is not set, use parent's stdout + errwrite = sys.__stdout__.fileno() + elif stderr == _devnull: + errwrite = self._get_devnull() + elif isinstance(stderr, int): + errwrite = stderr + else: + # Assuming file-like object + errwrite = stderr.fileno() + + return (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + def _set_cloexec_flag(self, fd, cloexec=True): + try: + cloexec_flag = fcntl.FD_CLOEXEC + except AttributeError: + cloexec_flag = 1 + + old = fcntl.fcntl(fd, fcntl.F_GETFD) + if cloexec: + fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) + + def _remove_nonblock_flag(self, fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFL) & (~os.O_NONBLOCK) + fcntl.fcntl(fd, fcntl.F_SETFL, flags) + + def pipe_cloexec(self): + """Create a pipe with FDs set CLOEXEC.""" + # Pipes' FDs are set CLOEXEC by default because we don't want them + # to be inherited by other subprocesses: the CLOEXEC flag is removed + # from the child's FDs by _dup2(), between fork() and exec(). + # This is not atomic: we would need the pipe2() syscall for that. + r, w = os.pipe() + self._set_cloexec_flag(r) + self._set_cloexec_flag(w) + return r, w + + _POSSIBLE_FD_DIRS = ( + '/proc/self/fd', # Linux + '/dev/fd', # BSD, including macOS + ) + + @classmethod + def _close_fds(cls, keep, errpipe_write): + # From the C code: + # errpipe_write is part of keep. It must be closed at + # exec(), but kept open in the child process until exec() is + # called. + for path in cls._POSSIBLE_FD_DIRS: + if os.path.isdir(path): + return cls._close_fds_from_path(path, keep, errpipe_write) + return cls._close_fds_brute_force(keep, errpipe_write) + + @classmethod + def _close_fds_from_path(cls, path, keep, errpipe_write): + # path names a directory whose only entries have + # names that are ascii strings of integers in base10, + # corresponding to the fds the current process has open + try: + fds = [int(fname) for fname in os.listdir(path)] + except (ValueError, OSError): + cls._close_fds_brute_force(keep, errpipe_write) + else: + for i in keep: + if i == errpipe_write: + continue + _set_inheritable(i, True) + + for fd in fds: + if fd in keep or fd < 3: + continue + try: + os.close(fd) + except: + pass + + @classmethod + def _close_fds_brute_force(cls, keep, errpipe_write): + # `keep` is a set of fds, so we + # use os.closerange from 3 to min(keep) + # and then from max(keep + 1) to MAXFD and + # loop through filling in the gaps. + + # Under new python versions, we need to explicitly set + # passed fds to be inheritable or they will go away on exec + + # XXX: Bug: We implicitly rely on errpipe_write being the largest open + # FD so that we don't change its cloexec flag. + + assert hasattr(os, 'closerange') # Added in 2.7 + keep = sorted(keep) + min_keep = min(keep) + max_keep = max(keep) + os.closerange(3, min_keep) + os.closerange(max_keep + 1, MAXFD) + + for i in xrange(min_keep, max_keep): + if i in keep: + _set_inheritable(i, True) + continue + + try: + os.close(i) + except: + pass + + def _execute_child(self, args, executable, preexec_fn, close_fds, + pass_fds, cwd, env, universal_newlines, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite, + restore_signals, start_new_session): + """Execute program (POSIX version)""" + + if PY3 and isinstance(args, (str, bytes)): + args = [args] + elif not PY3 and isinstance(args, string_types): + args = [args] + else: + try: + args = list(args) + except TypeError: # os.PathLike instead of a sequence? + args = [fsencode(args)] # os.PathLike -> [str] + + if shell: + args = ["/bin/sh", "-c"] + args + if executable: + args[0] = executable + + if executable is None: + executable = args[0] + + self._loop.install_sigchld() + + # For transferring possible exec failure from child to parent + # The first char specifies the exception type: 0 means + # OSError, 1 means some other error. + errpipe_read, errpipe_write = self.pipe_cloexec() + # errpipe_write must not be in the standard io 0, 1, or 2 fd range. + low_fds_to_close = [] + while errpipe_write < 3: + low_fds_to_close.append(errpipe_write) + errpipe_write = os.dup(errpipe_write) + for low_fd in low_fds_to_close: + os.close(low_fd) + try: + try: + gc_was_enabled = gc.isenabled() + # Disable gc to avoid bug where gc -> file_dealloc -> + # write to stderr -> hang. http://bugs.python.org/issue1336 + gc.disable() + try: + self.pid = fork_and_watch(self._on_child, self._loop, True, fork) + except: + if gc_was_enabled: + gc.enable() + raise + if self.pid == 0: + # Child + + # XXX: Technically we're doing a lot of stuff here that + # may not be safe to do before a exec(), depending on the OS. + # CPython 3 goes to great lengths to precompute a lot + # of this info before the fork and pass it all to C functions that + # try hard not to call things like malloc(). (Of course, + # CPython 2 pretty much did what we're doing.) + try: + # Close parent's pipe ends + if p2cwrite != -1: + os.close(p2cwrite) + if c2pread != -1: + os.close(c2pread) + if errread != -1: + os.close(errread) + os.close(errpipe_read) + + # When duping fds, if there arises a situation + # where one of the fds is either 0, 1 or 2, it + # is possible that it is overwritten (#12607). + if c2pwrite == 0: + c2pwrite = os.dup(c2pwrite) + while errwrite in (0, 1): + errwrite = os.dup(errwrite) + + # Dup fds for child + def _dup2(existing, desired): + # dup2() removes the CLOEXEC flag but + # we must do it ourselves if dup2() + # would be a no-op (issue #10806). + if existing == desired: + self._set_cloexec_flag(existing, False) + elif existing != -1: + os.dup2(existing, desired) + try: + self._remove_nonblock_flag(desired) + except OSError: + # Ignore EBADF, it may not actually be + # open yet. + # Tested beginning in 3.7.0b3 test_subprocess.py + pass + _dup2(p2cread, 0) + _dup2(c2pwrite, 1) + _dup2(errwrite, 2) + + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + closed = set([None]) + for fd in [p2cread, c2pwrite, errwrite]: + if fd not in closed and fd > 2: + os.close(fd) + closed.add(fd) + + if cwd is not None: + try: + os.chdir(cwd) + except OSError as e: + e._failed_chdir = True + raise + + if preexec_fn: + preexec_fn() + + # Close all other fds, if asked for. This must be done + # after preexec_fn runs. + if close_fds: + fds_to_keep = set(pass_fds) + fds_to_keep.add(errpipe_write) + self._close_fds(fds_to_keep, errpipe_write) + + if restore_signals: + # restore the documented signals back to sig_dfl; + # not all will be defined on every platform + for sig in 'SIGPIPE', 'SIGXFZ', 'SIGXFSZ': + sig = getattr(signal, sig, None) + if sig is not None: + signal.signal(sig, signal.SIG_DFL) + + if start_new_session: + os.setsid() + + if env is None: + os.execvp(executable, args) + else: + if PY3: + # Python 3.6 started testing for + # bytes values in the env; it also + # started encoding strs using + # fsencode and using a lower-level + # API that takes a list of keys + # and values. We don't have access + # to that API, so we go the reverse direction. + env = {os.fsdecode(k) if isinstance(k, bytes) else k: + os.fsdecode(v) if isinstance(v, bytes) else v + for k, v in env.items()} + os.execvpe(executable, args, env) + + except: + exc_type, exc_value, tb = sys.exc_info() + # Save the traceback and attach it to the exception object + exc_lines = traceback.format_exception(exc_type, + exc_value, + tb) + exc_value.child_traceback = ''.join(exc_lines) + os.write(errpipe_write, pickle.dumps(exc_value)) + + finally: + # Make sure that the process exits no matter what. + # The return code does not matter much as it won't be + # reported to the application + os._exit(1) + + # Parent + self._child_created = True + if gc_was_enabled: + gc.enable() + finally: + # be sure the FD is closed no matter what + os.close(errpipe_write) + + # self._devnull is not always defined. + devnull_fd = getattr(self, '_devnull', None) + if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: + os.close(p2cread) + if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: + os.close(c2pwrite) + if errwrite != -1 and errread != -1 and errwrite != devnull_fd: + os.close(errwrite) + if devnull_fd is not None: + os.close(devnull_fd) + # Prevent a double close of these fds from __init__ on error. + self._closed_child_pipe_fds = True + + # Wait for exec to fail or succeed; possibly raising exception + errpipe_read = FileObject(errpipe_read, 'rb') + data = errpipe_read.read() + finally: + if hasattr(errpipe_read, 'close'): + errpipe_read.close() + else: + os.close(errpipe_read) + + if data != b"": + self.wait() + child_exception = pickle.loads(data) + for fd in (p2cwrite, c2pread, errread): + if fd is not None and fd != -1: + os.close(fd) + if isinstance(child_exception, OSError): + child_exception.filename = executable + if hasattr(child_exception, '_failed_chdir'): + child_exception.filename = cwd + raise child_exception + + def _handle_exitstatus(self, sts): + if os.WIFSIGNALED(sts): + self.returncode = -os.WTERMSIG(sts) + elif os.WIFEXITED(sts): + self.returncode = os.WEXITSTATUS(sts) + else: + # Should never happen + raise RuntimeError("Unknown child exit status!") + + def _internal_poll(self): + """Check if child process has terminated. Returns returncode + attribute. + """ + if self.returncode is None: + if get_hub() is not getcurrent(): + sig_pending = getattr(self._loop, 'sig_pending', True) + if sig_pending: + sleep(0.00001) + return self.returncode + + def wait(self, timeout=None, _raise_exc=PY3): + """ + Wait for child process to terminate. Returns :attr:`returncode` + attribute. + + :keyword timeout: The floating point number of seconds to + wait. Under Python 2, this is a gevent extension, and + we simply return if it expires. Under Python 3, if + this time elapses without finishing the process, + :exc:`TimeoutExpired` is raised. + """ + return self._gevent_result_wait(timeout, _raise_exc) + + def send_signal(self, sig): + """Send a signal to the process + """ + # Skip signalling a process that we know has already died. + if self.returncode is None: + os.kill(self.pid, sig) + + def terminate(self): + """Terminate the process with SIGTERM + """ + self.send_signal(signal.SIGTERM) + + def kill(self): + """Kill the process with SIGKILL + """ + self.send_signal(signal.SIGKILL) + + +def write_and_close(fobj, data): + try: + if data: + fobj.write(data) + if hasattr(fobj, 'flush'): + # 3.6 started expecting flush to be called. + fobj.flush() + except (OSError, IOError) as ex: + if ex.errno != errno.EPIPE and ex.errno != errno.EINVAL: + raise + finally: + try: + fobj.close() + except EnvironmentError: + pass + +def _with_stdout_stderr(exc, stderr): + # Prior to Python 3.5, most exceptions didn't have stdout + # and stderr attributes and can't take the stderr attribute in their + # constructor + exc.stdout = exc.output + exc.stderr = stderr + return exc + +class CompletedProcess(object): + """ + A process that has finished running. + + This is returned by run(). + + Attributes: + - args: The list or str args passed to run(). + - returncode: The exit code of the process, negative for signals. + - stdout: The standard output (None if not captured). + - stderr: The standard error (None if not captured). + + .. versionadded:: 1.2a1 + This first appeared in Python 3.5 and is available to all + Python versions in gevent. + """ + def __init__(self, args, returncode, stdout=None, stderr=None): + self.args = args + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + def __repr__(self): + args = ['args={!r}'.format(self.args), + 'returncode={!r}'.format(self.returncode)] + if self.stdout is not None: + args.append('stdout={!r}'.format(self.stdout)) + if self.stderr is not None: + args.append('stderr={!r}'.format(self.stderr)) + return "{}({})".format(type(self).__name__, ', '.join(args)) + + def check_returncode(self): + """Raise CalledProcessError if the exit code is non-zero.""" + if self.returncode: + # pylint:disable=undefined-variable + raise _with_stdout_stderr(CalledProcessError(self.returncode, self.args, self.stdout), self.stderr) + + +def run(*popenargs, **kwargs): + """ + run(args, *, stdin=None, input=None, stdout=None, stderr=None, shell=False, timeout=None, check=False) -> CompletedProcess + + Run command with arguments and return a CompletedProcess instance. + + The returned instance will have attributes args, returncode, stdout and + stderr. By default, stdout and stderr are not captured, and those attributes + will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. + If check is True and the exit code was non-zero, it raises a + CalledProcessError. The CalledProcessError object will have the return code + in the returncode attribute, and output & stderr attributes if those streams + were captured. + + If timeout is given, and the process takes too long, a TimeoutExpired + exception will be raised. + + There is an optional argument "input", allowing you to + pass a string to the subprocess's stdin. If you use this argument + you may not also use the Popen constructor's "stdin" argument, as + it will be used internally. + The other arguments are the same as for the Popen constructor. + If universal_newlines=True is passed, the "input" argument must be a + string and stdout/stderr in the returned object will be strings rather than + bytes. + + .. versionadded:: 1.2a1 + This function first appeared in Python 3.5. It is available on all Python + versions gevent supports. + + .. versionchanged:: 1.3a2 + Add the ``capture_output`` argument from Python 3.7. It automatically sets + ``stdout`` and ``stderr`` to ``PIPE``. It is an error to pass either + of those arguments along with ``capture_output``. + """ + input = kwargs.pop('input', None) + timeout = kwargs.pop('timeout', None) + check = kwargs.pop('check', False) + capture_output = kwargs.pop('capture_output', False) + + if input is not None: + if 'stdin' in kwargs: + raise ValueError('stdin and input arguments may not both be used.') + kwargs['stdin'] = PIPE + + if capture_output: + if ('stdout' in kwargs) or ('stderr' in kwargs): + raise ValueError('stdout and stderr arguments may not be used ' + 'with capture_output.') + kwargs['stdout'] = PIPE + kwargs['stderr'] = PIPE + + with Popen(*popenargs, **kwargs) as process: + try: + stdout, stderr = process.communicate(input, timeout=timeout) + except TimeoutExpired: + process.kill() + stdout, stderr = process.communicate() + raise _with_stdout_stderr(TimeoutExpired(process.args, timeout, output=stdout), stderr) + except: + process.kill() + process.wait() + raise + retcode = process.poll() + if check and retcode: + # pylint:disable=undefined-variable + raise _with_stdout_stderr(CalledProcessError(retcode, process.args, stdout), stderr) + + return CompletedProcess(process.args, retcode, stdout, stderr) diff --git a/src/gevent/testing/__init__.py b/src/gevent/testing/__init__.py new file mode 100644 index 0000000..462d6f8 --- /dev/null +++ b/src/gevent/testing/__init__.py @@ -0,0 +1,137 @@ +# Copyright (c) 2008-2009 AG Projects +# Copyright 2018 gevent community +# Author: Denis Bilenko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +import unittest + +# pylint:disable=unused-import + +from .sysinfo import VERBOSE +from .sysinfo import WIN +from .sysinfo import LINUX +from .sysinfo import LIBUV +from .sysinfo import CFFI_BACKEND +from .sysinfo import DEBUG +from .sysinfo import RUN_LEAKCHECKS +from .sysinfo import RUN_COVERAGE + +from .sysinfo import PY2 +from .sysinfo import PY3 +from .sysinfo import PY34 +from .sysinfo import PY36 +from .sysinfo import PY37 + +from .sysinfo import PYPY +from .sysinfo import PYPY3 +from .sysinfo import CPYTHON + +from .sysinfo import PLATFORM_SPECIFIC_SUFFIXES +from .sysinfo import NON_APPLICABLE_SUFFIXES +from .sysinfo import SHARED_OBJECT_EXTENSION + +from .sysinfo import RUNNING_ON_TRAVIS +from .sysinfo import RUNNING_ON_APPVEYOR +from .sysinfo import RUNNING_ON_CI + +from .sysinfo import RESOLVER_NOT_SYSTEM +from .sysinfo import RESOLVER_DNSPYTHON +from .sysinfo import RESOLVER_ARES + +from .sysinfo import EXPECT_POOR_TIMER_RESOLUTION + +from .sysinfo import CONN_ABORTED_ERRORS + +from .skipping import skipOnWindows +from .skipping import skipOnAppVeyor +from .skipping import skipOnCI +from .skipping import skipOnPyPy3OnCI +from .skipping import skipOnPyPy +from .skipping import skipOnPyPyOnCI +from .skipping import skipOnPyPy3 +from .skipping import skipIf +from .skipping import skipUnless +from .skipping import skipOnLibev +from .skipping import skipOnLibuv +from .skipping import skipOnLibuvOnWin +from .skipping import skipOnLibuvOnCI +from .skipping import skipOnLibuvOnCIOnPyPy +from .skipping import skipOnLibuvOnPyPyOnWin +from .skipping import skipOnPurePython +from .skipping import skipWithCExtensions +from .skipping import skipOnLibuvOnTravisOnCPython27 +from .skipping import skipOnPy37 + +from .exception import ExpectedException + + +from .leakcheck import ignores_leakcheck + + + +from .params import LARGE_TIMEOUT + +from .params import DEFAULT_LOCAL_HOST_ADDR +from .params import DEFAULT_LOCAL_HOST_ADDR6 +from .params import DEFAULT_BIND_ADDR + + +from .params import DEFAULT_SOCKET_TIMEOUT +from .params import DEFAULT_XPC_SOCKET_TIMEOUT + +main = unittest.main + +from .hub import QuietHub + +import gevent.hub +gevent.hub.set_default_hub_class(QuietHub) + + + +from .sockets import bind_and_listen +from .sockets import tcp_listener + +from .openfiles import get_number_open_files +from .openfiles import get_open_files + +from .testcase import TestCase + +from .modules import walk_modules + +BaseTestCase = unittest.TestCase + +from .flaky import reraiseFlakyTestTimeout +from .flaky import reraiseFlakyTestRaceCondition +from .flaky import reraises_flaky_timeout +from .flaky import reraises_flaky_race_condition + +def gc_collect_if_needed(): + "Collect garbage if necessary for destructors to run" + import gc + if PYPY: # pragma: no cover + gc.collect() + +try: + from unittest import mock +except ImportError: # Python 2 + import mock + +mock = mock diff --git a/src/gevent/testing/coveragesite/sitecustomize.py b/src/gevent/testing/coveragesite/sitecustomize.py new file mode 100644 index 0000000..097dcec --- /dev/null +++ b/src/gevent/testing/coveragesite/sitecustomize.py @@ -0,0 +1,17 @@ +# When testrunner.py is invoked with --coverage, it puts this first +# on the path as per https://coverage.readthedocs.io/en/coverage-4.0b3/subprocess.html. +# Note that this disables other sitecustomize.py files. +import coverage +try: + coverage.process_startup() +except coverage.CoverageException as e: + if str(e) == "Can't support concurrency=greenlet with PyTracer, only threads are supported": + pass + else: + import traceback + traceback.print_exc() + raise +except: + import traceback + traceback.print_exc() + raise diff --git a/src/gevent/testing/errorhandler.py b/src/gevent/testing/errorhandler.py new file mode 100644 index 0000000..77302dc --- /dev/null +++ b/src/gevent/testing/errorhandler.py @@ -0,0 +1,53 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +from functools import wraps + + +def wrap_error_fatal(method): + import gevent + system_error = gevent.get_hub().SYSTEM_ERROR + + @wraps(method) + def wrapper(self, *args, **kwargs): + # XXX should also be able to do gevent.SYSTEM_ERROR = object + # which is a global default to all hubs + + gevent.get_hub().SYSTEM_ERROR = object + try: + return method(self, *args, **kwargs) + finally: + gevent.get_hub().SYSTEM_ERROR = system_error + return wrapper + + +def wrap_restore_handle_error(method): + import gevent + old = gevent.get_hub().handle_error + + @wraps(method) + def wrapper(self, *args, **kwargs): + try: + return method(self, *args, **kwargs) + finally: + gevent.get_hub().handle_error = old + if self.peek_error()[0] is not None: + gevent.getcurrent().throw(*self.peek_error()[1:]) + return wrapper diff --git a/src/gevent/testing/exception.py b/src/gevent/testing/exception.py new file mode 100644 index 0000000..baa9f96 --- /dev/null +++ b/src/gevent/testing/exception.py @@ -0,0 +1,23 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +class ExpectedException(Exception): + """An exception whose traceback should be ignored by the hub""" diff --git a/src/gevent/testing/flaky.py b/src/gevent/testing/flaky.py new file mode 100644 index 0000000..98f9fe8 --- /dev/null +++ b/src/gevent/testing/flaky.py @@ -0,0 +1,114 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +import sys +import functools +import unittest + +from . import sysinfo +from . import six + +class FlakyAssertionError(AssertionError): + "Re-raised so that we know it's a known-flaky test." + +# The next exceptions allow us to raise them in a highly +# greppable way so that we can debug them later. + +class FlakyTest(unittest.SkipTest): + """ + A unittest exception that causes the test to be skipped when raised. + + Use this carefully, it is a code smell and indicates an undebugged problem. + """ + +class FlakyTestRaceCondition(FlakyTest): + """ + Use this when the flaky test is definitely caused by a race condition. + """ + +class FlakyTestTimeout(FlakyTest): + """ + Use this when the flaky test is definitely caused by an + unexpected timeout. + """ + +class FlakyTestCrashes(FlakyTest): + """ + Use this when the test sometimes crashes. + """ + +def reraiseFlakyTestRaceCondition(): + six.reraise(FlakyAssertionError, + FlakyAssertionError(sys.exc_info()[1]), + sys.exc_info()[2]) + +reraiseFlakyTestTimeout = reraiseFlakyTestRaceCondition +reraiseFlakyTestRaceConditionLibuv = reraiseFlakyTestRaceCondition +reraiseFlakyTestTimeoutLibuv = reraiseFlakyTestRaceCondition + +if sysinfo.RUNNING_ON_CI or (sysinfo.PYPY and sysinfo.WIN): + # pylint: disable=function-redefined + def reraiseFlakyTestRaceCondition(): + # Getting stack traces is incredibly expensive + # in pypy on win, at least in test virtual machines. + # It can take minutes. The traceback consistently looks like + # the following when interrupted: + + # dump_stacks -> traceback.format_stack + # -> traceback.extract_stack -> linecache.checkcache + # -> os.stat -> _structseq.structseq_new + + # Moreover, without overriding __repr__ or __str__, + # the msg doesn't get printed like we would want (its basically + # unreadable, all printed on one line). So skip that. + + #msg = '\n'.join(dump_stacks()) + msg = str(sys.exc_info()[1]) + six.reraise(FlakyTestRaceCondition, + FlakyTestRaceCondition(msg), + sys.exc_info()[2]) + + def reraiseFlakyTestTimeout(): + msg = str(sys.exc_info()[1]) + six.reraise(FlakyTestTimeout, + FlakyTestTimeout(msg), + sys.exc_info()[2]) + + if sysinfo.LIBUV: + reraiseFlakyTestRaceConditionLibuv = reraiseFlakyTestRaceCondition + reraiseFlakyTestTimeoutLibuv = reraiseFlakyTestTimeout + + +def reraises_flaky_timeout(exc_kind=AssertionError, _func=reraiseFlakyTestTimeout): + + def wrapper(f): + @functools.wraps(f) + def m(*args): + try: + f(*args) + except exc_kind: + _func() + return m + + return wrapper + +def reraises_flaky_race_condition(exc_kind=AssertionError): + return reraises_flaky_timeout(exc_kind, _func=reraiseFlakyTestRaceCondition) diff --git a/src/gevent/testing/hub.py b/src/gevent/testing/hub.py new file mode 100644 index 0000000..645c1d0 --- /dev/null +++ b/src/gevent/testing/hub.py @@ -0,0 +1,35 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + + +from gevent.hub import Hub + +from .exception import ExpectedException + +class QuietHub(Hub): + + EXPECTED_TEST_ERROR = (ExpectedException,) + + def handle_error(self, context, type, value, tb): + if issubclass(type, self.EXPECTED_TEST_ERROR): + # Don't print these to cut down on the noise in the test logs + return + return Hub.handle_error(self, context, type, value, tb) diff --git a/src/gevent/testing/leakcheck.py b/src/gevent/testing/leakcheck.py new file mode 100644 index 0000000..5e5b7f0 --- /dev/null +++ b/src/gevent/testing/leakcheck.py @@ -0,0 +1,206 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import print_function + +import sys +import gc +import types +from functools import wraps +import unittest + +import objgraph + +import gevent +import gevent.core + + +def ignores_leakcheck(func): + """ + Ignore the given object during leakchecks. + + Can be applied to a method, in which case the method will run, but + will not be subject to leak checks. + + If applied to a class, the entire class will be skipped during leakchecks. This + is intended to be used for classes that are very slow and cause problems such as + test timeouts; typically it will be used for classes that are subclasses of a base + class and specify variants of behaviour (such as pool sizes). + """ + func.ignore_leakcheck = True + return func + +class _RefCountChecker(object): + + # Some builtin things that we ignore + IGNORED_TYPES = (tuple, dict, types.FrameType, types.TracebackType) + try: + CALLBACK_KIND = gevent.core.callback + except AttributeError: + # Must be using FFI. + from gevent._ffi.callback import callback as CALLBACK_KIND + + + def __init__(self, testcase, function): + self.testcase = testcase + self.function = function + self.deltas = [] + self.peak_stats = {} + + # The very first time we are called, we have already been + # self.setUp() by the test runner, so we don't need to do it again. + self.needs_setUp = False + + def _ignore_object_p(self, obj): + if ( + obj is self + or obj in self.__dict__.values() + or obj == self._ignore_object_p # pylint:disable=comparison-with-callable + ): + return False + kind = type(obj) + if kind in self.IGNORED_TYPES: + return False + if kind is self.CALLBACK_KIND and obj.callback is None and obj.args is None: + # these represent callbacks that have been stopped, but + # the event loop hasn't cycled around to run them. The only + # known cause of this is killing greenlets before they get a chance + # to run for the first time. + return False + return True + + + def _growth(self): + return objgraph.growth(limit=None, peak_stats=self.peak_stats, filter=self._ignore_object_p) + + def _report_diff(self, growth): + if not growth: + return "" + + lines = [] + width = max(len(name) for name, _, _ in growth) + for name, count, delta in growth: + lines.append('%-*s%9d %+9d' % (width, name, count, delta)) + + diff = '\n'.join(lines) + return diff + + + def _run_test(self, args, kwargs): + gc_enabled = gc.isenabled() + gc.disable() + + if self.needs_setUp: + self.testcase.setUp() + self.testcase.skipTearDown = False + try: + self.function(self.testcase, *args, **kwargs) + finally: + self.testcase.tearDown() + self.testcase.skipTearDown = True + self.needs_setUp = True + if gc_enabled: + gc.enable() + + def _growth_after(self): + # Grab post snapshot + if 'urlparse' in sys.modules: + sys.modules['urlparse'].clear_cache() + if 'urllib.parse' in sys.modules: + sys.modules['urllib.parse'].clear_cache() + + return self._growth() + + def _check_deltas(self, growth): + # Return false when we have decided there is no leak, + # true if we should keep looping, raises an assertion + # if we have decided there is a leak. + + deltas = self.deltas + if not deltas: + # We haven't run yet, no data, keep looping + return True + + if gc.garbage: + raise AssertionError("Generated uncollectable garbage %r" % (gc.garbage,)) + + + # the following configurations are classified as "no leak" + # [0, 0] + # [x, 0, 0] + # [... a, b, c, d] where a+b+c+d = 0 + # + # the following configurations are classified as "leak" + # [... z, z, z] where z > 0 + + if deltas[-2:] == [0, 0] and len(deltas) in (2, 3): + return False + + if deltas[-3:] == [0, 0, 0]: + return False + + if len(deltas) >= 4 and sum(deltas[-4:]) == 0: + return False + + if len(deltas) >= 3 and deltas[-1] > 0 and deltas[-1] == deltas[-2] and deltas[-2] == deltas[-3]: + diff = self._report_diff(growth) + raise AssertionError('refcount increased by %r\n%s' % (deltas, diff)) + + # OK, we don't know for sure yet. Let's search for more + if sum(deltas[-3:]) <= 0 or sum(deltas[-4:]) <= 0 or deltas[-4:].count(0) >= 2: + # this is suspicious, so give a few more runs + limit = 11 + else: + limit = 7 + if len(deltas) >= limit: + raise AssertionError('refcount increased by %r\n%s' + % (deltas, + self._report_diff(growth))) + + # We couldn't decide yet, keep going + return True + + def __call__(self, args, kwargs): + for _ in range(3): + gc.collect() + + # Capture state before; the incremental will be + # updated by each call to _growth_after + growth = self._growth() + + while self._check_deltas(growth): + self._run_test(args, kwargs) + + growth = self._growth_after() + + self.deltas.append(sum((stat[2] for stat in growth))) + + +def wrap_refcount(method): + if getattr(method, 'ignore_leakcheck', False): + return method + + + @wraps(method) + def wrapper(self, *args, **kwargs): # pylint:disable=too-many-branches + if getattr(self, 'ignore_leakcheck', False): + raise unittest.SkipTest("This class ignored during leakchecks") + return _RefCountChecker(self, method)(args, kwargs) + + return wrapper diff --git a/src/gevent/testing/modules.py b/src/gevent/testing/modules.py new file mode 100644 index 0000000..2fb04d5 --- /dev/null +++ b/src/gevent/testing/modules.py @@ -0,0 +1,85 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +import importlib +import os.path +import warnings + +import gevent + +from . import sysinfo + + +OPTIONAL_MODULES = [ + 'gevent.resolver_ares', + 'gevent.resolver.ares', + 'gevent.libev', + 'gevent.libev.watcher', +] + + +def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False): + # pylint:disable=too-many-branches + if sysinfo.PYPY: + include_so = False + if basedir is None: + basedir = os.path.dirname(gevent.__file__) + if modpath is None: + modpath = 'gevent.' + else: + if modpath is None: + modpath = '' + for fn in sorted(os.listdir(basedir)): + path = os.path.join(basedir, fn) + if os.path.isdir(path): + if not recursive: + continue + if fn in ['testing', 'tests']: + continue + pkg_init = os.path.join(path, '__init__.py') + if os.path.exists(pkg_init): + yield pkg_init, modpath + fn + for p, m in walk_modules(path, modpath + fn + "."): + yield p, m + continue + if fn.endswith('.py'): + x = fn[:-3] + if x.endswith('_d'): + x = x[:-2] + if x in ['__init__', 'core', 'ares', '_util', '_semaphore', + 'corecffi', '_corecffi', '_corecffi_build']: + continue + modname = modpath + x + if modname in OPTIONAL_MODULES: + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + importlib.import_module(modname) + except ImportError: + continue + yield path, modname + elif include_so and fn.endswith(sysinfo.SHARED_OBJECT_EXTENSION): + if '.pypy-' in fn: + continue + if fn.endswith('_d.so'): + yield path, modpath + fn[:-5] + else: + yield path, modpath + fn[:-3] diff --git a/src/gevent/testing/monkey_test.py b/src/gevent/testing/monkey_test.py new file mode 100644 index 0000000..f807673 --- /dev/null +++ b/src/gevent/testing/monkey_test.py @@ -0,0 +1,75 @@ +import sys +import os + + +kwargs = {} + +if sys.argv[1] == '--Event': + kwargs['Event'] = True + del sys.argv[1] +else: + kwargs['Event'] = False + +test_filename = sys.argv[1] +del sys.argv[1] + +print('Running with patch_all(%s): %s' % (','.join('%s=%r' % x for x in kwargs.items()), test_filename)) + +from gevent import monkey +monkey.patch_all(**kwargs) + +from .sysinfo import RUNNING_ON_APPVEYOR +from .sysinfo import PY37 +from .patched_tests_setup import disable_tests_in_source +try: + from test import support +except ImportError: + from test import test_support as support +support.is_resource_enabled = lambda *args: True +del support.use_resources +if RUNNING_ON_APPVEYOR and PY37: + # 3.7 added a stricter mode for thread cleanup. + # It appears to be unstable on Windows (at least appveyor) + # and test_socket.py constantly fails with an extra thread + # on some random test. We disable it entirely. + import contextlib + @contextlib.contextmanager + def wait_threads_exit(timeout=None): # pylint:disable=unused-argument + yield + support.wait_threads_exit = wait_threads_exit + + +__file__ = os.path.join(os.getcwd(), test_filename) + +test_name = os.path.splitext(test_filename)[0] + +# It's important that the `module_source` be a native +# string. Passing unicode to `compile` on Python 2 can +# do bad things: it conflicts with a 'coding:' directive, +# and it can cause some TypeError with string literals +if sys.version_info[0] >= 3: + module_file = open(test_filename, encoding='utf-8') +else: + module_file = open(test_filename) +with module_file: + module_source = module_file.read() +module_source = disable_tests_in_source(module_source, test_name) + +# We write the module source to a file so that tracebacks +# show correctly, since disabling the tests changes line +# numbers. However, note that __file__ must still point to the +# real location so that data files can be found. +# See https://github.com/gevent/gevent/issues/1306 +import tempfile +temp_handle, temp_path = tempfile.mkstemp(prefix=test_name, suffix='.py', text=True) +os.write(temp_handle, + module_source.encode('utf-8') if not isinstance(module_source, bytes) else module_source) +os.close(temp_handle) +try: + module_code = compile(module_source, + temp_path, + 'exec', + dont_inherit=True) + exec(module_code, globals()) +finally: + os.remove(temp_path) diff --git a/src/gevent/testing/openfiles.py b/src/gevent/testing/openfiles.py new file mode 100644 index 0000000..fe916e7 --- /dev/null +++ b/src/gevent/testing/openfiles.py @@ -0,0 +1,125 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +import os +import unittest +import re + +from . import sysinfo + +# Linux/OS X/BSD platforms can implement this by calling out to lsof + + +if sysinfo.WIN: + def _run_lsof(): + raise unittest.SkipTest("lsof not expected on Windows") +else: + def _run_lsof(): + import tempfile + pid = os.getpid() + fd, tmpname = tempfile.mkstemp('get_open_files') + os.close(fd) + lsof_command = 'lsof -p %s > %s' % (pid, tmpname) + if os.system(lsof_command): + # XXX: This prints to the console an annoying message: 'lsof is not recognized' + raise unittest.SkipTest("lsof failed") + with open(tmpname) as fobj: + data = fobj.read().strip() + os.remove(tmpname) + return data + +def default_get_open_files(pipes=False): + data = _run_lsof() + results = {} + for line in data.split('\n'): + line = line.strip() + if not line or line.startswith("COMMAND"): + # Skip header and blank lines + continue + split = re.split(r'\s+', line) + _command, _pid, _user, fd = split[:4] + # Pipes (on OS X, at least) get an fd like "3" while normal files get an fd like "1u" + if fd[:-1].isdigit() or fd.isdigit(): + if not pipes and fd[-1].isdigit(): + continue + fd = int(fd[:-1]) if not fd[-1].isdigit() else int(fd) + if fd in results: + params = (fd, line, split, results.get(fd), data) + raise AssertionError('error when parsing lsof output: duplicate fd=%r\nline=%r\nsplit=%r\nprevious=%r\ndata:\n%s' % params) + results[fd] = line + if not results: + raise AssertionError('failed to parse lsof:\n%s' % (data, )) + results['data'] = data + return results + +def default_get_number_open_files(): + if os.path.exists('/proc/'): + # Linux only + fd_directory = '/proc/%d/fd' % os.getpid() + return len(os.listdir(fd_directory)) + + try: + return len(get_open_files(pipes=True)) - 1 + except (OSError, AssertionError, unittest.SkipTest): + return 0 + +lsof_get_open_files = default_get_open_files + +try: + # psutil import subprocess which on Python 3 imports selectors. + # This can expose issues with monkey-patching. + import psutil +except ImportError: + get_open_files = default_get_open_files + get_number_open_files = default_get_number_open_files +else: + # If psutil is available (it is cross-platform) use that. + # It is *much* faster than shelling out to lsof each time + # (Running 14 tests takes 3.964s with lsof and 0.046 with psutil) + # However, it still doesn't completely solve the issue on Windows: fds are reported + # as -1 there, so we can't fully check those. + + def get_open_files(): + """ + Return a list of popenfile and pconn objects. + + Note that other than `fd`, they have different attributes. + + .. important:: If you want to find open sockets, on Windows + and linux, it is important that the socket at least be listening + (socket.listen(1)). Unlike the lsof implementation, this will only + return sockets in a state like that. + """ + results = dict() + process = psutil.Process() + results['data'] = process.open_files() + process.connections('all') + for x in results['data']: + results[x.fd] = x + results['data'] += ['From psutil', process] + return results + + def get_number_open_files(): + process = psutil.Process() + try: + return process.num_fds() + except AttributeError: + # num_fds is unix only. Is num_handles close enough on Windows? + return 0 diff --git a/src/gevent/testing/params.py b/src/gevent/testing/params.py new file mode 100644 index 0000000..efa5416 --- /dev/null +++ b/src/gevent/testing/params.py @@ -0,0 +1,83 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + + +from .sysinfo import PY3 +from .sysinfo import PYPY +from .sysinfo import WIN +from .sysinfo import LIBUV +from .sysinfo import OSX + +from .sysinfo import RUNNING_ON_TRAVIS +from .sysinfo import RUNNING_ON_APPVEYOR +from .sysinfo import EXPECT_POOR_TIMER_RESOLUTION +from .sysinfo import RESOLVER_ARES + + +# Travis is slow and overloaded; Appveyor used to be faster, but +# as of Dec 2015 it's almost always slower and/or has much worse timer +# resolution +CI_TIMEOUT = 15 +if (PY3 and PYPY) or (PYPY and WIN and LIBUV): + # pypy3 is very slow right now, + # as is PyPy2 on windows (which only has libuv) + CI_TIMEOUT = 20 +if PYPY and LIBUV: + # slow and flaky timeouts + LOCAL_TIMEOUT = CI_TIMEOUT +else: + LOCAL_TIMEOUT = 1 + +LARGE_TIMEOUT = max(LOCAL_TIMEOUT, CI_TIMEOUT) + +DEFAULT_LOCAL_HOST_ADDR = 'localhost' +DEFAULT_LOCAL_HOST_ADDR6 = DEFAULT_LOCAL_HOST_ADDR +DEFAULT_BIND_ADDR = '' + +if RUNNING_ON_TRAVIS: + # As of November 2017 (probably Sept or Oct), after a + # Travis upgrade, using "localhost" no longer works, + # producing 'OSError: [Errno 99] Cannot assign + # requested address'. This is apparently something to do with + # docker containers. Sigh. + DEFAULT_LOCAL_HOST_ADDR = '127.0.0.1' + DEFAULT_LOCAL_HOST_ADDR6 = '::1' + # Likewise, binding to '' appears to work, but it cannot be + # connected to with the same error. + DEFAULT_BIND_ADDR = '127.0.0.1' + +if RUNNING_ON_APPVEYOR: + DEFAULT_BIND_ADDR = '127.0.0.1' + DEFAULT_LOCAL_HOST_ADDR = '127.0.0.1' + + +if RESOLVER_ARES and OSX: + # Ares likes to raise "malformed domain name" on '', at least + # on OS X + DEFAULT_BIND_ADDR = '127.0.0.1' + +DEFAULT_CONNECT = DEFAULT_LOCAL_HOST_ADDR +DEFAULT_BIND_ADDR_TUPLE = (DEFAULT_BIND_ADDR, 0) + +# For in-process sockets +DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0 + +# For cross-process sockets +DEFAULT_XPC_SOCKET_TIMEOUT = 2.0 if not EXPECT_POOR_TIMER_RESOLUTION else 4.0 diff --git a/src/gevent/testing/patched_tests_setup.py b/src/gevent/testing/patched_tests_setup.py new file mode 100644 index 0000000..39c881d --- /dev/null +++ b/src/gevent/testing/patched_tests_setup.py @@ -0,0 +1,1139 @@ +# pylint:disable=missing-docstring,invalid-name,too-many-lines +from __future__ import print_function, absolute_import, division + +import collections +import contextlib +import functools +import sys +import os +# At least on 3.6+, importing platform +# imports subprocess, which imports selectors. That +# can expose issues with monkey patching. We don't need it +# though. +# import platform +import re + +from .sysinfo import RUNNING_ON_APPVEYOR as APPVEYOR +from .sysinfo import RUNNING_ON_TRAVIS as TRAVIS +from .sysinfo import RESOLVER_NOT_SYSTEM as ARES +from .sysinfo import RUN_COVERAGE + + +from .sysinfo import PYPY +from .sysinfo import PYPY3 +from .sysinfo import PY3 +from .sysinfo import PY2 +from .sysinfo import PY34 +from .sysinfo import PY35 +from .sysinfo import PY36 +from .sysinfo import PY37 + +from .sysinfo import WIN +from .sysinfo import OSX + +from .sysinfo import LIBUV +from .sysinfo import CFFI_BACKEND + +from . import flaky + +CPYTHON = not PYPY + +# By default, test cases are expected to switch and emit warnings if there was none +# If a test is found in this list, it's expected not to switch. +no_switch_tests = '''test_patched_select.SelectTestCase.test_error_conditions +test_patched_ftplib.*.test_all_errors +test_patched_ftplib.*.test_getwelcome +test_patched_ftplib.*.test_sanitize +test_patched_ftplib.*.test_set_pasv +#test_patched_ftplib.TestIPv6Environment.test_af +test_patched_socket.TestExceptions.testExceptionTree +test_patched_socket.Urllib2FileobjectTest.testClose +test_patched_socket.TestLinuxAbstractNamespace.testLinuxAbstractNamespace +test_patched_socket.TestLinuxAbstractNamespace.testMaxName +test_patched_socket.TestLinuxAbstractNamespace.testNameOverflow +test_patched_socket.FileObjectInterruptedTestCase.* +test_patched_urllib.* +test_patched_asyncore.HelperFunctionTests.* +test_patched_httplib.BasicTest.* +test_patched_httplib.HTTPSTimeoutTest.test_attributes +test_patched_httplib.HeaderTests.* +test_patched_httplib.OfflineTest.* +test_patched_httplib.HTTPSTimeoutTest.test_host_port +test_patched_httplib.SourceAddressTest.testHTTPSConnectionSourceAddress +test_patched_select.SelectTestCase.test_error_conditions +test_patched_smtplib.NonConnectingTests.* +test_patched_urllib2net.OtherNetworkTests.* +test_patched_wsgiref.* +test_patched_subprocess.HelperFunctionTests.* +''' + +ignore_switch_tests = ''' +test_patched_socket.GeneralModuleTests.* +test_patched_httpservers.BaseHTTPRequestHandlerTestCase.* +test_patched_queue.* +test_patched_signal.SiginterruptTest.* +test_patched_urllib2.* +test_patched_ssl.* +test_patched_signal.BasicSignalTests.* +test_patched_threading_local.* +test_patched_threading.* +''' + + +def make_re(tests): + tests = [x.strip().replace(r'\.', r'\\.').replace('*', '.*?') + for x in tests.split('\n') if x.strip()] + return re.compile('^%s$' % '|'.join(tests)) + + +no_switch_tests = make_re(no_switch_tests) +ignore_switch_tests = make_re(ignore_switch_tests) + + +def get_switch_expected(fullname): + """ + >>> get_switch_expected('test_patched_select.SelectTestCase.test_error_conditions') + False + >>> get_switch_expected('test_patched_socket.GeneralModuleTests.testCrucialConstants') + False + >>> get_switch_expected('test_patched_socket.SomeOtherTest.testHello') + True + >>> get_switch_expected("test_patched_httplib.BasicTest.test_bad_status_repr") + False + """ + # certain pylint versions mistype the globals as + # str, not re. + # pylint:disable=no-member + if ignore_switch_tests.match(fullname) is not None: + return None + if no_switch_tests.match(fullname) is not None: + return False + return True + + +disabled_tests = [ + # The server side takes awhile to shut down + 'test_httplib.HTTPSTest.test_local_bad_hostname', + + 'test_threading.ThreadTests.test_PyThreadState_SetAsyncExc', + # uses some internal C API of threads not available when threads are emulated with greenlets + + 'test_threading.ThreadTests.test_join_nondaemon_on_shutdown', + # asserts that repr(sleep) is '' + + 'test_urllib2net.TimeoutTest.test_ftp_no_timeout', + 'test_urllib2net.TimeoutTest.test_ftp_timeout', + 'test_urllib2net.TimeoutTest.test_http_no_timeout', + 'test_urllib2net.TimeoutTest.test_http_timeout', + # accesses _sock.gettimeout() which is always in non-blocking mode + + 'test_urllib2net.OtherNetworkTests.test_ftp', + # too slow + + 'test_urllib2net.OtherNetworkTests.test_urlwithfrag', + # fails dues to some changes on python.org + + 'test_urllib2net.OtherNetworkTests.test_sites_no_connection_close', + # flaky + + 'test_socket.UDPTimeoutTest.testUDPTimeout', + # has a bug which makes it fail with error: (107, 'Transport endpoint is not connected') + # (it creates a TCP socket, not UDP) + + 'test_socket.GeneralModuleTests.testRefCountGetNameInfo', + # fails with "socket.getnameinfo loses a reference" while the reference is only "lost" + # because it is referenced by the traceback - any Python function would lose a reference like that. + # the original getnameinfo does not "lose" it because it's in C. + + 'test_socket.NetworkConnectionNoServer.test_create_connection_timeout', + # replaces socket.socket with MockSocket and then calls create_connection. + # this unfortunately does not work with monkey patching, because gevent.socket.create_connection + # is bound to gevent.socket.socket and updating socket.socket does not affect it. + # this issues also manifests itself when not monkey patching DNS: http://code.google.com/p/gevent/issues/detail?id=54 + # create_connection still uses gevent.socket.getaddrinfo while it should be using socket.getaddrinfo + + 'test_asyncore.BaseTestAPI.test_handle_expt', + # sends some OOB data and expect it to be detected as such; gevent.select.select does not support that + + # This one likes to check its own filename, but we rewrite + # the file to a temp location during patching. + 'test_asyncore.HelperFunctionTests.test_compact_traceback', + + 'test_signal.WakeupSignalTests.test_wakeup_fd_early', + # expects time.sleep() to return prematurely in case of a signal; + # gevent.sleep() is better than that and does not get interrupted (unless signal handler raises an error) + + 'test_signal.WakeupSignalTests.test_wakeup_fd_during', + # expects select.select() to raise select.error(EINTR'interrupted system call') + # gevent.select.select() does not get interrupted (unless signal handler raises an error) + # maybe it should? + + 'test_signal.SiginterruptTest.test_without_siginterrupt', + 'test_signal.SiginterruptTest.test_siginterrupt_on', + # these rely on os.read raising EINTR which never happens with gevent.os.read + + 'test_subprocess.ProcessTestCase.test_leak_fast_process_del_killed', + 'test_subprocess.ProcessTestCase.test_zombie_fast_process_del', + # relies on subprocess._active which we don't use + + # Very slow, tries to open lots and lots of subprocess and files, + # tends to timeout on CI. + 'test_subprocess.ProcessTestCase.test_no_leaking', + + # This test is also very slow, and has been timing out on Travis + # since November of 2016 on Python 3, but now also seen on Python 2/Pypy. + 'test_subprocess.ProcessTestCase.test_leaking_fds_on_error', + + 'test_ssl.ThreadedTests.test_default_ciphers', + 'test_ssl.ThreadedTests.test_empty_cert', + 'test_ssl.ThreadedTests.test_malformed_cert', + 'test_ssl.ThreadedTests.test_malformed_key', + 'test_ssl.NetworkedTests.test_non_blocking_connect_ex', + # XXX needs investigating + + 'test_ssl.NetworkedTests.test_algorithms', + # The host this wants to use, sha256.tbs-internet.com, is not resolvable + # right now (2015-10-10), and we need to get Windows wheels + + # This started timing out randomly on Travis in oct/nov 2018. It appears + # to be something with random number generation taking too long. + 'test_ssl.BasicSocketTests.test_random_fork', + + # Relies on the repr of objects (Py3) + 'test_ssl.BasicSocketTests.test_dealloc_warn', + + 'test_urllib2.HandlerTests.test_cookie_redirect', + # this uses cookielib which we don't care about + + 'test_thread.ThreadRunningTests.test__count', + 'test_thread.TestForkInThread.test_forkinthread', + # XXX needs investigating + + 'test_subprocess.POSIXProcessTestCase.test_preexec_errpipe_does_not_double_close_pipes', + # Does not exist in the test suite until 2.7.4+. Subclasses Popen, and overrides + # _execute_child. But our version has a different parameter list than the + # version that comes with PyPy/CPython, so fails with a TypeError. +] + +if 'thread' in os.getenv('GEVENT_FILE', ''): + disabled_tests += [ + 'test_subprocess.ProcessTestCase.test_double_close_on_error' + # Fails with "OSError: 9 invalid file descriptor"; expect GC/lifetime issues + ] + + +if LIBUV: + # epoll appears to work with these just fine in some cases; + # kqueue (at least on OS X, the only tested kqueue system) + # never does (failing with abort()) + # (epoll on Raspbian 8.0/Debian Jessie/Linux 4.1.20 works; + # on a VirtualBox image of Ubuntu 15.10/Linux 4.2.0 both tests fail; + # Travis CI Ubuntu 12.04 precise/Linux 3.13 causes one of these tests to hang forever) + # XXX: Retry this with libuv 1.12+ + disabled_tests += [ + # A 2.7 test. Tries to fork, and libuv cannot fork + 'test_signal.InterProcessSignalTests.test_main', + # Likewise, a forking problem + 'test_signal.SiginterruptTest.test_siginterrupt_off', + ] + + if PY2: + + if TRAVIS: + + if CPYTHON: + + disabled_tests += [ + # This appears to crash the process, for some reason, + # but only on CPython 2.7.14 on Travis. Cannot reproduce in + # 2.7.14 on macOS or 2.7.12 in local Ubuntu 16.04 + 'test_subprocess.POSIXProcessTestCase.test_close_fd_0', + 'test_subprocess.POSIXProcessTestCase.test_close_fds_0_1', + 'test_subprocess.POSIXProcessTestCase.test_close_fds_0_2', + ] + + if PYPY: + disabled_tests += [ + # This seems to crash the interpreter. I cannot reproduce + # on macOS or local Linux VM. + # See https://travis-ci.org/gevent/gevent/jobs/348661604#L709 + 'test_smtplib.TooLongLineTests.testLineTooLong', + ] + if ARES: + + disabled_tests += [ + # This can timeout with a socket timeout in ssl.wrap_socket(c) + # on Travis. I can't reproduce locally. + 'test_ssl.ThreadedTests.test_handshake_timeout', + ] + + if PY3: + + disabled_tests += [ + # This test wants to pass an arbitrary fileno + # to a socket and do things with it. libuv doesn't like this, + # it raises EPERM. It is disabled on windows already. + # It depends on whether we had a fd already open and multiplexed with + 'test_socket.GeneralModuleTests.test_unknown_socket_family_repr', + # And yes, there's a typo in some versions. + 'test_socket.GeneralModuleTests.test_uknown_socket_family_repr', + ] + + if PY37: + + disabled_tests += [ + # This test sometimes fails at line 358. It's apparently + # extremely sensitive to timing. + 'test_selectors.PollSelectorTestCase.test_timeout', + ] + + if OSX: + disabled_tests += [ + # XXX: Starting when we upgraded from libuv 1.18.0 + # to 1.19.2, this sometimes (usually) started having + # a series of calls ('select.poll(0)', 'select.poll(-1)') + # take longer than the allowed 0.5 seconds. Debugging showed that + # it was the second call that took longer, for no apparent reason. + # There doesn't seem to be a change in the source code to libuv that + # would affect this. + # XXX-XXX: This actually disables too many tests :( + 'test_selectors.PollSelectorTestCase.test_timeout', + ] + + if RUN_COVERAGE: + + disabled_tests += [ + # Starting with #1145 this test (actually + # TestTLS_FTPClassMixin) becomes sensitive to timings + # under coverage. + 'test_ftplib.TestFTPClass.test_storlines', + ] + + + if sys.platform.startswith('linux'): + disabled_tests += [ + # crashes with EPERM, which aborts the epoll loop, even + # though it was allowed in in the first place. + 'test_asyncore.FileWrapperTest.test_dispatcher', + ] + + + + if WIN and PY2: + # From PyPy2-v5.9.0 and CPython 2.7.14, using its version of tests, + # which do work on darwin (and possibly linux?) + # I can't produce them in a local VM running Windows 10 + # and the same pypy version. + disabled_tests += [ + # These, which use asyncore, fail with + # 'NoneType is not iterable' on 'conn, addr = self.accept()' + # That returns None when the underlying socket raises + # EWOULDBLOCK, which it will do because it's set to non-blocking + # both by gevent and by libuv (at the level below python's knowledge) + # I can *usually* reproduce these locally; it seems to be some sort + # of race condition. + 'test_ftplib.TestFTPClass.test_acct', + 'test_ftplib.TestFTPClass.test_all_errors', + 'test_ftplib.TestFTPClass.test_cwd', + 'test_ftplib.TestFTPClass.test_delete', + 'test_ftplib.TestFTPClass.test_dir', + 'test_ftplib.TestFTPClass.test_exceptions', + 'test_ftplib.TestFTPClass.test_getwelcome', + 'test_ftplib.TestFTPClass.test_line_too_long', + 'test_ftplib.TestFTPClass.test_login', + 'test_ftplib.TestFTPClass.test_makepasv', + 'test_ftplib.TestFTPClass.test_mkd', + 'test_ftplib.TestFTPClass.test_nlst', + 'test_ftplib.TestFTPClass.test_pwd', + 'test_ftplib.TestFTPClass.test_quit', + 'test_ftplib.TestFTPClass.test_makepasv', + 'test_ftplib.TestFTPClass.test_rename', + 'test_ftplib.TestFTPClass.test_retrbinary', + 'test_ftplib.TestFTPClass.test_retrbinary_rest', + 'test_ftplib.TestFTPClass.test_retrlines', + 'test_ftplib.TestFTPClass.test_retrlines_too_long', + 'test_ftplib.TestFTPClass.test_rmd', + 'test_ftplib.TestFTPClass.test_sanitize', + 'test_ftplib.TestFTPClass.test_set_pasv', + 'test_ftplib.TestFTPClass.test_size', + 'test_ftplib.TestFTPClass.test_storbinary', + 'test_ftplib.TestFTPClass.test_storbinary_rest', + 'test_ftplib.TestFTPClass.test_storlines', + 'test_ftplib.TestFTPClass.test_storlines_too_long', + 'test_ftplib.TestFTPClass.test_voidcmd', + 'test_ftplib.TestTLS_FTPClass.test_data_connection', + 'test_ftplib.TestTLS_FTPClass.test_control_connection', + 'test_ftplib.TestTLS_FTPClass.test_context', + 'test_ftplib.TestTLS_FTPClass.test_check_hostname', + 'test_ftplib.TestTLS_FTPClass.test_auth_ssl', + 'test_ftplib.TestTLS_FTPClass.test_auth_issued_twice', + + # This one times out, but it's still a non-blocking socket + 'test_ftplib.TestFTPClass.test_makeport', + + # A timeout, possibly because of the way we handle interrupts? + 'test_socketserver.SocketServerTest.test_InterruptedServerSelectCall', + 'test_socketserver.SocketServerTest.test_InterruptServerSelectCall', + + # times out with something about threading? + # The apparent hang is just after the print of "waiting for server" + 'test_socketserver.SocketServerTest.test_ThreadingTCPServer', + 'test_socketserver.SocketServerTest.test_ThreadingUDPServer', + 'test_socketserver.SocketServerTest.test_TCPServer', + 'test_socketserver.SocketServerTest.test_UDPServer', + + # This one might be like 'test_urllib2_localnet.TestUrlopen.test_https_with_cafile'? + # XXX: Look at newer pypy and verify our usage of drop/reuse matches + # theirs. + 'test_httpservers.BaseHTTPServerTestCase.test_command', + 'test_httpservers.BaseHTTPServerTestCase.test_handler', + 'test_httpservers.BaseHTTPServerTestCase.test_head_keep_alive', + 'test_httpservers.BaseHTTPServerTestCase.test_head_via_send_error', + 'test_httpservers.BaseHTTPServerTestCase.test_header_close', + 'test_httpservers.BaseHTTPServerTestCase.test_internal_key_error', + 'test_httpservers.BaseHTTPServerTestCase.test_request_line_trimming', + 'test_httpservers.BaseHTTPServerTestCase.test_return_custom_status', + 'test_httpservers.BaseHTTPServerTestCase.test_send_blank', + 'test_httpservers.BaseHTTPServerTestCase.test_send_error', + 'test_httpservers.BaseHTTPServerTestCase.test_version_bogus', + 'test_httpservers.BaseHTTPServerTestCase.test_version_digits', + 'test_httpservers.BaseHTTPServerTestCase.test_version_invalid', + 'test_httpservers.BaseHTTPServerTestCase.test_version_none', + 'test_httpservers.SimpleHTTPServerTestCase.test_get', + 'test_httpservers.SimpleHTTPServerTestCase.test_head', + 'test_httpservers.SimpleHTTPServerTestCase.test_invalid_requests', + 'test_httpservers.SimpleHTTPServerTestCase.test_path_without_leading_slash', + 'test_httpservers.CGIHTTPServerTestCase.test_invaliduri', + 'test_httpservers.CGIHTTPServerTestCase.test_issue19435', + + # Unexpected timeouts sometimes + 'test_smtplib.TooLongLineTests.testLineTooLong', + 'test_smtplib.GeneralTests.testTimeoutValue', + + ] + + if PYPY: + disabled_tests += [ + # appears to timeout? + 'test_threading.ThreadTests.test_finalize_with_trace', + 'test_asyncore.DispatcherWithSendTests_UsePoll.test_send', + 'test_asyncore.DispatcherWithSendTests.test_send', + + # More unexpected timeouts + 'test_ssl.ContextTests.test__https_verify_envvar', + 'test_subprocess.ProcessTestCase.test_check_output', + 'test_telnetlib.ReadTests.test_read_eager_A', + + # But on Windows, our gc fix for that doesn't work anyway + # so we have to disable it. + 'test_urllib2_localnet.TestUrlopen.test_https_with_cafile', + + # These tests hang. see above. + 'test_threading.ThreadJoinOnShutdown.test_1_join_on_shutdown', + 'test_threading.ThreadingExceptionTests.test_print_exception', + + # Our copy of these in test__subprocess.py also hangs. + # Anything that uses Popen.communicate or directly uses + # Popen.stdXXX.read hangs. It's not clear why. + 'test_subprocess.ProcessTestCase.test_communicate', + 'test_subprocess.ProcessTestCase.test_cwd', + 'test_subprocess.ProcessTestCase.test_env', + 'test_subprocess.ProcessTestCase.test_stderr_pipe', + 'test_subprocess.ProcessTestCase.test_stdout_pipe', + 'test_subprocess.ProcessTestCase.test_stdout_stderr_pipe', + 'test_subprocess.ProcessTestCase.test_stderr_redirect_with_no_stdout_redirect', + 'test_subprocess.ProcessTestCase.test_stdout_filedes_of_stdout', + 'test_subprocess.ProcessTestcase.test_stdout_none', + 'test_subprocess.ProcessTestcase.test_universal_newlines', + 'test_subprocess.ProcessTestcase.test_writes_before_communicate', + 'test_subprocess.Win32ProcessTestCase._kill_process', + 'test_subprocess.Win32ProcessTestCase._kill_dead_process', + 'test_subprocess.Win32ProcessTestCase.test_shell_sequence', + 'test_subprocess.Win32ProcessTestCase.test_shell_string', + 'test_subprocess.CommandsWithSpaces.with_spaces', + ] + + + if WIN: + + disabled_tests += [ + # This test winds up hanging a long time. + # Inserting GCs doesn't fix it. + 'test_ssl.ThreadedTests.test_handshake_timeout', + + # These sometimes raise LoopExit, for no apparent reason, + # mostly but not exclusively on Python 2. + 'test_socket.BufferIOTest.testRecvFromIntoBytearray', + 'test_socket.BufferIOTest.testRecvFromIntoArray', + 'test_socket.BufferIOTest.testRecvIntoArray', + 'test_socket.BufferIOTest.testRecvFromIntoEmptyBuffer', + 'test_socket.BufferIOTest.testRecvFromIntoMemoryview', + 'test_socket.BufferIOTest.testRecvFromIntoSmallBuffer', + ] + + if PY3: + + disabled_tests += [ + ] + + if APPVEYOR: + + disabled_tests += [ + ] + + if PYPY: + + if TRAVIS: + + disabled_tests += [ + # This sometimes causes a segfault for no apparent reason. + # See https://travis-ci.org/gevent/gevent/jobs/327328704 + # Can't reproduce locally. + 'test_subprocess.ProcessTestCase.test_universal_newlines_communicate', + ] + +if RUN_COVERAGE and CFFI_BACKEND: + disabled_tests += [ + # This test hangs in this combo for some reason + 'test_socket.GeneralModuleTests.test_sendall_interrupted', + # This can get a timeout exception instead of the Alarm + 'test_socket.TCPTimeoutTest.testInterruptedTimeout', + + # This test sometimes gets the wrong answer (due to changed timing?) + 'test_socketserver.SocketServerTest.test_ForkingUDPServer', + + # Timing and signals are off, so a handler exception doesn't get raised. + # Seen under libev + 'test_signal.InterProcessSignalTests.test_main', + ] + +if PY2: + if TRAVIS: + disabled_tests += [ + # When we moved to group:travis_latest and dist:xenial, + # this started returning a value (33554432L) != 0; presumably + # because of updated SSL library? Only on CPython. + 'test_ssl.ContextTests.test_options', + # When we moved to group:travis_latest and dist:xenial, + # one of the values used started *working* when it was expected to fail. + # The list of values and systems is long and complex, so + # presumably something needs to be updated. Only on PyPy. + 'test_ssl.ThreadedTests.test_alpn_protocols', + ] + +def _make_run_with_original(mod_name, func_name): + @contextlib.contextmanager + def with_orig(): + mod = __import__(mod_name) + now = getattr(mod, func_name) + from gevent.monkey import get_original + orig = get_original(mod_name, func_name) + try: + setattr(mod, func_name, orig) + yield + finally: + setattr(mod, func_name, now) + return with_orig + +@contextlib.contextmanager +def _gc_at_end(): + try: + yield + finally: + import gc + gc.collect() + gc.collect() + +@contextlib.contextmanager +def _flaky_socket_timeout(): + import socket + try: + yield + except socket.timeout: + flaky.reraiseFlakyTestTimeout() + +# Map from FQN to a context manager that will be wrapped around +# that test. +wrapped_tests = { +} + + + +class _PatchedTest(object): + def __init__(self, test_fqn): + self._patcher = wrapped_tests[test_fqn] + + def __call__(self, orig_test_fn): + + @functools.wraps(orig_test_fn) + def test(*args, **kwargs): + with self._patcher(): + return orig_test_fn(*args, **kwargs) + return test + + + +if sys.version_info[:3] <= (2, 7, 11): + + disabled_tests += [ + # These were added/fixed in 2.7.12+ + 'test_ssl.ThreadedTests.test__https_verify_certificates', + 'test_ssl.ThreadedTests.test__https_verify_envvar', + ] + +if OSX: + disabled_tests += [ + 'test_subprocess.POSIXProcessTestCase.test_run_abort', + # causes Mac OS X to show "Python crashes" dialog box which is annoying + ] + +if WIN: + disabled_tests += [ + # Issue with Unix vs DOS newlines in the file vs from the server + 'test_ssl.ThreadedTests.test_socketserver', + ] + + # These are a problem on 3.5; on 3.6+ they wind up getting (accidentally) disabled. + wrapped_tests.update({ + 'test_socket.SendfileUsingSendTest.testWithTimeout': _flaky_socket_timeout, + 'test_socket.SendfileUsingSendTest.testOffset': _flaky_socket_timeout, + 'test_socket.SendfileUsingSendTest.testRegularFile': _flaky_socket_timeout, + 'test_socket.SendfileUsingSendTest.testCount': _flaky_socket_timeout, + }) + +if PYPY: + disabled_tests += [ + # Does not exist in the CPython test suite, tests for a specific bug + # in PyPy's forking. Only runs on linux and is specific to the PyPy + # implementation of subprocess (possibly explains the extra parameter to + # _execut_child) + 'test_subprocess.ProcessTestCase.test_failed_child_execute_fd_leak', + # On some platforms, this returns "zlib_compression", but the test is looking for + # "ZLIB" + 'test_ssl.ThreadedTests.test_compression', + ] + +# Generic Python 3 + +if PY3: + + disabled_tests += [ + # Triggers the crash reporter + 'test_threading.SubinterpThreadingTests.test_daemon_threads_fatal_error', + + # Relies on an implementation detail, Thread._tstate_lock + 'test_threading.ThreadTests.test_tstate_lock', + # Relies on an implementation detail (reprs); we have our own version + 'test_threading.ThreadTests.test_various_ops', + 'test_threading.ThreadTests.test_various_ops_large_stack', + 'test_threading.ThreadTests.test_various_ops_small_stack', + + # Relies on Event having a _cond and an _reset_internal_locks() + # XXX: These are commented out in the source code of test_threading because + # this doesn't work. + # 'lock_tests.EventTests.test_reset_internal_locks', + + # Python bug 13502. We may or may not suffer from this as its + # basically a timing race condition. + # XXX Same as above + # 'lock_tests.EventTests.test_set_and_clear', + + # These tests want to assert on the type of the class that implements + # `Popen.stdin`; we use a FileObject, but they expect different subclasses + # from the `io` module + 'test_subprocess.ProcessTestCase.test_io_buffered_by_default', + 'test_subprocess.ProcessTestCase.test_io_unbuffered_works', + + # 3.3 exposed the `endtime` argument to wait accidentally. + # It is documented as deprecated and not to be used since 3.4 + # This test in 3.6.3 wants to use it though, and we don't have it. + 'test_subprocess.ProcessTestCase.test_wait_endtime', + + # These all want to inspect the string value of an exception raised + # by the exec() call in the child. The _posixsubprocess module arranges + # for better exception handling and printing than we do. + 'test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0', + 'test_subprocess.POSIXProcessTestCase.test_exception_bad_executable', + 'test_subprocess.POSIXProcessTestCase.test_exception_cwd', + # Relies on a 'fork_exec' attribute that we don't provide + 'test_subprocess.POSIXProcessTestCase.test_exception_errpipe_bad_data', + 'test_subprocess.POSIXProcessTestCase.test_exception_errpipe_normal', + + # Python 3 fixed a bug if the stdio file descriptors were closed; + # we still have that bug + 'test_subprocess.POSIXProcessTestCase.test_small_errpipe_write_fd', + + # Relies on implementation details (some of these tests were added in 3.4, + # but PyPy3 is also shipping them.) + 'test_socket.GeneralModuleTests.test_SocketType_is_socketobject', + 'test_socket.GeneralModuleTests.test_dealloc_warn', + 'test_socket.GeneralModuleTests.test_repr', + 'test_socket.GeneralModuleTests.test_str_for_enums', + 'test_socket.GeneralModuleTests.testGetaddrinfo', + + ] + if TRAVIS: + disabled_tests += [ + # test_cwd_with_relative_executable tends to fail + # on Travis...it looks like the test processes are stepping + # on each other and messing up their temp directories. We tend to get things like + # saved_dir = os.getcwd() + # FileNotFoundError: [Errno 2] No such file or directory + 'test_subprocess.ProcessTestCase.test_cwd_with_relative_arg', + 'test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_arg', + 'test_subprocess.ProcessTestCase.test_cwd_with_relative_executable', + + ] + + wrapped_tests.update({ + # XXX: BUG: We simply don't handle this correctly. On CPython, + # we wind up raising a BlockingIOError and then + # BrokenPipeError and then some random TypeErrors, all on the + # server. CPython 3.5 goes directly to socket.send() (via + # socket.makefile), whereas CPython 3.6 uses socket.sendall(). + # On PyPy, the behaviour is much worse: we hang indefinitely, perhaps exposing a problem + # with our signal handling. + # In actuality, though, this test doesn't fully test the EINTR it expects + # to under gevent (because if its EWOULDBLOCK retry behaviour.) + # Instead, the failures were all due to `pthread_kill` trying to send a signal + # to a greenlet instead of a real thread. The solution is to deliver the signal + # to the real thread by letting it get the correct ID. + 'test_wsgiref.IntegrationTests.test_interrupted_write': _make_run_with_original('threading', 'get_ident') + }) + +# PyPy3 3.5.5 v5.8-beta + +if PYPY3: + + + disabled_tests += [ + # This raises 'RuntimeError: reentrant call' when exiting the + # process tries to close the stdout stream; no other platform does this. + # Seen in both 3.3 and 3.5 (5.7 and 5.8) + 'test_signal.SiginterruptTest.test_siginterrupt_off', + ] + + +if PYPY and PY3: + disabled_tests += [ + # This fails to close all the FDs, at least on CI. On OS X, many of the + # POSIXProcessTestCase fd tests have issues. + 'test_subprocess.POSIXProcessTestCase.test_close_fds_when_max_fd_is_lowered', + + # This has the wrong constants in 5.8 (but worked in 5.7), at least on + # OS X. It finds "zlib compression" but expects "ZLIB". + 'test_ssl.ThreadedTests.test_compression', + + # The below are new with 5.10.1 + # This gets an EOF in violation of protocol; again, even without gevent + # (at least on OS X; it's less consistent about that on travis) + 'test_ssl.NetworkedBIOTests.test_handshake', + + ] + + if OSX: + disabled_tests += [ + # These all fail with "invalid_literal for int() with base 10: b''" + 'test_subprocess.POSIXProcessTestCase.test_close_fds', + 'test_subprocess.POSIXProcessTestCase.test_close_fds_after_preexec', + 'test_subprocess.POSIXProcessTestCase.test_pass_fds', + 'test_subprocess.POSIXProcessTestCase.test_pass_fds_inheritable', + 'test_subprocess.POSIXProcessTestCase.test_pipe_cloexec', + + # The below are new with 5.10.1 + # These fail with 'OSError: received malformed or improperly truncated ancillary data' + 'test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0', + 'test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Plus1', + 'test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1', + 'test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1', + + # Using the provided High Sierra binary, these fail with + # 'ValueError: invalid protocol version _SSLMethod.PROTOCOL_SSLv3'. + # gevent code isn't involved and running them unpatched has the same issue. + 'test_ssl.ContextTests.test_constructor', + 'test_ssl.ContextTests.test_protocol', + 'test_ssl.ContextTests.test_session_stats', + 'test_ssl.ThreadedTests.test_echo', + 'test_ssl.ThreadedTests.test_protocol_sslv23', + 'test_ssl.ThreadedTests.test_protocol_sslv3', + 'test_ssl.ThreadedTests.test_protocol_tlsv1', + 'test_ssl.ThreadedTests.test_protocol_tlsv1_1', + + # This gets None instead of http1.1, even without gevent + 'test_ssl.ThreadedTests.test_npn_protocols', + + # This fails to decode a filename even without gevent, + # at least on High Sierarr. + 'test_httpservers.SimpleHTTPServerTestCase.test_undecodable_filename', + ] + + disabled_tests += [ + # This seems to be a buffering issue? Something isn't + # getting flushed. (The output is wrong). Under PyPy3 5.7, + # I couldn't reproduce locally in Ubuntu 16 in a VM + # or a laptop with OS X. Under 5.8.0, I can reproduce it, but only + # when run by the testrunner, not when run manually on the command line, + # so something is changing in stdout buffering in those situations. + 'test_threading.ThreadJoinOnShutdown.test_2_join_in_forked_process', + 'test_threading.ThreadJoinOnShutdown.test_1_join_in_forked_process', + ] + + if TRAVIS: + disabled_tests += [ + # Likewise, but I haven't produced it locally. + 'test_threading.ThreadJoinOnShutdown.test_1_join_on_shutdown', + ] + +if PYPY: + + wrapped_tests.update({ + # XXX: gevent: The error that was raised by that last call + # left a socket open on the server or client. The server gets + # to http/server.py(390)handle_one_request and blocks on + # self.rfile.readline which apparently is where the SSL + # handshake is done. That results in the exception being + # raised on the client above, but apparently *not* on the + # server. Consequently it sits trying to read from that + # socket. On CPython, when the client socket goes out of scope + # it is closed and the server raises an exception, closing the + # socket. On PyPy, we need a GC cycle for that to happen. + # Without the socket being closed and exception being raised, + # the server cannot be stopped (it runs each request in the + # same thread that would notice it had been stopped), and so + # the cleanup method added by start_https_server to stop the + # server blocks "forever". + + # This is an important test, so rather than skip it in patched_tests_setup, + # we do the gc before we return. + 'test_urllib2_localnet.TestUrlopen.test_https_with_cafile': _gc_at_end, + }) + + +if PY34 and sys.version_info[:3] < (3, 4, 4): + # Older versions have some issues with the SSL tests. Seen on Appveyor + disabled_tests += [ + 'test_ssl.ContextTests.test_options', + 'test_ssl.ThreadedTests.test_protocol_sslv23', + 'test_ssl.ThreadedTests.test_protocol_sslv3', + 'test_httplib.HTTPSTest.test_networked', + ] + +if PY34: + disabled_tests += [ + 'test_subprocess.ProcessTestCase.test_threadsafe_wait', + # XXX: It seems that threading.Timer is not being greened properly, possibly + # due to a similar issue to what gevent.threading documents for normal threads. + # In any event, this test hangs forever + + + 'test_subprocess.POSIXProcessTestCase.test_preexec_errpipe_does_not_double_close_pipes', + # Subclasses Popen, and overrides _execute_child. Expects things to be done + # in a particular order in an exception case, but we don't follow that + # exact order + + + 'test_selectors.PollSelectorTestCase.test_above_fd_setsize', + # This test attempts to open many many file descriptors and + # poll on them, expecting them all to be ready at once. But + # libev limits the number of events it will return at once. Specifically, + # on linux with epoll, it returns a max of 64 (ev_epoll.c). + + # XXX: Hangs (Linux only) + 'test_socket.NonBlockingTCPTests.testInitNonBlocking', + # We don't handle the Linux-only SOCK_NONBLOCK option + 'test_socket.NonblockConstantTest.test_SOCK_NONBLOCK', + + # Tries to use multiprocessing which doesn't quite work in + # monkey_test module (Windows only) + 'test_socket.TestSocketSharing.testShare', + + # Windows-only: Sockets have a 'ioctl' method in Python 3 + # implemented in the C code. This test tries to check + # for the presence of the method in the class, which we don't + # have because we don't inherit the C implementation. But + # it should be found at runtime. + 'test_socket.GeneralModuleTests.test_sock_ioctl', + + # See comments for 2.7; these hang + 'test_httplib.HTTPSTest.test_local_good_hostname', + 'test_httplib.HTTPSTest.test_local_unknown_cert', + + # XXX This fails for an unknown reason + 'test_httplib.HeaderTests.test_parse_all_octets', + ] + + if OSX: + disabled_tests += [ + # These raise "OSError: 12 Cannot allocate memory" on both + # patched and unpatched runs + 'test_socket.RecvmsgSCMRightsStreamTest.testFDPassEmpty', + ] + + if sys.version_info[:2] == (3, 4): + disabled_tests += [ + # These are all expecting that a signal (sigalarm) that + # arrives during a blocking call should raise + # InterruptedError with errno=EINTR. gevent does not do + # this, instead its loop keeps going and raises a timeout + # (which fails the test). HOWEVER: Python 3.5 fixed this + # problem and started raising a timeout, + # (https://docs.python.org/3/whatsnew/3.5.html#pep-475-retry-system-calls-failing-with-eintr) + # and removed these tests (InterruptedError is no longer + # raised). So basically, gevent was ahead of its time. + 'test_socket.InterruptedRecvTimeoutTest.testInterruptedRecvIntoTimeout', + 'test_socket.InterruptedRecvTimeoutTest.testInterruptedRecvTimeout', + 'test_socket.InterruptedRecvTimeoutTest.testInterruptedRecvfromIntoTimeout', + 'test_socket.InterruptedRecvTimeoutTest.testInterruptedRecvfromTimeout', + 'test_socket.InterruptedRecvTimeoutTest.testInterruptedSendTimeout', + 'test_socket.InterruptedRecvTimeoutTest.testInterruptedSendtoTimeout', + 'test_socket.InterruptedRecvTimeoutTest.testInterruptedRecvmsgTimeout', + 'test_socket.InterruptedRecvTimeoutTest.testInterruptedRecvmsgIntoTimeout', + 'test_socket.InterruptedSendTimeoutTest.testInterruptedSendmsgTimeout', + ] + + if TRAVIS: + # This has been seen to produce "Inconsistency detected by + # ld.so: dl-open.c: 231: dl_open_worker: Assertion + # `_dl_debug_initialize (0, args->nsid)->r_state == + # RT_CONSISTENT' failed!" and fail. + disabled_tests += [ + 'test_threading.ThreadTests.test_is_alive_after_fork', + ] + + if TRAVIS: + disabled_tests += [ + 'test_subprocess.ProcessTestCase.test_double_close_on_error', + # This test is racy or OS-dependent. It passes locally (sufficiently fast machine) + # but fails under Travis + ] + +if PY35: + disabled_tests += [ + # XXX: Hangs + 'test_ssl.ThreadedTests.test_nonblocking_send', + 'test_ssl.ThreadedTests.test_socketserver', + # Uses direct sendfile, doesn't properly check for it being enabled + 'test_socket.GeneralModuleTests.test__sendfile_use_sendfile', + + + # Relies on the regex of the repr having the locked state (TODO: it'd be nice if + # we did that). + # XXX: These are commented out in the source code of test_threading because + # this doesn't work. + # 'lock_tests.LockTests.lest_locked_repr', + # 'lock_tests.LockTests.lest_repr', + + # Added between 3.6.0 and 3.6.3, uses _testcapi and internals + # of the subprocess module. + 'test_subprocess.POSIXProcessTestCase.test_stopped', + + # This test opens a socket, creates a new socket with the same fileno, + # closes the original socket (and hence fileno) and then + # expects that the calling setblocking() on the duplicate socket + # will raise an error. Our implementation doesn't work that way because + # setblocking() doesn't actually touch the file descriptor. + # That's probably OK because this was a GIL state error in CPython + # see https://github.com/python/cpython/commit/fa22b29960b4e683f4e5d7e308f674df2620473c + 'test_socket.TestExceptions.test_setblocking_invalidfd', + ] + + if ARES: + disabled_tests += [ + # These raise different errors or can't resolve + # the IP address correctly + 'test_socket.GeneralModuleTests.test_host_resolution', + 'test_socket.GeneralModuleTests.test_getnameinfo', + ] + + if sys.version_info[1] == 5: + disabled_tests += [ + # This test tends to time out, but only under 3.5, not under + # 3.6 or 3.7. Seen with both libev and libuv + 'test_socket.SendfileUsingSendTest.testWithTimeoutTriggeredSend', + ] + +if sys.version_info[:3] <= (3, 5, 1): + # Python issue 26499 was fixed in 3.5.2 and these tests were added. + disabled_tests += [ + 'test_httplib.BasicTest.test_mixed_reads', + 'test_httplib.BasicTest.test_read1_bound_content_length', + 'test_httplib.BasicTest.test_read1_content_length', + 'test_httplib.BasicTest.test_readline_bound_content_length', + 'test_httplib.BasicTest.test_readlines_content_length', + ] + +if PY36: + disabled_tests += [ + 'test_threading.MiscTestCase.test__all__', + ] + + # We don't actually implement socket._sendfile_use_sendfile, + # so these tests, which think they're using that and os.sendfile, + # fail. + disabled_tests += [ + 'test_socket.SendfileUsingSendfileTest.testCount', + 'test_socket.SendfileUsingSendfileTest.testCountSmall', + 'test_socket.SendfileUsingSendfileTest.testCountWithOffset', + 'test_socket.SendfileUsingSendfileTest.testOffset', + 'test_socket.SendfileUsingSendfileTest.testRegularFile', + 'test_socket.SendfileUsingSendfileTest.testWithTimeout', + 'test_socket.SendfileUsingSendfileTest.testEmptyFileSend', + 'test_socket.SendfileUsingSendfileTest.testNonBlocking', + 'test_socket.SendfileUsingSendfileTest.test_errors', + ] + + # Ditto + disabled_tests += [ + 'test_socket.GeneralModuleTests.test__sendfile_use_sendfile', + ] + + disabled_tests += [ + # This test requires Linux >= 4.3. When we were running 'dist: + # trusty' on the 4.4 kernel, it passed (~July 2017). But when + # trusty became the default dist in September 2017 and updated + # the kernel to 4.11.6, it begain failing. It fails on `res = + # op.recv(assoclen + len(plain) + taglen)` (where 'op' is the + # client socket) with 'OSError: [Errno 22] Invalid argument' + # for unknown reasons. This is *after* having successfully + # called `op.sendmsg_afalg`. Post 3.6.0, what we test with, + # the test was changed to require Linux 4.9 and the data was changed, + # so this is not our fault. We should eventually update this when we + # update our 3.6 version. + # See https://bugs.python.org/issue29324 + 'test_socket.LinuxKernelCryptoAPI.test_aead_aes_gcm', + ] + +if PY37: + disabled_tests += [ + # These want to use the private '_communicate' method, which + # our Popen doesn't have. + 'test_subprocess.MiscTests.test_call_keyboardinterrupt_no_kill', + 'test_subprocess.MiscTests.test_context_manager_keyboardinterrupt_no_kill', + 'test_subprocess.MiscTests.test_run_keyboardinterrupt_no_kill', + + # This wants to check that the underlying fileno is blocking, + # but it isn't. + 'test_socket.NonBlockingTCPTests.testSetBlocking', + + # 3.7b2 made it impossible to instantiate SSLSocket objects + # directly, and this tests for that, but we don't follow that change. + 'test_ssl.BasicSocketTests.test_private_init', + + # 3.7b2 made a change to this test that on the surface looks incorrect, + # but it passes when they run it and fails when we do. It's not + # clear why. + 'test_ssl.ThreadedTests.test_check_hostname_idn', + ] + + if APPVEYOR: + disabled_tests += [ + + ] + +# if 'signalfd' in os.environ.get('GEVENT_BACKEND', ''): +# # tests that don't interact well with signalfd +# disabled_tests.extend([ +# 'test_signal.SiginterruptTest.test_siginterrupt_off', +# 'test_socketserver.SocketServerTest.test_ForkingTCPServer', +# 'test_socketserver.SocketServerTest.test_ForkingUDPServer', +# 'test_socketserver.SocketServerTest.test_ForkingUnixStreamServer']) + +# LibreSSL reports OPENSSL_VERSION_INFO (2, 0, 0, 0, 0) regardless of its version, +# so this is known to fail on some distros. We don't want to detect this because we +# don't want to trigger the side-effects of importing ssl prematurely if we will +# be monkey-patching, so we skip this test everywhere. It doesn't do much for us +# anyway. +disabled_tests += [ + 'test_ssl.BasicSocketTests.test_openssl_version' +] + +# Now build up the data structure we'll use to actually find disabled tests +# to avoid a linear scan for every file (it seems the list could get quite large) +# (First, freeze the source list to make sure it isn't modified anywhere) + +def _build_test_structure(sequence_of_tests): + + _disabled_tests = frozenset(sequence_of_tests) + + disabled_tests_by_file = collections.defaultdict(set) + for file_case_meth in _disabled_tests: + file_name, _case, _meth = file_case_meth.split('.') + + by_file = disabled_tests_by_file[file_name] + + by_file.add(file_case_meth) + + return disabled_tests_by_file + +_disabled_tests_by_file = _build_test_structure(disabled_tests) + +_wrapped_tests_by_file = _build_test_structure(wrapped_tests) + + +def disable_tests_in_source(source, filename): + # Source and filename are both native strings. + + if filename.startswith('./'): + # turn "./test_socket.py" (used for auto-complete) into "test_socket.py" + filename = filename[2:] + + if filename.endswith('.py'): + filename = filename[:-3] + + + # XXX ignoring TestCase class name (just using function name). + # Maybe we should do this with the AST, or even after the test is + # imported. + my_disabled_tests = _disabled_tests_by_file.get(filename, ()) + my_wrapped_tests = _wrapped_tests_by_file.get(filename, {}) + + + if my_disabled_tests or my_wrapped_tests: + # Insert our imports early in the file. + # If we do it on a def-by-def basis, we can break syntax + # if the function is already decorated + pattern = r'^import .*' + replacement = r'from gevent.testing import patched_tests_setup as _GEVENT_PTS;' + replacement += r'import unittest as _GEVENT_UTS;' + replacement += r'\g<0>' + source, n = re.subn(pattern, replacement, source, 1, re.MULTILINE) + + print("Added imports", n) + + # Test cases will always be indented some, + # so use [ \t]+. Without indentation, test_main, commonly used as the + # __main__ function at the top level, could get matched. \s matches + # newlines even in MULTILINE mode so it would still match that. + my_disabled_testcases = set() + for test in my_disabled_tests: + testcase = test.split('.')[-1] + my_disabled_testcases.add(testcase) + # def foo_bar(self) + # -> + # @_GEVENT_UTS.skip('Removed by patched_tests_setup') + # def foo_bar(self) + pattern = r"^([ \t]+)def " + testcase + replacement = r"\1@_GEVENT_UTS.skip('Removed by patched_tests_setup: %s')\n" % (test,) + replacement += r"\g<0>" + source, n = re.subn(pattern, replacement, source, 0, re.MULTILINE) + print('Skipped %s (%d)' % (testcase, n), file=sys.stderr) + + + for test in my_wrapped_tests: + testcase = test.split('.')[-1] + if testcase in my_disabled_testcases: + print("Not wrapping %s because it is skipped" % (test,)) + continue + + # def foo_bar(self) + # -> + # @_GEVENT_PTS._PatchedTest('file.Case.name') + # def foo_bar(self) + pattern = r"^([ \t]+)def " + testcase + replacement = r"\1@_GEVENT_PTS._PatchedTest('%s')\n" % (test,) + replacement += r"\g<0>" + + source, n = re.subn(pattern, replacement, source, 0, re.MULTILINE) + print('Wrapped %s (%d)' % (testcase, n), file=sys.stderr) + + return source diff --git a/src/gevent/testing/six.py b/src/gevent/testing/six.py new file mode 100644 index 0000000..955d14b --- /dev/null +++ b/src/gevent/testing/six.py @@ -0,0 +1,42 @@ +import sys +# pylint:disable=unused-argument,import-error + +PY3 = sys.version_info[0] >= 3 + +if PY3: + import builtins + exec_ = getattr(builtins, "exec") + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + xrange = range + string_types = (str,) + text_type = str + +else: + def exec_(code, globs=None, locs=None): + """Execute code in a namespace.""" + if globs is None: + frame = sys._getframe(1) + globs = frame.f_globals + if locs is None: + locs = frame.f_locals + del frame + elif locs is None: + locs = globs + exec("""exec code in globs, locs""") + + import __builtin__ as builtins + xrange = builtins.xrange + string_types = (builtins.basestring,) + text_type = builtins.unicode + + exec_("""def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""") diff --git a/src/gevent/testing/skipping.py b/src/gevent/testing/skipping.py new file mode 100644 index 0000000..d2de2f5 --- /dev/null +++ b/src/gevent/testing/skipping.py @@ -0,0 +1,116 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +import unittest + +from . import sysinfo + +def _identity(f): + return f + +def _do_not_skip(reason): + assert reason + return _identity + + +skipOnWindows = _do_not_skip +skipOnAppVeyor = _do_not_skip +skipOnCI = _do_not_skip + +skipOnPyPy = _do_not_skip +skipOnPyPyOnCI = _do_not_skip +skipOnPyPy3OnCI = _do_not_skip +skipOnPyPy3 = _do_not_skip +skipOnPyPyOnWindows = _do_not_skip + +skipOnPy37 = unittest.skip if sysinfo.PY37 else _do_not_skip + +skipOnPurePython = unittest.skip if sysinfo.PURE_PYTHON else _do_not_skip +skipWithCExtensions = unittest.skip if not sysinfo.PURE_PYTHON else _do_not_skip + +skipOnLibuv = _do_not_skip +skipOnLibuvOnWin = _do_not_skip +skipOnLibuvOnCI = _do_not_skip +skipOnLibuvOnCIOnPyPy = _do_not_skip +skipOnLibuvOnPyPyOnWin = _do_not_skip +skipOnLibuvOnTravisOnCPython27 = _do_not_skip + +skipOnLibev = _do_not_skip + +if sysinfo.WIN: + skipOnWindows = unittest.skip + + +if sysinfo.RUNNING_ON_APPVEYOR: + # See comments scattered around about timeouts and the timer + # resolution available on appveyor (lots of jitter). this + # seems worse with the 62-bit builds. + # Note that we skip/adjust these tests only on AppVeyor, not + # win32---we don't think there's gevent related problems but + # environment related problems. These can be tested and debugged + # separately on windows in a more stable environment. + skipOnAppVeyor = unittest.skip + + +if sysinfo.RUNNING_ON_CI: + skipOnCI = unittest.skip + + +if sysinfo.PYPY: + skipOnPyPy = unittest.skip + if sysinfo.RUNNING_ON_CI: + skipOnPyPyOnCI = unittest.skip + + if sysinfo.WIN: + skipOnPyPyOnWindows = unittest.skip + + if sysinfo.PYPY3: + skipOnPyPy3 = unittest.skip + if sysinfo.RUNNING_ON_CI: + # Same as above, for PyPy3.3-5.5-alpha and 3.5-5.7.1-beta and 3.5-5.8 + skipOnPyPy3OnCI = unittest.skip + + +skipUnderCoverage = unittest.skip if sysinfo.RUN_COVERAGE else _do_not_skip + +skipIf = unittest.skipIf +skipUnless = unittest.skipUnless + + + +if sysinfo.LIBUV: + skipOnLibuv = unittest.skip + + if sysinfo.RUNNING_ON_CI: + skipOnLibuvOnCI = unittest.skip + if sysinfo.PYPY: + skipOnLibuvOnCIOnPyPy = unittest.skip + if sysinfo.RUNNING_ON_TRAVIS: + if sysinfo.CPYTHON: + if sysinfo.PY27_ONLY: + skipOnLibuvOnTravisOnCPython27 = unittest.skip + + if sysinfo.WIN: + skipOnLibuvOnWin = unittest.skip + if sysinfo.PYPY: + skipOnLibuvOnPyPyOnWin = unittest.skip +else: + skipOnLibev = unittest.skip diff --git a/src/gevent/testing/sockets.py b/src/gevent/testing/sockets.py new file mode 100644 index 0000000..147da63 --- /dev/null +++ b/src/gevent/testing/sockets.py @@ -0,0 +1,41 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +from .params import DEFAULT_BIND_ADDR_TUPLE + +def bind_and_listen(sock, address=DEFAULT_BIND_ADDR_TUPLE, backlog=50, reuse_addr=True): + from socket import SOL_SOCKET, SO_REUSEADDR, error + if reuse_addr: + try: + sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, + sock.getsockopt(SOL_SOCKET, SO_REUSEADDR) | 1) + except error: + pass + sock.bind(address) + sock.listen(backlog) + + +def tcp_listener(address=DEFAULT_BIND_ADDR_TUPLE, backlog=50, reuse_addr=True): + """A shortcut to create a TCP socket, bind it and put it into listening state.""" + from gevent import socket + sock = socket.socket() + bind_and_listen(sock, address, backlog=backlog, reuse_addr=reuse_addr) + return sock diff --git a/src/gevent/testing/switching.py b/src/gevent/testing/switching.py new file mode 100644 index 0000000..d846dc8 --- /dev/null +++ b/src/gevent/testing/switching.py @@ -0,0 +1,64 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +from functools import wraps + +from gevent.hub import _get_hub + +from .hub import QuietHub + +from .patched_tests_setup import get_switch_expected + +def wrap_switch_count_check(method): + @wraps(method) + def wrapper(self, *args, **kwargs): + initial_switch_count = getattr(_get_hub(), 'switch_count', None) + self.switch_expected = getattr(self, 'switch_expected', True) + if initial_switch_count is not None: + fullname = getattr(self, 'fullname', None) + if self.switch_expected == 'default' and fullname: + self.switch_expected = get_switch_expected(fullname) + result = method(self, *args, **kwargs) + if initial_switch_count is not None and self.switch_expected is not None: + switch_count = _get_hub().switch_count - initial_switch_count + if self.switch_expected is True: + assert switch_count >= 0 + if not switch_count: + raise AssertionError('%s did not switch' % fullname) + elif self.switch_expected is False: + if switch_count: + raise AssertionError('%s switched but not expected to' % fullname) + else: + raise AssertionError('Invalid value for switch_expected: %r' % (self.switch_expected, )) + return result + return wrapper + + + + +class CountingHub(QuietHub): + + switch_count = 0 + + def switch(self, *args): + # pylint:disable=arguments-differ + self.switch_count += 1 + return QuietHub.switch(self, *args) diff --git a/src/gevent/testing/sysinfo.py b/src/gevent/testing/sysinfo.py new file mode 100644 index 0000000..b20b595 --- /dev/null +++ b/src/gevent/testing/sysinfo.py @@ -0,0 +1,137 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import os +import sys + +import gevent.core +from gevent import _compat as gsysinfo + +PYPY = gsysinfo.PYPY +CPYTHON = not PYPY +VERBOSE = sys.argv.count('-v') > 1 +WIN = gsysinfo.WIN +LINUX = gsysinfo.LINUX +OSX = gsysinfo.OSX + +PURE_PYTHON = gsysinfo.PURE_PYTHON + +# XXX: Formalize this better +LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member +CFFI_BACKEND = PYPY or LIBUV or 'cffi' in os.getenv('GEVENT_LOOP', '') + +if '--debug-greentest' in sys.argv: + sys.argv.remove('--debug-greentest') + DEBUG = True +else: + DEBUG = False + +RUN_LEAKCHECKS = os.getenv('GEVENTTEST_LEAKCHECK') +RUN_COVERAGE = os.getenv("COVERAGE_PROCESS_START") or os.getenv("GEVENTTEST_COVERAGE") + +# Generally, ignore the portions that are only implemented +# on particular platforms; they generally contain partial +# implementations completed in different modules. +PLATFORM_SPECIFIC_SUFFIXES = ('2', '279', '3') +if WIN: + PLATFORM_SPECIFIC_SUFFIXES += ('posix',) + +PY2 = None +PY3 = None +PY34 = None +PY35 = None +PY36 = None +PY37 = None + +NON_APPLICABLE_SUFFIXES = () +if sys.version_info[0] == 3: + # Python 3 + NON_APPLICABLE_SUFFIXES += ('2', '279') + PY2 = False + PY3 = True + if sys.version_info[1] >= 4: + PY34 = True + if sys.version_info[1] >= 5: + PY35 = True + if sys.version_info[1] >= 6: + PY36 = True + if sys.version_info[1] >= 7: + PY37 = True + +elif sys.version_info[0] == 2: + # Any python 2 + PY3 = False + PY2 = True + NON_APPLICABLE_SUFFIXES += ('3',) + if (sys.version_info[1] < 7 + or (sys.version_info[1] == 7 and sys.version_info[2] < 9)): + # Python 2, < 2.7.9 + NON_APPLICABLE_SUFFIXES += ('279',) + +PYPY3 = PYPY and PY3 + +PY27_ONLY = sys.version_info[0] == 2 and sys.version_info[1] == 7 + +PYGTE279 = ( + sys.version_info[0] == 2 + and sys.version_info[1] >= 7 + and sys.version_info[2] >= 9 +) + +if WIN: + NON_APPLICABLE_SUFFIXES += ("posix",) + # This is intimately tied to FileObjectPosix + NON_APPLICABLE_SUFFIXES += ("fileobject2",) + SHARED_OBJECT_EXTENSION = ".pyd" +else: + SHARED_OBJECT_EXTENSION = ".so" + + +RUNNING_ON_TRAVIS = os.environ.get('TRAVIS') +RUNNING_ON_APPVEYOR = os.environ.get('APPVEYOR') +RUNNING_ON_CI = RUNNING_ON_TRAVIS or RUNNING_ON_APPVEYOR + +if RUNNING_ON_APPVEYOR: + # We can't exec corecext on appveyor if we haven't run setup.py in + # 'develop' mode (i.e., we install) + NON_APPLICABLE_SUFFIXES += ('corecext',) + +EXPECT_POOR_TIMER_RESOLUTION = (PYPY3 + or RUNNING_ON_APPVEYOR + or (LIBUV and PYPY) + or RUN_COVERAGE) + + +CONN_ABORTED_ERRORS = [] +try: + from errno import WSAECONNABORTED + CONN_ABORTED_ERRORS.append(WSAECONNABORTED) +except ImportError: + pass + +from errno import ECONNRESET +CONN_ABORTED_ERRORS.append(ECONNRESET) + +CONN_ABORTED_ERRORS = frozenset(CONN_ABORTED_ERRORS) + +RESOLVER_ARES = os.getenv('GEVENT_RESOLVER') == 'ares' +RESOLVER_DNSPYTHON = os.getenv('GEVENT_RESOLVER') == 'dnspython' + +RESOLVER_NOT_SYSTEM = RESOLVER_ARES or RESOLVER_DNSPYTHON diff --git a/src/gevent/testing/testcase.py b/src/gevent/testing/testcase.py new file mode 100644 index 0000000..c16413b --- /dev/null +++ b/src/gevent/testing/testcase.py @@ -0,0 +1,340 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +from __future__ import absolute_import, print_function, division + +import sys +from time import time +import os.path +from contextlib import contextmanager +from unittest import TestCase as BaseTestCase +from functools import wraps + +import gevent + +from . import sysinfo +from . import params +from . import leakcheck +from . import errorhandler +from . import flaky + +from .patched_tests_setup import get_switch_expected + +class TimeAssertMixin(object): + @flaky.reraises_flaky_timeout() + def assertTimeoutAlmostEqual(self, first, second, places=None, msg=None, delta=None): + try: + self.assertAlmostEqual(first, second, places=places, msg=msg, delta=delta) + except AssertionError: + flaky.reraiseFlakyTestTimeout() + + + if sysinfo.EXPECT_POOR_TIMER_RESOLUTION: + # pylint:disable=unused-argument + def assertTimeWithinRange(self, time_taken, min_time, max_time): + return + else: + def assertTimeWithinRange(self, time_taken, min_time, max_time): + self.assertLessEqual(time_taken, max_time) + self.assertGreaterEqual(time_taken, min_time) + + @contextmanager + def runs_in_given_time(self, expected, fuzzy=None): + if fuzzy is None: + if sysinfo.EXPECT_POOR_TIMER_RESOLUTION or sysinfo.LIBUV: + # The noted timer jitter issues on appveyor/pypy3 + fuzzy = expected * 5.0 + else: + fuzzy = expected / 2.0 + start = time() + yield + elapsed = time() - start + try: + self.assertTrue( + expected - fuzzy <= elapsed <= expected + fuzzy, + 'Expected: %r; elapsed: %r; fuzzy %r' % (expected, elapsed, fuzzy)) + except AssertionError: + flaky.reraiseFlakyTestRaceCondition() + + def runs_in_no_time( + self, + fuzzy=(0.01 if not sysinfo.EXPECT_POOR_TIMER_RESOLUTION and not sysinfo.LIBUV else 1.0)): + return self.runs_in_given_time(0.0, fuzzy) + + +def _wrap_timeout(timeout, method): + if timeout is None: + return method + + @wraps(method) + def wrapper(self, *args, **kwargs): + with gevent.Timeout(timeout, 'test timed out', ref=False): + return method(self, *args, **kwargs) + + return wrapper + +def _get_class_attr(classDict, bases, attr, default=AttributeError): + NONE = object() + value = classDict.get(attr, NONE) + if value is not NONE: + return value + for base in bases: + value = getattr(base, attr, NONE) + if value is not NONE: + return value + if default is AttributeError: + raise AttributeError('Attribute %r not found\n%s\n%s\n' % (attr, classDict, bases)) + return default + + +class TestCaseMetaClass(type): + # wrap each test method with + # a) timeout check + # b) fatal error check + # c) restore the hub's error handler (see expect_one_error) + # d) totalrefcount check + def __new__(cls, classname, bases, classDict): + # pylint and pep8 fight over what this should be called (mcs or cls). + # pylint gets it right, but we cant scope disable pep8, so we go with + # its convention. + # pylint: disable=bad-mcs-classmethod-argument + timeout = classDict.get('__timeout__', 'NONE') + if timeout == 'NONE': + timeout = getattr(bases[0], '__timeout__', None) + if sysinfo.RUN_LEAKCHECKS and timeout is not None: + timeout *= 6 + check_totalrefcount = _get_class_attr(classDict, bases, 'check_totalrefcount', True) + + error_fatal = _get_class_attr(classDict, bases, 'error_fatal', True) + uses_handle_error = _get_class_attr(classDict, bases, 'uses_handle_error', True) + # Python 3: must copy, we mutate the classDict. Interestingly enough, + # it doesn't actually error out, but under 3.6 we wind up wrapping + # and re-wrapping the same items over and over and over. + for key, value in list(classDict.items()): + if key.startswith('test') and callable(value): + classDict.pop(key) + # XXX: When did we stop doing this? + #value = wrap_switch_count_check(value) + value = _wrap_timeout(timeout, value) + error_fatal = getattr(value, 'error_fatal', error_fatal) + if error_fatal: + value = errorhandler.wrap_error_fatal(value) + if uses_handle_error: + value = errorhandler.wrap_restore_handle_error(value) + if check_totalrefcount and sysinfo.RUN_LEAKCHECKS: + value = leakcheck.wrap_refcount(value) + classDict[key] = value + return type.__new__(cls, classname, bases, classDict) + +def _noop(): + return + +class SubscriberCleanupMixin(object): + + def setUp(self): + super(SubscriberCleanupMixin, self).setUp() + from gevent import events + self.__old_subscribers = events.subscribers[:] + + def tearDown(self): + from gevent import events + events.subscribers[:] = self.__old_subscribers + super(SubscriberCleanupMixin, self).tearDown() + + +class TestCase(TestCaseMetaClass("NewBase", + (SubscriberCleanupMixin, TimeAssertMixin, BaseTestCase,), + {})): + __timeout__ = params.LOCAL_TIMEOUT if not sysinfo.RUNNING_ON_CI else params.CI_TIMEOUT + + switch_expected = 'default' + error_fatal = True + uses_handle_error = True + close_on_teardown = () + __old_subscribers = () + + def run(self, *args, **kwargs): + # pylint:disable=arguments-differ + if self.switch_expected == 'default': + self.switch_expected = get_switch_expected(self.fullname) + return BaseTestCase.run(self, *args, **kwargs) + + def setUp(self): + super(TestCase, self).setUp() + # Especially if we're running in leakcheck mode, where + # the same test gets executed repeatedly, we need to update the + # current time. Tests don't always go through the full event loop, + # so that doesn't always happen. test__pool.py:TestPoolYYY.test_async + # tends to show timeouts that are too short if we don't. + # XXX: Should some core part of the loop call this? + gevent.get_hub().loop.update_now() + self.close_on_teardown = [] + + def tearDown(self): + if getattr(self, 'skipTearDown', False): + return + + cleanup = getattr(self, 'cleanup', _noop) + cleanup() + self._error = self._none + self._tearDownCloseOnTearDown() + self.close_on_teardown = [] + super(TestCase, self).tearDown() + + def _tearDownCloseOnTearDown(self): + # XXX: Should probably reverse this + for x in self.close_on_teardown: + close = getattr(x, 'close', x) + try: + close() + except Exception: # pylint:disable=broad-except + pass + + @classmethod + def setUpClass(cls): + import warnings + cls._warning_cm = warnings.catch_warnings() + cls._warning_cm.__enter__() + if not sys.warnoptions: + warnings.simplefilter('default') + super(TestCase, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + cls._warning_cm.__exit__(None, None, None) + super(TestCase, cls).tearDownClass() + + def _close_on_teardown(self, resource): + """ + *resource* either has a ``close`` method, or is a + callable. + """ + self.close_on_teardown.append(resource) + return resource + + @property + def testname(self): + return getattr(self, '_testMethodName', '') or getattr(self, '_TestCase__testMethodName') + + @property + def testcasename(self): + return self.__class__.__name__ + '.' + self.testname + + @property + def modulename(self): + return os.path.basename(sys.modules[self.__class__.__module__].__file__).rsplit('.', 1)[0] + + @property + def fullname(self): + return os.path.splitext(os.path.basename(self.modulename))[0] + '.' + self.testcasename + + _none = (None, None, None) + # (context, kind, value) + _error = _none + + def expect_one_error(self): + self.assertEqual(self._error, self._none) + gevent.get_hub().handle_error = self._store_error + + def _store_error(self, where, t, value, tb): + del tb + if self._error != self._none: + gevent.get_hub().parent.throw(t, value) + else: + self._error = (where, t, value) + + def peek_error(self): + return self._error + + def get_error(self): + try: + return self._error + finally: + self._error = self._none + + def assert_error(self, kind=None, value=None, error=None, where_type=None): + if error is None: + error = self.get_error() + econtext, ekind, evalue = error + if kind is not None: + self.assertIsInstance(kind, type) + self.assertIsNotNone( + ekind, + "Error must not be none %r" % (error,)) + assert issubclass(ekind, kind), error + if value is not None: + if isinstance(value, str): + self.assertEqual(str(evalue), value) + else: + self.assertIs(evalue, value) + if where_type is not None: + self.assertIsInstance(econtext, where_type) + return error + + def assertMonkeyPatchedFuncSignatures(self, mod_name, func_names=(), exclude=()): + # We use inspect.getargspec because it's the only thing available + # in Python 2.7, but it is deprecated + # pylint:disable=deprecated-method,too-many-locals + import inspect + import warnings + from gevent.monkey import get_original + # XXX: Very similar to gevent.monkey.patch_module. Should refactor? + gevent_module = getattr(__import__('gevent.' + mod_name), mod_name) + module_name = getattr(gevent_module, '__target__', mod_name) + + funcs_given = True + if not func_names: + funcs_given = False + func_names = getattr(gevent_module, '__implements__') + + for func_name in func_names: + if func_name in exclude: + continue + gevent_func = getattr(gevent_module, func_name) + if not inspect.isfunction(gevent_func) and not funcs_given: + continue + + func = get_original(module_name, func_name) + + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + gevent_sig = inspect.getargspec(gevent_func) + sig = inspect.getargspec(func) + except TypeError: + if funcs_given: + raise + # Can't do this one. If they specifically asked for it, + # it's an error, otherwise it's not. + # Python 3 can check a lot more than Python 2 can. + continue + self.assertEqual(sig.args, gevent_sig.args, func_name) + # The next three might not actually matter? + self.assertEqual(sig.varargs, gevent_sig.varargs, func_name) + self.assertEqual(sig.keywords, gevent_sig.keywords, func_name) + self.assertEqual(sig.defaults, gevent_sig.defaults, func_name) + + def assertEqualFlakyRaceCondition(self, a, b): + try: + self.assertEqual(a, b) + except AssertionError: + flaky.reraiseFlakyTestRaceCondition() + + assertRaisesRegex = getattr(BaseTestCase, 'assertRaisesRegex', + getattr(BaseTestCase, 'assertRaisesRegexp')) diff --git a/src/gevent/testing/testrunner.py b/src/gevent/testing/testrunner.py new file mode 100644 index 0000000..2eb4deb --- /dev/null +++ b/src/gevent/testing/testrunner.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python +from __future__ import print_function, absolute_import, division + +import sys +import os +import glob +import traceback +import time +import importlib +from datetime import timedelta + +from multiprocessing.pool import ThreadPool +from multiprocessing import cpu_count +from . import util +from .util import log +from .sysinfo import RUNNING_ON_CI +from .sysinfo import PYPY +from .sysinfo import PY2 +from .sysinfo import RESOLVER_ARES +from .sysinfo import RUN_LEAKCHECKS +from . import six + +# Import this while we're probably single-threaded/single-processed +# to try to avoid issues with PyPy 5.10. +# See https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception +try: + __import__('_testcapi') +except (ImportError, OSError, IOError): + # This can raise a wide variety of errors + pass + +TIMEOUT = 100 +NWORKERS = int(os.environ.get('NWORKERS') or max(cpu_count() - 1, 4)) +if NWORKERS > 10: + NWORKERS = 10 + +if RUN_LEAKCHECKS: + # Capturing the stats takes time, and we run each + # test at least twice + TIMEOUT = 200 + +DEFAULT_RUN_OPTIONS = { + 'timeout': TIMEOUT +} + + +if RUNNING_ON_CI: + # Too many and we get spurious timeouts + NWORKERS = 4 + + + + +def _package_relative_filename(filename, package): + if not os.path.isfile(filename) and package: + # Ok, try to locate it as a module in the package + package_dir = _dir_from_package_name(package) + return os.path.join(package_dir, filename) + return filename + +def _dir_from_package_name(package): + package_mod = importlib.import_module(package) + package_dir = os.path.dirname(package_mod.__file__) + return package_dir + + +def run_many(tests, + configured_failing_tests=(), + failfast=False, + quiet=False, + configured_run_alone_tests=()): + # pylint:disable=too-many-locals,too-many-statements + global NWORKERS + start = time.time() + total = 0 + failed = {} + passed = {} + total_cases = [0] + total_skipped = [0] + + NWORKERS = min(len(tests), NWORKERS) or 1 + + pool = ThreadPool(NWORKERS) + util.BUFFER_OUTPUT = NWORKERS > 1 or quiet + + def run_one(cmd, **kwargs): + kwargs['quiet'] = quiet + result = util.run(cmd, **kwargs) + if result: + if failfast: + sys.exit(1) + failed[result.name] = [cmd, kwargs] + else: + passed[result.name] = True + total_cases[0] += result.run_count + total_skipped[0] += result.skipped_count + + results = [] + + def reap(): + for r in results[:]: + if not r.ready(): + continue + if r.successful(): + results.remove(r) + else: + r.get() + sys.exit('Internal error in testrunner.py: %r' % (r, )) + return len(results) + + def reap_all(): + while reap() > 0: + time.sleep(0.1) + + def spawn(cmd, options): + while True: + if reap() < NWORKERS: + r = pool.apply_async(run_one, (cmd, ), options or {}) + results.append(r) + return + + time.sleep(0.05) + + run_alone = [] + + try: + try: + log("Running tests in parallel with concurrency %s" % (NWORKERS,),) + for cmd, options in tests: + total += 1 + options = options or {} + if matches(configured_run_alone_tests, cmd): + run_alone.append((cmd, options)) + else: + spawn(cmd, options) + pool.close() + pool.join() + + log("Running tests marked standalone") + for cmd, options in run_alone: + run_one(cmd, **options) + + except KeyboardInterrupt: + try: + log('Waiting for currently running to finish...') + reap_all() + except KeyboardInterrupt: + pool.terminate() + report(total, failed, passed, exit=False, took=time.time() - start, + configured_failing_tests=configured_failing_tests, + total_cases=total_cases[0], total_skipped=total_skipped[0]) + log('(partial results)\n') + raise + except: + traceback.print_exc() + pool.terminate() + raise + + reap_all() + report(total, failed, passed, took=time.time() - start, + configured_failing_tests=configured_failing_tests, + total_cases=total_cases[0], total_skipped=total_skipped[0]) + +def discover( + tests=None, ignore_files=None, + ignored=(), coverage=False, + package=None, + configured_ignore_coverage=(), + configured_test_options=None, +): + # pylint:disable=too-many-locals,too-many-branches + configured_test_options = configured_test_options or {} + olddir = os.getcwd() + ignore = set(ignored or ()) + + if ignore_files: + ignore_files = ignore_files.split(',') + for f in ignore_files: + ignore.update(set(load_list_from_file(f, package))) + + if coverage: + ignore.update(configured_ignore_coverage) + + if package: + package_dir = _dir_from_package_name(package) + # We need to glob relative names, our config is based on filenames still + os.chdir(package_dir) + + if not tests: + tests = set(glob.glob('test_*.py')) - set(['test_support.py']) + else: + tests = set(tests) + + if ignore: + # Always ignore the designated list, even if tests were specified + # on the command line. This fixes a nasty interaction with test__threading_vs_settrace.py + # being run under coverage when 'grep -l subprocess test*py' is used to list the tests + # to run. + tests -= ignore + tests = sorted(tests) + + to_process = [] + to_import = [] + + for filename in tests: + module_name = os.path.splitext(filename)[0] + qualified_name = package + '.' + module_name if package else module_name + with open(os.path.abspath(filename), 'rb') as f: + # Some of the test files (e.g., test__socket_dns) are + # UTF8 encoded. Depending on the environment, Python 3 may + # try to decode those as ASCII, which fails with UnicodeDecodeError. + # Thus, be sure to open and compare in binary mode. + # Open the absolute path to make errors more clear, + # but we can't store the absolute path, our configuration is based on + # relative file names. + contents = f.read() + if b'TESTRUNNER' in contents: # test__monkey_patching.py + # XXX: Rework this to avoid importing. + to_import.append(qualified_name) + else: + cmd = [sys.executable, '-u'] + if PYPY and PY2: + # Doesn't seem to be an env var for this + cmd.extend(('-X', 'track-resources')) + if package: + # Using a package is the best way to work with coverage 5 + # when we specify 'source = ' + cmd.append('-m' + qualified_name) + else: + cmd.append(filename) + + options = DEFAULT_RUN_OPTIONS.copy() + options.update(configured_test_options.get(filename, {})) + to_process.append((cmd, options)) + + os.chdir(olddir) + # When we actually execute, do so from the original directory, + # this helps find setup.py + for qualified_name in to_import: + module = importlib.import_module(qualified_name) + for cmd, options in module.TESTRUNNER(): + if remove_options(cmd)[-1] in ignore: + continue + to_process.append((cmd, options)) + + return to_process + + +def remove_options(lst): + return [x for x in lst if x and not x.startswith('-')] + +def load_list_from_file(filename, package): + result = [] + if filename: + with open(_package_relative_filename(filename, package)) as f: + for x in f: + x = x.split('#', 1)[0].strip() + if x: + result.append(x) + return result + + +def matches(possibilities, command, include_flaky=True): + if isinstance(command, list): + command = ' '.join(command) + for line in possibilities: + if not include_flaky and line.startswith('FLAKY '): + continue + line = line.replace('FLAKY ', '') + # Our configs are still mostly written in terms of file names, + # but the non-monkey tests are now using package names. + # Strip off '.py' from filenames to see if we match a module. + # XXX: This could be much better. Our command needs better structure. + if command.endswith(' ' + line) or command.endswith(line.replace(".py", '')): + return True + return False + + +def format_seconds(seconds): + if seconds < 20: + return '%.1fs' % seconds + seconds = str(timedelta(seconds=round(seconds))) + if seconds.startswith('0:'): + seconds = seconds[2:] + return seconds + + +def report(total, failed, passed, exit=True, took=None, + configured_failing_tests=(), + total_cases=0, total_skipped=0): + # pylint:disable=redefined-builtin,too-many-branches,too-many-locals + runtimelog = util.runtimelog + if runtimelog: + log('\nLongest-running tests:') + runtimelog.sort() + length = len('%.1f' % -runtimelog[0][0]) + frmt = '%' + str(length) + '.1f seconds: %s' + for delta, name in runtimelog[:5]: + log(frmt, -delta, name) + if took: + took = ' in %s' % format_seconds(took) + else: + took = '' + + failed_expected = [] + failed_unexpected = [] + passed_unexpected = [] + + for name in passed: + if matches(configured_failing_tests, name, include_flaky=False): + passed_unexpected.append(name) + + if passed_unexpected: + log('\n%s/%s unexpected passes', len(passed_unexpected), total, color='error') + print_list(passed_unexpected) + + if failed: + log('\n%s/%s tests failed%s', len(failed), total, took) + + for name in failed: + if matches(configured_failing_tests, name, include_flaky=True): + failed_expected.append(name) + else: + failed_unexpected.append(name) + + if failed_expected: + log('\n%s/%s expected failures', len(failed_expected), total) + print_list(failed_expected) + + if failed_unexpected: + log('\n%s/%s unexpected failures', len(failed_unexpected), total, color='error') + print_list(failed_unexpected) + else: + log( + '\nRan %s tests%s in %s files%s', + total_cases, + util._colorize('skipped', " (skipped=%d)" % total_skipped) if total_skipped else '', + total, + took, + ) + + if exit: + if failed_unexpected: + sys.exit(min(100, len(failed_unexpected))) + if passed_unexpected: + sys.exit(101) + if total <= 0: + sys.exit('No tests found.') + + +def print_list(lst): + for name in lst: + log(' - %s', name) + +def _setup_environ(debug=False): + if 'PYTHONWARNINGS' not in os.environ and not sys.warnoptions: + + # action:message:category:module:line + os.environ['PYTHONWARNINGS'] = ','.join([ + # Enable default warnings such as ResourceWarning. + 'default', + # On Python 3[.6], the system site.py module has + # "open(fullname, 'rU')" which produces the warning that + # 'U' is deprecated, so ignore warnings from site.py + 'ignore:::site:', + # pkgutil on Python 2 complains about missing __init__.py + 'ignore:::pkgutil', + # importlib/_bootstrap.py likes to spit out "ImportWarning: + # can't resolve package from __spec__ or __package__, falling + # back on __name__ and __path__". I have no idea what that means, but it seems harmless + # and is annoying. + 'ignore:::importlib._bootstrap:', + 'ignore:::importlib._bootstrap_external:', + # importing ABCs from collections, not collections.abc + 'ignore:::pkg_resources._vendor.pyparsing:', + ]) + + if 'PYTHONFAULTHANDLER' not in os.environ: + os.environ['PYTHONFAULTHANDLER'] = 'true' + + if 'GEVENT_DEBUG' not in os.environ and debug: + os.environ['GEVENT_DEBUG'] = 'debug' + + if 'PYTHONTRACEMALLOC' not in os.environ: + os.environ['PYTHONTRACEMALLOC'] = '10' + + if 'PYTHONDEVMODE' not in os.environ: + # Python 3.7 + os.environ['PYTHONDEVMODE'] = '1' + + if 'PYTHONMALLOC' not in os.environ: + # Python 3.6 + os.environ['PYTHONMALLOC'] = 'debug' + + + +def main(): + # pylint:disable=too-many-locals,too-many-statements + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--ignore') + parser.add_argument('--discover', action='store_true') + parser.add_argument('--full', action='store_true') + parser.add_argument('--config', default='known_failures.py') + parser.add_argument('--failfast', action='store_true') + parser.add_argument("--coverage", action="store_true") + parser.add_argument("--quiet", action="store_true", default=True) + parser.add_argument("--verbose", action="store_false", dest='quiet') + parser.add_argument("--debug", action="store_true", default=False) + parser.add_argument("--package", default="gevent.tests") + parser.add_argument('tests', nargs='*') + options = parser.parse_args() + FAILING_TESTS = [] + IGNORED_TESTS = [] + RUN_ALONE = [] + TEST_FILE_OPTIONS = {} + + coverage = False + if options.coverage or os.environ.get("GEVENTTEST_COVERAGE"): + coverage = True + os.environ['COVERAGE_PROCESS_START'] = os.path.abspath(".coveragerc") + if PYPY: + os.environ['COVERAGE_PROCESS_START'] = os.path.abspath(".coveragerc-pypy") + this_dir = os.path.dirname(__file__) + site_dir = os.path.join(this_dir, 'coveragesite') + site_dir = os.path.abspath(site_dir) + os.environ['PYTHONPATH'] = site_dir + os.pathsep + os.environ.get("PYTHONPATH", "") + # We change directory often, use an absolute path to keep all the + # coverage files (which will have distinct suffixes because of parallel=true in .coveragerc + # in this directory; makes them easier to combine and use with coverage report) + os.environ['COVERAGE_FILE'] = os.path.abspath(".") + os.sep + ".coverage" + print("Enabling coverage to", os.environ['COVERAGE_FILE'], "with site", site_dir) + + _setup_environ(debug=options.debug) + + if options.config: + config = {} + options.config = _package_relative_filename(options.config, options.package) + with open(options.config) as f: + config_data = f.read() + six.exec_(config_data, config) + FAILING_TESTS = config['FAILING_TESTS'] + IGNORED_TESTS = config['IGNORED_TESTS'] + RUN_ALONE = config['RUN_ALONE'] + TEST_FILE_OPTIONS = config['TEST_FILE_OPTIONS'] + IGNORE_COVERAGE = config['IGNORE_COVERAGE'] + + + tests = discover( + options.tests, + ignore_files=options.ignore, + ignored=IGNORED_TESTS, + coverage=coverage, + package=options.package, + configured_ignore_coverage=IGNORE_COVERAGE, + configured_test_options=TEST_FILE_OPTIONS, + ) + if options.discover: + for cmd, options in tests: + print(util.getname(cmd, env=options.get('env'), setenv=options.get('setenv'))) + print('%s tests found.' % len(tests)) + else: + if PYPY and RESOLVER_ARES: + # XXX: Add a way to force these. + print("Not running tests on pypy with c-ares; not a supported configuration") + return + if options.package: + # Put this directory on the path so relative imports work. + package_dir = _dir_from_package_name(options.package) + os.environ['PYTHONPATH'] = os.environ.get('PYTHONPATH', "") + os.pathsep + package_dir + run_many( + tests, + configured_failing_tests=FAILING_TESTS, + failfast=options.failfast, + quiet=options.quiet, + configured_run_alone_tests=RUN_ALONE, + ) + + +if __name__ == '__main__': + main() diff --git a/src/gevent/testing/timing.py b/src/gevent/testing/timing.py new file mode 100644 index 0000000..bc62edf --- /dev/null +++ b/src/gevent/testing/timing.py @@ -0,0 +1,139 @@ +# Copyright (c) 2018 gevent community +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import time + +import gevent + +from . import sysinfo +from . import leakcheck +from .testcase import TestCase + +SMALLEST_RELIABLE_DELAY = 0.001 # 1ms, because of libuv + +SMALL_TICK = 0.01 +SMALL_TICK_MIN_ADJ = SMALLEST_RELIABLE_DELAY +SMALL_TICK_MAX_ADJ = 0.11 +if sysinfo.RUNNING_ON_APPVEYOR: + # Timing resolution is extremely poor on Appveyor + # and subject to jitter. + SMALL_TICK_MAX_ADJ = 1.5 + + +LARGE_TICK = 0.2 +LARGE_TICK_MIN_ADJ = LARGE_TICK / 2.0 +LARGE_TICK_MAX_ADJ = SMALL_TICK_MAX_ADJ + + +class _DelayWaitMixin(object): + + _default_wait_timeout = SMALL_TICK + _default_delay_min_adj = SMALL_TICK_MIN_ADJ + _default_delay_max_adj = SMALL_TICK_MAX_ADJ + + def wait(self, timeout): + raise NotImplementedError('override me in subclass') + + def _check_delay_bounds(self, timeout, delay, + delay_min_adj=None, + delay_max_adj=None): + delay_min_adj = self._default_delay_min_adj if not delay_min_adj else delay_min_adj + delay_max_adj = self._default_delay_max_adj if not delay_max_adj else delay_max_adj + self.assertTimeWithinRange(delay, + timeout - delay_min_adj, + timeout + delay_max_adj) + + def _wait_and_check(self, timeout=None): + if timeout is None: + timeout = self._default_wait_timeout + + # gevent.timer instances have a 'seconds' attribute, + # otherwise it's the raw number + seconds = getattr(timeout, 'seconds', timeout) + + gevent.get_hub().loop.update_now() + start = time.time() + try: + result = self.wait(timeout) + finally: + self._check_delay_bounds(seconds, time.time() - start, + self._default_delay_min_adj, + self._default_delay_max_adj) + return result + + def test_outer_timeout_is_not_lost(self): + timeout = gevent.Timeout.start_new(SMALLEST_RELIABLE_DELAY, ref=False) + try: + with self.assertRaises(gevent.Timeout) as exc: + self.wait(timeout=1) + self.assertIs(exc.exception, timeout) + finally: + timeout.close() + + +class AbstractGenericWaitTestCase(_DelayWaitMixin, TestCase): + # pylint:disable=abstract-method + + _default_wait_timeout = LARGE_TICK + _default_delay_min_adj = LARGE_TICK_MIN_ADJ + _default_delay_max_adj = LARGE_TICK_MAX_ADJ + + @leakcheck.ignores_leakcheck # waiting checks can be very sensitive to timing + def test_returns_none_after_timeout(self): + result = self._wait_and_check() + # join and wait simply return after timeout expires + self.assertIsNone(result) + + +class AbstractGenericGetTestCase(_DelayWaitMixin, TestCase): + # pylint:disable=abstract-method + + Timeout = gevent.Timeout + + def cleanup(self): + pass + + def test_raises_timeout_number(self): + with self.assertRaises(self.Timeout): + self._wait_and_check(timeout=SMALL_TICK) + # get raises Timeout after timeout expired + self.cleanup() + + def test_raises_timeout_Timeout(self): + timeout = gevent.Timeout(self._default_wait_timeout) + try: + self._wait_and_check(timeout=timeout) + except gevent.Timeout as ex: + self.assertIs(ex, timeout) + finally: + timeout.close() + self.cleanup() + + def test_raises_timeout_Timeout_exc_customized(self): + error = RuntimeError('expected error') + timeout = gevent.Timeout(self._default_wait_timeout, exception=error) + try: + with self.assertRaises(RuntimeError) as exc: + self._wait_and_check(timeout=timeout) + + self.assertIs(exc.exception, error) + self.cleanup() + finally: + timeout.close() diff --git a/src/gevent/testing/util.py b/src/gevent/testing/util.py new file mode 100644 index 0000000..afcd83b --- /dev/null +++ b/src/gevent/testing/util.py @@ -0,0 +1,453 @@ +from __future__ import print_function, absolute_import, division +import re +import sys +import os +from . import six +import traceback +import unittest +import threading +import subprocess +import time + +# pylint: disable=broad-except,attribute-defined-outside-init + +runtimelog = [] +MIN_RUNTIME = 1.0 +BUFFER_OUTPUT = False +QUIET = False + + +class Popen(subprocess.Popen): + + def __enter__(self): + return self + + def __exit__(self, *args): + kill(self) + + +# Coloring code based on zope.testrunner + +# These colors are carefully chosen to have enough contrast +# on terminals with both black and white background. +_colorscheme = { + 'normal': 'normal', + 'default': 'default', + 'info': 'normal', + 'suboptimal-behaviour': 'magenta', + 'error': 'brightred', + 'number': 'green', + 'slow-test': 'brightmagenta', + 'ok-number': 'green', + 'error-number': 'brightred', + 'filename': 'lightblue', + 'lineno': 'lightred', + 'testname': 'lightcyan', + 'failed-example': 'cyan', + 'expected-output': 'green', + 'actual-output': 'red', + 'character-diffs': 'magenta', + 'diff-chunk': 'magenta', + 'exception': 'red', + 'skipped': 'brightyellow', +} + +_prefixes = [ + ('dark', '0;'), + ('light', '1;'), + ('bright', '1;'), + ('bold', '1;'), +] + +_colorcodes = { + 'default': 0, + 'normal': 0, + 'black': 30, + 'red': 31, + 'green': 32, + 'brown': 33, 'yellow': 33, + 'blue': 34, + 'magenta': 35, + 'cyan': 36, + 'grey': 37, 'gray': 37, 'white': 37 +} + +def _color_code(color): + prefix_code = '' + for prefix, code in _prefixes: + if color.startswith(prefix): + color = color[len(prefix):] + prefix_code = code + break + color_code = _colorcodes[color] + return '\033[%s%sm' % (prefix_code, color_code) + +def _color(what): + return _color_code(_colorscheme[what]) + +def _colorize(what, message, normal='normal'): + return _color(what) + message + _color(normal) + +def log(message, *args, **kwargs): + color = kwargs.pop('color', 'normal') + try: + if args: + string = message % args + else: + string = message + except Exception: + traceback.print_exc() + try: + string = '%r %% %r\n\n' % (message, args) + except Exception: + pass + try: + string = _colorize('exception', string) + sys.stderr.write(string) + except Exception: + traceback.print_exc() + else: + string = _colorize(color, string) + sys.stderr.write(string + '\n') + + +def killpg(pid): + if not hasattr(os, 'killpg'): + return + try: + return os.killpg(pid, 9) + except OSError as ex: + if ex.errno != 3: + log('killpg(%r, 9) failed: %s: %s', pid, type(ex).__name__, ex) + except Exception as ex: + log('killpg(%r, 9) failed: %s: %s', pid, type(ex).__name__, ex) + + +def kill_processtree(pid): + ignore_msg = 'ERROR: The process "%s" not found.' % pid + err = subprocess.Popen('taskkill /F /PID %s /T' % pid, stderr=subprocess.PIPE).communicate()[1] + if err and err.strip() not in [ignore_msg, '']: + log('%r', err) + + +def _kill(popen): + if hasattr(popen, 'kill'): + try: + popen.kill() + except OSError as ex: + if ex.errno == 3: # No such process + return + if ex.errno == 13: # Permission denied (translated from windows error 5: "Access is denied") + return + raise + else: + try: + os.kill(popen.pid, 9) + except EnvironmentError: + pass + + +def kill(popen): + if popen.timer is not None: + popen.timer.cancel() + if popen.poll() is not None: + return + popen.was_killed = True + try: + if getattr(popen, 'setpgrp_enabled', None): + killpg(popen.pid) + elif sys.platform.startswith('win'): + kill_processtree(popen.pid) + except Exception: + traceback.print_exc() + try: + _kill(popen) + except Exception: + traceback.print_exc() + try: + popen.wait() + except Exception: + traceback.print_exc() + + +def getname(command, env=None, setenv=None): + result = [] + + env = (env or os.environ).copy() + env.update(setenv or {}) + + for key, value in sorted(env.items()): + if key.startswith('GEVENT'): + result.append('%s=%s' % (key, value)) + + if isinstance(command, six.string_types): + result.append(command) + else: + result.extend(command) + + return ' '.join(result) + + +def start(command, quiet=False, **kwargs): + timeout = kwargs.pop('timeout', None) + preexec_fn = None + if not os.environ.get('DO_NOT_SETPGRP'): + preexec_fn = getattr(os, 'setpgrp', None) + env = kwargs.pop('env', None) + setenv = kwargs.pop('setenv', None) or {} + name = getname(command, env=env, setenv=setenv) + if preexec_fn is not None: + setenv['DO_NOT_SETPGRP'] = '1' + if setenv: + if env: + env = env.copy() + else: + env = os.environ.copy() + env.update(setenv) + + if not quiet: + log('+ %s', name) + popen = Popen(command, preexec_fn=preexec_fn, env=env, **kwargs) + popen.name = name + popen.setpgrp_enabled = preexec_fn is not None + popen.was_killed = False + popen.timer = None + if timeout is not None: + t = threading.Timer(timeout, kill, args=(popen, )) + t.setDaemon(True) + t.start() + popen.timer = t + return popen + + +class RunResult(object): + + def __init__(self, code, + output=None, name=None, + run_count=0, skipped_count=0): + self.code = code + self.output = output + self.name = name + self.run_count = run_count + self.skipped_count = skipped_count + + + def __bool__(self): + return bool(self.code) + + __nonzero__ = __bool__ + + def __int__(self): + return self.code + + +def _should_show_warning_output(out): + if 'Warning' in out: + # Strip out some patterns we specifically do not + # care about. + # from test.support for monkey-patched tests + out = out.replace('Warning -- reap_children', 'NADA') + out = out.replace("Warning -- threading_cleanup", 'NADA') + + # The below *could* be done with sophisticated enough warning + # filters passed to the children + + # collections.abc is the new home; setuptools uses the old one, + # as does dnspython + out = out.replace("DeprecationWarning: Using or importing the ABCs", 'NADA') + # libuv poor timer resolution + out = out.replace('UserWarning: libuv only supports', 'NADA') + # Packages on Python 2 + out = out.replace('ImportWarning: Not importing directory', 'NADA') + return 'Warning' in out + +output_lock = threading.Lock() + +def _find_test_status(took, out): + status = '[took %.1fs%s]' + skipped = '' + run_count = 0 + skipped_count = 0 + if out: + m = re.search(r"Ran (\d+) tests in", out) + if m: + result = out[m.start():m.end()] + status = status.replace('took', result) + run_count = int(out[m.start(1):m.end(1)]) + + m = re.search(r' \(skipped=(\d+)\)$', out) + if m: + skipped = _colorize('skipped', out[m.start():m.end()]) + skipped_count = int(out[m.start(1):m.end(1)]) + status = status % (took, skipped) + if took > 10: + status = _colorize('slow-test', status) + return status, run_count, skipped_count + + +def run(command, **kwargs): # pylint:disable=too-many-locals + buffer_output = kwargs.pop('buffer_output', BUFFER_OUTPUT) + quiet = kwargs.pop('quiet', QUIET) + verbose = not quiet + nested = kwargs.pop('nested', False) + if buffer_output: + assert 'stdout' not in kwargs and 'stderr' not in kwargs, kwargs + kwargs['stderr'] = subprocess.STDOUT + kwargs['stdout'] = subprocess.PIPE + popen = start(command, quiet=nested, **kwargs) + name = popen.name + try: + time_start = time.time() + out, err = popen.communicate() + took = time.time() - time_start + if popen.was_killed or popen.poll() is None: + result = 'TIMEOUT' + else: + result = popen.poll() + finally: + kill(popen) + assert not err + with output_lock: # pylint:disable=not-context-manager + failed = bool(result) + if out: + out = out.strip() + out = out if isinstance(out, str) else out.decode('utf-8', 'ignore') + if out and (failed or verbose or _should_show_warning_output(out)): + if out: + out = ' ' + out.replace('\n', '\n ') + out = out.rstrip() + out += '\n' + log('| %s\n%s', name, out) + status, run_count, skipped_count = _find_test_status(took, out) + if result: + log('! %s [code %s] %s', name, result, status, color='error') + elif not nested: + log('- %s %s', name, status) + if took >= MIN_RUNTIME: + runtimelog.append((-took, name)) + return RunResult(result, out, name, run_count, skipped_count) + + +class NoSetupPyFound(Exception): + "Raised by find_setup_py_above" + +def find_setup_py_above(a_file): + "Return the directory containing setup.py somewhere above *a_file*" + root = os.path.dirname(os.path.abspath(a_file)) + while not os.path.exists(os.path.join(root, 'setup.py')): + prev, root = root, os.path.dirname(root) + if root == prev: + # Let's avoid infinite loops at root + raise NoSetupPyFound('could not find my setup.py above %r' % (a_file,)) + return root + +def search_for_setup_py(a_file=None, a_module_name=None, a_class=None, climb_cwd=True): + if a_file is not None: + try: + return find_setup_py_above(a_file) + except NoSetupPyFound: + pass + + if a_class is not None: + try: + return find_setup_py_above(sys.modules[a_class.__module__].__file__) + except NoSetupPyFound: + pass + + if a_module_name is not None: + try: + return find_setup_py_above(sys.modules[a_module_name].__file__) + except NoSetupPyFound: + pass + + if climb_cwd: + return find_setup_py_above("./dne") + + raise NoSetupPyFound("After checking %r" % (locals(),)) + + +class ExampleMixin(object): + "Something that uses the examples/ directory" + + def find_setup_py(self): + "Return the directory containing setup.py" + return search_for_setup_py( + a_file=__file__, + a_class=type(self) + ) + + @property + def cwd(self): + try: + root = self.find_setup_py() + except NoSetupPyFound as e: + raise unittest.SkipTest("Unable to locate file/dir to run: %s" % (e,)) + + return os.path.join(root, 'examples') + +class TestServer(ExampleMixin, + unittest.TestCase): + args = [] + before_delay = 3 + after_delay = 0.5 + popen = None + server = None # subclasses define this to be the path to the server.py + start_kwargs = None + + def start(self): + try: + kwargs = self.start_kwargs or {} + return start([sys.executable, '-u', self.server] + self.args, cwd=self.cwd, **kwargs) + except NoSetupPyFound as e: + raise unittest.SkipTest("Unable to locate file/dir to run: %s" % (e,)) + + def running_server(self): + from contextlib import contextmanager + + @contextmanager + def running_server(): + with self.start() as popen: + self.popen = popen + self.before() + yield + self.after() + return running_server() + + def test(self): + with self.running_server(): + self._run_all_tests() + + def before(self): + if self.before_delay is not None: + time.sleep(self.before_delay) + assert self.popen.poll() is None, '%s died with code %s' % (self.server, self.popen.poll(), ) + + def after(self): + if self.after_delay is not None: + time.sleep(self.after_delay) + assert self.popen.poll() is None, '%s died with code %s' % (self.server, self.popen.poll(), ) + + def _run_all_tests(self): + ran = False + for method in sorted(dir(self)): + if method.startswith('_test'): + function = getattr(self, method) + if callable(function): + function() + ran = True + assert ran + + +class alarm(threading.Thread): + # can't use signal.alarm because of Windows + + def __init__(self, timeout): + threading.Thread.__init__(self) + self.setDaemon(True) + self.timeout = timeout + self.start() + + def run(self): + time.sleep(self.timeout) + sys.stderr.write('Timeout.\n') + os._exit(5) diff --git a/src/gevent/tests/2_7_keycert.pem b/src/gevent/tests/2_7_keycert.pem new file mode 100644 index 0000000..64318aa --- /dev/null +++ b/src/gevent/tests/2_7_keycert.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/gevent/tests/__init__.py b/src/gevent/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/gevent/tests/__main__.py b/src/gevent/tests/__main__.py new file mode 100644 index 0000000..e43891f --- /dev/null +++ b/src/gevent/tests/__main__.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +from __future__ import print_function, absolute_import, division + +if __name__ == '__main__': + from gevent.testing import testrunner + testrunner.main() diff --git a/src/gevent/tests/_blocks_at_top_level.py b/src/gevent/tests/_blocks_at_top_level.py new file mode 100644 index 0000000..9f907aa --- /dev/null +++ b/src/gevent/tests/_blocks_at_top_level.py @@ -0,0 +1,3 @@ +from gevent import sleep +sleep(0.01) +x = "done" diff --git a/src/gevent/tests/_import_import_patch.py b/src/gevent/tests/_import_import_patch.py new file mode 100644 index 0000000..aa85abd --- /dev/null +++ b/src/gevent/tests/_import_import_patch.py @@ -0,0 +1 @@ +__import__('_import_patch') diff --git a/src/gevent/tests/_import_patch.py b/src/gevent/tests/_import_patch.py new file mode 100644 index 0000000..9d7cc3c --- /dev/null +++ b/src/gevent/tests/_import_patch.py @@ -0,0 +1,2 @@ +import gevent.monkey +gevent.monkey.patch_all() diff --git a/src/gevent/tests/_import_wait.py b/src/gevent/tests/_import_wait.py new file mode 100644 index 0000000..80850a5 --- /dev/null +++ b/src/gevent/tests/_import_wait.py @@ -0,0 +1,26 @@ +# test__import_wait.py calls this via an import statement, +# so all of this is happening with import locks held (especially on py2) +import gevent + + +def fn2(): + return 2 + + +# A blocking function doesn't raise LoopExit +def fn(): + return gevent.wait([gevent.spawn(fn2), gevent.spawn(fn2)]) + +gevent.spawn(fn).get() + + +# Marshalling the traceback across greenlets doesn't +# raise LoopExit +def raise_name_error(): + raise NameError("ThisIsExpected") + +try: + gevent.spawn(raise_name_error).get() + raise AssertionError("Should fail") +except NameError as e: + x = e diff --git a/src/gevent/tests/_imports_at_top_level.py b/src/gevent/tests/_imports_at_top_level.py new file mode 100644 index 0000000..d11f66b --- /dev/null +++ b/src/gevent/tests/_imports_at_top_level.py @@ -0,0 +1,2 @@ +# We simply import a stdlib module +__import__('netrc') diff --git a/src/gevent/tests/_imports_imports_at_top_level.py b/src/gevent/tests/_imports_imports_at_top_level.py new file mode 100644 index 0000000..00bbf51 --- /dev/null +++ b/src/gevent/tests/_imports_imports_at_top_level.py @@ -0,0 +1,13 @@ +import gevent + +# For reproducing #728: We spawn a greenlet at import time, +# that itself wants to import, and wait on it at import time. +# If we're the only greenlet running, and locks aren't granular +# enough, this results in a LoopExit (and also a lock deadlock) + + +def f(): + __import__('_imports_at_top_level') + +g = gevent.spawn(f) +g.get() diff --git a/src/gevent/tests/badcert.pem b/src/gevent/tests/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/gevent/tests/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/gevent/tests/badkey.pem b/src/gevent/tests/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/gevent/tests/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/gevent/tests/getaddrinfo_module.py b/src/gevent/tests/getaddrinfo_module.py new file mode 100644 index 0000000..75a25df --- /dev/null +++ b/src/gevent/tests/getaddrinfo_module.py @@ -0,0 +1,4 @@ +import socket +import gevent.socket as gevent_socket + +gevent_socket.getaddrinfo(u'gevent.org', None, socket.AF_INET) diff --git a/src/gevent/tests/hosts_file.txt b/src/gevent/tests/hosts_file.txt new file mode 100644 index 0000000..a33da68 --- /dev/null +++ b/src/gevent/tests/hosts_file.txt @@ -0,0 +1,10351 @@ +## +# Host Database +# +# localhost is used to configure the loopback interface +# when the system is booting. Do not change this entry. +## +127.0.0.1 localhost Localhost localhost.localdomain testsite.mc.com mathcounts.mc.com platform.osu.edu + +255.255.255.255 broadcasthost +::1 localhost +fe80::1%lo0 localhost +172.178.0.51 excelsior excelsior.example.com +162.168.8.27 memoryprime.local memoryprime +122.168.9.64 isy.local isy + + +192.168.1.172 drivefoo.local + +172.168.15.95 aragefoo.local +172.168.15.105 livgfoo.local +172.168.16.109 upsirsfoo.local +172.168.15.140 bacorthfoo.local +172.168.15.142 bacouthfoo.local +172.168.16.144 drisfoo.local +172.168.15.152 nghborfoo.local +172.168.15.154 fntfoo.local +172.168.18.151 as.local + +# Internals +146.120.241.22 ds3 +146.120.241.23 ds4 +146.120.241.21 ds2 +146.120.241.20 ds1 + + +# Not blocked by Mar 18 2013 +0.0.0.0 h.ppjol.com +0.0.0.0 s.ppjol.net +0.0.0.0 yayfollowers.com +0.0.0.0 pagead2.googlesyndication.com +0.0.0.0 www.googletagservices.com +0.0.0.0 cdn.teads.tv +0.0.0.0 js.moatads.com +0.0.0.0 cdn2.teads.tv + + +# This hosts file is brought to you by Dan Pollock and can be found at +# http://someonewhocares.org/hosts/zero/ + +# +# For example, to block unpleasant pages, try: +0.0.0.0 goatse.cx # More information on sites such as +0.0.0.0 www.goatse.cx # these can be found in this article +0.0.0.0 oralse.cx # en.wikipedia.org/wiki/List_of_shock_sites +0.0.0.0 www.oralse.cx +0.0.0.0 goatse.ca +0.0.0.0 www.goatse.ca +0.0.0.0 oralse.ca +0.0.0.0 www.oralse.ca +0.0.0.0 goat.cx +0.0.0.0 www.goat.cx +0.0.0.0 goatse.ru +0.0.0.0 www.goatse.ru + +0.0.0.0 1girl1pitcher.com +0.0.0.0 1girl1pitcher.org +0.0.0.0 1guy1cock.com +0.0.0.0 1man1jar.org +0.0.0.0 1man2needles.com +0.0.0.0 1priest1nun.com +0.0.0.0 2girls1cup.com +0.0.0.0 2girls1cup-free.com +0.0.0.0 2girls1cup.nl +0.0.0.0 2girls1cup.ws +0.0.0.0 2girls1finger.com +0.0.0.0 2girls1finger.org +0.0.0.0 2guys1stump.org +0.0.0.0 3guys1hammer.ws +0.0.0.0 4girlsfingerpaint.com +0.0.0.0 4girlsfingerpaint.org +0.0.0.0 bagslap.com +0.0.0.0 ballsack.org +0.0.0.0 bluewaffle.biz +0.0.0.0 bottleguy.com +0.0.0.0 bowlgirl.com +0.0.0.0 cadaver.org +0.0.0.0 clownsong.com +0.0.0.0 copyright-reform.info +0.0.0.0 cshacks.partycat.us +0.0.0.0 cyberscat.com +0.0.0.0 dadparty.com +0.0.0.0 detroithardcore.com +0.0.0.0 donotwatch.org +0.0.0.0 dontwatch.us +0.0.0.0 eelsoup.net +0.0.0.0 fruitlauncher.com +0.0.0.0 fuck.org +0.0.0.0 funnelchair.com +0.0.0.0 goatse.bz +0.0.0.0 goatsegirl.org +0.0.0.0 goatse.ru +0.0.0.0 hai2u.com +0.0.0.0 homewares.org +0.0.0.0 howtotroll.org +0.0.0.0 japscat.org +0.0.0.0 jiztini.com +0.0.0.0 junecleeland.com +0.0.0.0 kids-in-sandbox.com +0.0.0.0 kidsinsandbox.info +0.0.0.0 lemonparty.biz +0.0.0.0 lemonparty.org +0.0.0.0 lolhello.com +0.0.0.0 loltrain.com +0.0.0.0 meatspin.biz +0.0.0.0 meatspin.com +0.0.0.0 merryholidays.org +0.0.0.0 milkfountain.com +0.0.0.0 mudfall.com +0.0.0.0 mudmonster.org +0.0.0.0 nimp.org +0.0.0.0 nobrain.dk +0.0.0.0 nutabuse.com +0.0.0.0 octopusgirl.com +0.0.0.0 on.nimp.org +0.0.0.0 painolympics.info +0.0.0.0 phonejapan.com +0.0.0.0 pressurespot.com +0.0.0.0 prolapseman.com +0.0.0.0 scrollbelow.com +0.0.0.0 selfpwn.org +0.0.0.0 sexitnow.com +0.0.0.0 sourmath.com +0.0.0.0 suckdude.com +0.0.0.0 thatsjustgay.com +0.0.0.0 thatsphucked.com +0.0.0.0 thehomo.org +0.0.0.0 themacuser.org +0.0.0.0 thepounder.com +0.0.0.0 tubgirl.me +0.0.0.0 tubgirl.org +0.0.0.0 turdgasm.com +0.0.0.0 vomitgirl.org +0.0.0.0 walkthedinosaur.com +0.0.0.0 whipcrack.org +0.0.0.0 wormgush.com +0.0.0.0 www.1girl1pitcher.org +0.0.0.0 www.1guy1cock.com +0.0.0.0 www.1man1jar.org +0.0.0.0 www.1man2needles.com +0.0.0.0 www.1priest1nun.com +0.0.0.0 www.2girls1cup-free.com +0.0.0.0 www.2girls1cup.nl +0.0.0.0 www.2girls1cup.ws +0.0.0.0 www.2girls1finger.org +0.0.0.0 www.2guys1stump.org +0.0.0.0 www.3guys1hammer.ws +0.0.0.0 www.4girlsfingerpaint.org +0.0.0.0 www.bagslap.com +0.0.0.0 www.ballsack.org +0.0.0.0 www.bluewaffle.biz +0.0.0.0 www.bottleguy.com +0.0.0.0 www.bowlgirl.com +0.0.0.0 www.cadaver.org +0.0.0.0 www.clownsong.com +0.0.0.0 www.copyright-reform.info +0.0.0.0 www.cshacks.partycat.us +0.0.0.0 www.cyberscat.com +0.0.0.0 www.dadparty.com +0.0.0.0 www.detroithardcore.com +0.0.0.0 www.donotwatch.org +0.0.0.0 www.dontwatch.us +0.0.0.0 www.eelsoup.net +0.0.0.0 www.fruitlauncher.com +0.0.0.0 www.fuck.org +0.0.0.0 www.funnelchair.com +0.0.0.0 www.goatse.bz +0.0.0.0 www.goatsegirl.org +0.0.0.0 www.goatse.ru +0.0.0.0 www.hai2u.com +0.0.0.0 www.homewares.org +0.0.0.0 www.howtotroll.org +0.0.0.0 www.japscat.org +0.0.0.0 www.jiztini.com +0.0.0.0 www.junecleeland.com +0.0.0.0 www.kids-in-sandbox.com +0.0.0.0 www.kidsinsandbox.info +0.0.0.0 www.lemonparty.biz +0.0.0.0 www.lemonparty.org +0.0.0.0 www.lolhello.com +0.0.0.0 www.loltrain.com +0.0.0.0 www.meatspin.biz +0.0.0.0 www.meatspin.com +0.0.0.0 www.merryholidays.org +0.0.0.0 www.milkfountain.com +0.0.0.0 www.mudfall.com +0.0.0.0 www.mudmonster.org +0.0.0.0 www.nimp.org +0.0.0.0 www.nobrain.dk +0.0.0.0 www.nutabuse.com +0.0.0.0 www.octopusgirl.com +0.0.0.0 www.on.nimp.org +0.0.0.0 www.painolympics.info +0.0.0.0 www.phonejapan.com +0.0.0.0 www.pressurespot.com +0.0.0.0 www.prolapseman.com +0.0.0.0 www.punishtube.com +0.0.0.0 www.scrollbelow.com +0.0.0.0 www.selfpwn.org +0.0.0.0 www.sourmath.com +0.0.0.0 www.suckdude.com +0.0.0.0 www.thatsjustgay.com +0.0.0.0 www.thatsphucked.com +0.0.0.0 www.theexgirlfriends.com +0.0.0.0 www.thehomo.org +0.0.0.0 www.themacuser.org +0.0.0.0 www.thepounder.com +0.0.0.0 www.tubgirl.me +0.0.0.0 www.tubgirl.org +0.0.0.0 www.turdgasm.com +0.0.0.0 www.vomitgirl.org +0.0.0.0 www.walkthedinosaur.com +0.0.0.0 www.whipcrack.org +0.0.0.0 www.wormgush.com +0.0.0.0 www.xvideoslive.com +0.0.0.0 www.y8.com +0.0.0.0 www.youaresogay.com +0.0.0.0 www.ypmate.com +0.0.0.0 www.zentastic.com +0.0.0.0 youaresogay.com +0.0.0.0 zentastic.com +# + +0.0.0.0 ads234.com +0.0.0.0 ads345.com +0.0.0.0 www.ads234.com +0.0.0.0 www.ads345.com +# + + +# + +# +0.0.0.0 auto.search.msn.com # Microsoft uses this server to redirect + # mistyped URLs to search engines. They + # log all such errors. +0.0.0.0 sitefinder.verisign.com # Verisign has joined the game +0.0.0.0 sitefinder-idn.verisign.com # of trying to hijack mistyped + # URLs to their site. + # May break iOS Game Center. + +0.0.0.0 s0.2mdn.net # This may interfere with some streaming + # video on sites such as cbc.ca +0.0.0.0 ad.doubleclick.net # This may interefere with www.sears.com + # and potentially other sites. +0.0.0.0 media.fastclick.net # Likewise, this may interfere with some +0.0.0.0 cdn.fastclick.net # sites. +0.0.0.0 ebay.doubleclick.net # may interfere with ebay +#0.0.0.0 google-analytics.com # breaks some sites +#0.0.0.0 ssl.google-analytics.com +#0.0.0.0 www.google-analytics.l.google.com +0.0.0.0 stat.livejournal.com # There are reports that this may mess + # up CSS on livejournal +0.0.0.0 stats.surfaid.ihost.com # This has been known cause + # problems with NPR.org +0.0.0.0 www.google-analytics.com # breaks some sites +0.0.0.0 ads.imeem.com # Seems to interfere with the functioning of imeem.com +# + +0.0.0.0 006.free-counter.co.uk +0.0.0.0 006.freecounters.co.uk +0.0.0.0 06272002-dbase.hitcountz.net # Web bugs in spam +0.0.0.0 123counter.mycomputer.com +0.0.0.0 123counter.superstats.com +0.0.0.0 1ca.cqcounter.com +0.0.0.0 1uk.cqcounter.com +0.0.0.0 1us.cqcounter.com +0.0.0.0 1xxx.cqcounter.com +0.0.0.0 2001-007.com +0.0.0.0 3bc3fd26-91cf-46b2-8ec6-b1559ada0079.statcamp.net +0.0.0.0 3ps.go.com +0.0.0.0 4-counter.com +0.0.0.0 a796faee-7163-4757-a34f-e5b48cada4cb.statcamp.net +0.0.0.0 abscbn.spinbox.net +0.0.0.0 activity.serving-sys.com #eyeblaster.com +0.0.0.0 adadvisor.net +0.0.0.0 adclient.rottentomatoes.com +0.0.0.0 adcodes.aim4media.com +0.0.0.0 adcounter.globeandmail.com +0.0.0.0 adcounter.theglobeandmail.com +0.0.0.0 addfreestats.com +0.0.0.0 ademails.com +0.0.0.0 adlog.com.com # Used by Ziff Davis to serve + # ads and track users across + # the com.com family of sites +0.0.0.0 ad-logics.com +0.0.0.0 admanmail.com +0.0.0.0 adopt.specificclick.net +0.0.0.0 ads.tiscali.com +0.0.0.0 ads.tiscali.it +0.0.0.0 adult.foxcounter.com +0.0.0.0 affiliate.ab1trk.com +0.0.0.0 affiliate.irotracker.com +0.0.0.0 ai062.insightexpress.com +0.0.0.0 ai078.insightexpressai.com +0.0.0.0 ai087.insightexpress.com +0.0.0.0 ai113.insightexpressai.com +0.0.0.0 ai125.insightexpressai.com +0.0.0.0 alpha.easy-hit-counters.com +0.0.0.0 amateur.xxxcounter.com +0.0.0.0 amer.hops.glbdns.microsoft.com +0.0.0.0 amer.rel.msn.com +0.0.0.0 analytics.msnbc.msn.com +0.0.0.0 analytics.prx.org +0.0.0.0 anm.intelli-direct.com +0.0.0.0 ant.conversive.nl +0.0.0.0 apac.rel.msn.com +0.0.0.0 api.bizographics.com +0.0.0.0 apprep.smartscreen.microsoft.com +0.0.0.0 app.yesware.com +0.0.0.0 arbo.hit.gemius.pl +0.0.0.0 au052.insightexpress.com +0.0.0.0 auspice.augur.io +0.0.0.0 au.track.decideinteractive.com +0.0.0.0 a.visualrevenue.com +0.0.0.0 banner.0catch.com +0.0.0.0 banners.webcounter.com +0.0.0.0 beacon-1.newrelic.com +0.0.0.0 beacon.scorecardresearch.com +0.0.0.0 beacons.hottraffic.nl +0.0.0.0 be.sitestat.com +0.0.0.0 best-search.cc #spyware +0.0.0.0 beta.easy-hit-counter.com +0.0.0.0 beta.easy-hit-counters.com +0.0.0.0 beta.easyhitcounters.com +0.0.0.0 bilbo.counted.com +0.0.0.0 bin.clearspring.com +0.0.0.0 birta.stats.is +0.0.0.0 bluekai.com +0.0.0.0 bluestreak.com +0.0.0.0 bookproplus.com +0.0.0.0 broadcastpc.tv +0.0.0.0 report.broadcastpc.tv +0.0.0.0 www.broadcastpc.tv +0.0.0.0 bserver.blick.com +0.0.0.0 bstats.adbrite.com +0.0.0.0 b.stats.paypal.com +0.0.0.0 by.optimost.com +0.0.0.0 c10.statcounter.com +0.0.0.0 c11.statcounter.com +0.0.0.0 c12.statcounter.com +0.0.0.0 c13.statcounter.com +0.0.0.0 c14.statcounter.com +0.0.0.0 c15.statcounter.com +0.0.0.0 c16.statcounter.com +0.0.0.0 c17.statcounter.com +0.0.0.0 c1.statcounter.com +0.0.0.0 c1.thecounter.com +0.0.0.0 c1.thecounter.de +0.0.0.0 c1.xxxcounter.com +0.0.0.0 c2.gostats.com +0.0.0.0 c2.thecounter.com +0.0.0.0 c2.thecounter.de +0.0.0.0 c2.xxxcounter.com +0.0.0.0 c3.gostats.com +0.0.0.0 c3.statcounter.com +0.0.0.0 c3.thecounter.com +0.0.0.0 c3.xxxcounter.com +0.0.0.0 c4.myway.com +0.0.0.0 c4.statcounter.com +0.0.0.0 c5.statcounter.com +0.0.0.0 c6.statcounter.com +0.0.0.0 c7.statcounter.com +0.0.0.0 c8.statcounter.com +0.0.0.0 c9.statcounter.com +0.0.0.0 ca.cqcounter.com +0.0.0.0 cashcounter.com +0.0.0.0 cb1.counterbot.com +0.0.0.0 cdn.krxd.net +0.0.0.0 cdn.oggifinogi.com +0.0.0.0 cdn.taboolasyndication.com +0.0.0.0 cdxbin.vulnerap.com +0.0.0.0 cf.addthis.com +0.0.0.0 cgicounter.onlinehome.de +0.0.0.0 cgicounter.puretec.de +0.0.0.0 cgi.hotstat.nl +0.0.0.0 cgi.sexlist.com +0.0.0.0 ci-mpsnare.iovation.com # See http://www.codingthewheel.com/archives/online-gambling-privacy-iesnare +0.0.0.0 citrix.tradedoubler.com +0.0.0.0 cjt1.net +0.0.0.0 click.atdmt.com +0.0.0.0 clickauditor.net +0.0.0.0 click.fivemtn.com +0.0.0.0 click.investopedia.com +0.0.0.0 click.jve.net +0.0.0.0 clickmeter.com +0.0.0.0 click.payserve.com +0.0.0.0 clicks.emarketmakers.com +0.0.0.0 click.silvercash.com +0.0.0.0 clicks.m4n.nl +0.0.0.0 clicks.natwest.com +0.0.0.0 clickspring.net #used by a spyware product called PurityScan +0.0.0.0 clicks.rbs.co.uk +0.0.0.0 clicktrack.onlineemailmarketing.com +0.0.0.0 clicktracks.webmetro.com +0.0.0.0 clit10.sextracker.com +0.0.0.0 clit13.sextracker.com +0.0.0.0 clit15.sextracker.com +0.0.0.0 clit2.sextracker.com +0.0.0.0 clit4.sextracker.com +0.0.0.0 clit6.sextracker.com +0.0.0.0 clit7.sextracker.com +0.0.0.0 clit8.sextracker.com +0.0.0.0 clit9.sextracker.com +0.0.0.0 clk.aboxdeal.com +0.0.0.0 clk.relestar.com +0.0.0.0 cnn.entertainment.printthis.clickability.com +0.0.0.0 cnt.xcounter.com +0.0.0.0 collector.deepmetrix.com +0.0.0.0 collector.newsx.cc +0.0.0.0 connectionlead.com +0.0.0.0 connexity.net +0.0.0.0 cookies.cmpnet.com +0.0.0.0 count.channeladvisor.com +0.0.0.0 counter10.bravenet.com +0.0.0.0 counter10.sextracker.be +0.0.0.0 counter10.sextracker.com +0.0.0.0 counter11.bravenet.com +0.0.0.0 counter11.sextracker.be +0.0.0.0 counter11.sextracker.com +0.0.0.0 counter.123counts.com +0.0.0.0 counter12.bravenet.com +0.0.0.0 counter12.sextracker.be +0.0.0.0 counter12.sextracker.com +0.0.0.0 counter13.bravenet.com +0.0.0.0 counter13.sextracker.be +0.0.0.0 counter13.sextracker.com +0.0.0.0 counter14.bravenet.com +0.0.0.0 counter14.sextracker.be +0.0.0.0 counter14.sextracker.com +0.0.0.0 counter15.bravenet.com +0.0.0.0 counter15.sextracker.be +0.0.0.0 counter15.sextracker.com +0.0.0.0 counter16.bravenet.com +0.0.0.0 counter16.sextracker.be +0.0.0.0 counter16.sextracker.com +0.0.0.0 counter17.bravenet.com +0.0.0.0 counter18.bravenet.com +0.0.0.0 counter19.bravenet.com +0.0.0.0 counter1.bravenet.com +0.0.0.0 counter1.sextracker.be +0.0.0.0 counter1.sextracker.com +0.0.0.0 counter.1stblaze.com +0.0.0.0 counter20.bravenet.com +0.0.0.0 counter21.bravenet.com +0.0.0.0 counter22.bravenet.com +0.0.0.0 counter23.bravenet.com +0.0.0.0 counter24.bravenet.com +0.0.0.0 counter25.bravenet.com +0.0.0.0 counter26.bravenet.com +0.0.0.0 counter27.bravenet.com +0.0.0.0 counter28.bravenet.com +0.0.0.0 counter29.bravenet.com +0.0.0.0 counter2.bravenet.com +0.0.0.0 counter2.freeware.de +0.0.0.0 counter2.hitslink.com +0.0.0.0 counter2.sextracker.be +0.0.0.0 counter2.sextracker.com +0.0.0.0 counter30.bravenet.com +0.0.0.0 counter31.bravenet.com +0.0.0.0 counter32.bravenet.com +0.0.0.0 counter33.bravenet.com +0.0.0.0 counter34.bravenet.com +0.0.0.0 counter35.bravenet.com +0.0.0.0 counter36.bravenet.com +0.0.0.0 counter37.bravenet.com +0.0.0.0 counter38.bravenet.com +0.0.0.0 counter39.bravenet.com +0.0.0.0 counter3.bravenet.com +0.0.0.0 counter3.sextracker.be +0.0.0.0 counter3.sextracker.com +0.0.0.0 counter40.bravenet.com +0.0.0.0 counter41.bravenet.com +0.0.0.0 counter42.bravenet.com +0.0.0.0 counter43.bravenet.com +0.0.0.0 counter44.bravenet.com +0.0.0.0 counter45.bravenet.com +0.0.0.0 counter46.bravenet.com +0.0.0.0 counter47.bravenet.com +0.0.0.0 counter48.bravenet.com +0.0.0.0 counter49.bravenet.com +0.0.0.0 counter4all.dk +0.0.0.0 counter4.bravenet.com +0.0.0.0 counter4.sextracker.be +0.0.0.0 counter4.sextracker.com +0.0.0.0 counter4u.de +0.0.0.0 counter50.bravenet.com +0.0.0.0 counter5.bravenet.com +0.0.0.0 counter5.sextracker.be +0.0.0.0 counter5.sextracker.com +0.0.0.0 counter6.bravenet.com +0.0.0.0 counter6.sextracker.be +0.0.0.0 counter6.sextracker.com +0.0.0.0 counter7.bravenet.com +0.0.0.0 counter7.sextracker.be +0.0.0.0 counter7.sextracker.com +0.0.0.0 counter8.bravenet.com +0.0.0.0 counter8.sextracker.be +0.0.0.0 counter8.sextracker.com +0.0.0.0 counter9.bravenet.com +0.0.0.0 counter9.sextracker.be +0.0.0.0 counter9.sextracker.com +0.0.0.0 counter.aaddzz.com +0.0.0.0 counterad.de +0.0.0.0 counter.adultcheck.com +0.0.0.0 counter.adultrevenueservice.com +0.0.0.0 counter.advancewebhosting.com +0.0.0.0 counter.aport.ru +0.0.0.0 counteraport.spylog.com +0.0.0.0 counter.asexhound.com +0.0.0.0 counter.avp2000.com +0.0.0.0 counter.bizland.com +0.0.0.0 counter.bloke.com +0.0.0.0 counterbot.com +0.0.0.0 counter.clubnet.ro +0.0.0.0 counter.cnw.cz +0.0.0.0 countercrazy.com +0.0.0.0 counter.credo.ru +0.0.0.0 counter.cz +0.0.0.0 counter.digits.com +0.0.0.0 counter.dreamhost.com +0.0.0.0 counter.e-audit.it +0.0.0.0 counter.execpc.com +0.0.0.0 counter.fateback.com +0.0.0.0 counter.gamespy.com +0.0.0.0 counter.hitslink.com +0.0.0.0 counter.hitslinks.com +0.0.0.0 counter.htmlvalidator.com +0.0.0.0 counter.impressur.com +0.0.0.0 counter.inetusa.com +0.0.0.0 counter.inti.fr +0.0.0.0 counter.kaspersky.com +0.0.0.0 counter.letssingit.com +0.0.0.0 counter.mtree.com +0.0.0.0 counter.mycomputer.com +0.0.0.0 counter.netmore.net +0.0.0.0 counter.nope.dk +0.0.0.0 counter.nowlinux.com +0.0.0.0 counter.pcgames.de +0.0.0.0 counter.rambler.ru +0.0.0.0 counters.auctionhelper.com # comment these +0.0.0.0 counters.auctionwatch.com # out to allow +0.0.0.0 counters.auctiva.com # tracking by +0.0.0.0 counters.honesty.com # ebay users +0.0.0.0 counter.search.bg +0.0.0.0 counter.sexhound.nl +0.0.0.0 counters.gigya.com +0.0.0.0 counter.sparklit.com +0.0.0.0 counter.superstats.com +0.0.0.0 counter.surfcounters.com +0.0.0.0 counters.xaraonline.com +0.0.0.0 counter.times.lv +0.0.0.0 counter.topping.com.ua +0.0.0.0 counter.tripod.com +0.0.0.0 counter.uq.edu.au +0.0.0.0 counter.w3open.com +0.0.0.0 counter.webcom.com +0.0.0.0 counter.webmedia.pl +0.0.0.0 counter.webtrends.com +0.0.0.0 counter.webtrends.net +0.0.0.0 counter.xxxcool.com +0.0.0.0 counter.yadro.ru +0.0.0.0 count.paycounter.com +0.0.0.0 count.xhit.com +0.0.0.0 cs.sexcounter.com +0.0.0.0 c.statcounter.com +0.0.0.0 c.thecounter.de +0.0.0.0 cw.nu +0.0.0.0 cyseal.cyveillance.com +0.0.0.0 cz3.clickzs.com +0.0.0.0 cz6.clickzs.com +0.0.0.0 da.ce.bd.a9.top.list.ru +0.0.0.0 da.newstogram.com +0.0.0.0 data2.perf.overture.com +0.0.0.0 data.coremetrics.com +0.0.0.0 data.webads.co.nz +0.0.0.0 dclk.haaretz.co.il +0.0.0.0 dclk.themarker.com +0.0.0.0 dclk.themarketer.com +0.0.0.0 delivery.loopingclick.com +0.0.0.0 de.sitestat.com +0.0.0.0 didtheyreadit.com # email bugs +0.0.0.0 digistats.westjet.com +0.0.0.0 dimeprice.com # "spam bugs" +0.0.0.0 directads.mcafee.com +0.0.0.0 dotcomsecrets.com +0.0.0.0 dpbolvw.net +0.0.0.0 ds.247realmedia.com +0.0.0.0 ds.amateurmatch.com +0.0.0.0 dwclick.com +0.0.0.0 e-2dj6wfk4ehd5afq.stats.esomniture.com +0.0.0.0 e-2dj6wfk4ggdzkbo.stats.esomniture.com +0.0.0.0 e-2dj6wfk4gkcpiep.stats.esomniture.com +0.0.0.0 e-2dj6wfk4skdpogo.stats.esomniture.com +0.0.0.0 e-2dj6wfkiakdjgcp.stats.esomniture.com +0.0.0.0 e-2dj6wfkiepczoeo.stats.esomniture.com +0.0.0.0 e-2dj6wfkikjd5glq.stats.esomniture.com +0.0.0.0 e-2dj6wfkiokc5odp.stats.esomniture.com +0.0.0.0 e-2dj6wfkiqjcpifp.stats.esomniture.com +0.0.0.0 e-2dj6wfkocjczedo.stats.esomniture.com +0.0.0.0 e-2dj6wfkokjajseq.stats.esomniture.com +0.0.0.0 e-2dj6wfkowkdjokp.stats.esomniture.com +0.0.0.0 e-2dj6wfkykpazskq.stats.esomniture.com +0.0.0.0 e-2dj6wflicocjklo.stats.esomniture.com +0.0.0.0 e-2dj6wfligpd5iap.stats.esomniture.com +0.0.0.0 e-2dj6wflikgdpodo.stats.esomniture.com +0.0.0.0 e-2dj6wflikiajslo.stats.esomniture.com +0.0.0.0 e-2dj6wflioldzoco.stats.esomniture.com +0.0.0.0 e-2dj6wfliwpczolp.stats.esomniture.com +0.0.0.0 e-2dj6wfloenczmkq.stats.esomniture.com +0.0.0.0 e-2dj6wflokmajedo.stats.esomniture.com +0.0.0.0 e-2dj6wfloqgc5mho.stats.esomniture.com +0.0.0.0 e-2dj6wfmysgdzobo.stats.esomniture.com +0.0.0.0 e-2dj6wgkigpcjedo.stats.esomniture.com +0.0.0.0 e-2dj6wgkisnd5abo.stats.esomniture.com +0.0.0.0 e-2dj6wgkoandzieq.stats.esomniture.com +0.0.0.0 e-2dj6wgkycpcpsgq.stats.esomniture.com +0.0.0.0 e-2dj6wgkyepajmeo.stats.esomniture.com +0.0.0.0 e-2dj6wgkyknd5sko.stats.esomniture.com +0.0.0.0 e-2dj6wgkyomdpalp.stats.esomniture.com +0.0.0.0 e-2dj6whkiandzkko.stats.esomniture.com +0.0.0.0 e-2dj6whkiepd5iho.stats.esomniture.com +0.0.0.0 e-2dj6whkiwjdjwhq.stats.esomniture.com +0.0.0.0 e-2dj6wjk4amd5mfp.stats.esomniture.com +0.0.0.0 e-2dj6wjk4kkcjalp.stats.esomniture.com +0.0.0.0 e-2dj6wjk4ukazebo.stats.esomniture.com +0.0.0.0 e-2dj6wjkosodpmaq.stats.esomniture.com +0.0.0.0 e-2dj6wjkouhd5eao.stats.esomniture.com +0.0.0.0 e-2dj6wjkowhd5ggo.stats.esomniture.com +0.0.0.0 e-2dj6wjkowjajcbo.stats.esomniture.com +0.0.0.0 e-2dj6wjkyandpogq.stats.esomniture.com +0.0.0.0 e-2dj6wjkycpdzckp.stats.esomniture.com +0.0.0.0 e-2dj6wjkyqmdzcgo.stats.esomniture.com +0.0.0.0 e-2dj6wjkysndzigp.stats.esomniture.com +0.0.0.0 e-2dj6wjl4qhd5kdo.stats.esomniture.com +0.0.0.0 e-2dj6wjlichdjoep.stats.esomniture.com +0.0.0.0 e-2dj6wjliehcjglp.stats.esomniture.com +0.0.0.0 e-2dj6wjlignajgaq.stats.esomniture.com +0.0.0.0 e-2dj6wjloagc5oco.stats.esomniture.com +0.0.0.0 e-2dj6wjlougazmao.stats.esomniture.com +0.0.0.0 e-2dj6wjlyamdpogo.stats.esomniture.com +0.0.0.0 e-2dj6wjlyckcpelq.stats.esomniture.com +0.0.0.0 e-2dj6wjlyeodjkcq.stats.esomniture.com +0.0.0.0 e-2dj6wjlygkd5ecq.stats.esomniture.com +0.0.0.0 e-2dj6wjmiekc5olo.stats.esomniture.com +0.0.0.0 e-2dj6wjmyehd5mfo.stats.esomniture.com +0.0.0.0 e-2dj6wjmyooczoeo.stats.esomniture.com +0.0.0.0 e-2dj6wjny-1idzkh.stats.esomniture.com +0.0.0.0 e-2dj6wjnyagcpkko.stats.esomniture.com +0.0.0.0 e-2dj6wjnyeocpcdo.stats.esomniture.com +0.0.0.0 e-2dj6wjnygidjskq.stats.esomniture.com +0.0.0.0 e-2dj6wjnyqkajabp.stats.esomniture.com +0.0.0.0 easy-web-stats.com +0.0.0.0 ecestats.theglobeandmail.com +0.0.0.0 economisttestcollect.insightfirst.com +0.0.0.0 ehg.fedex.com +0.0.0.0 eitbglobal.ojdinteractiva.com +0.0.0.0 emea.rel.msn.com +0.0.0.0 engine.cmmeglobal.com +0.0.0.0 enoratraffic.com +0.0.0.0 entry-stats.huffingtonpost.com +0.0.0.0 environmentalgraffiti.uk.intellitxt.com +0.0.0.0 e-n.y-1shz2prbmdj6wvny-1sez2pra2dj6wjmyepdzadpwudj6x9ny-1seq-2-2.stats.esomniture.com +0.0.0.0 e-ny.a-1shz2prbmdj6wvny-1sez2pra2dj6wjny-1jcpgbowsdj6x9ny-1seq-2-2.stats.esomniture.com +0.0.0.0 es.optimost.com +0.0.0.0 fastcounter.bcentral.com +0.0.0.0 fastcounter.com +0.0.0.0 fastcounter.linkexchange.com +0.0.0.0 fastcounter.linkexchange.net +0.0.0.0 fastcounter.linkexchange.nl +0.0.0.0 fastcounter.onlinehoster.net +0.0.0.0 fastwebcounter.com +0.0.0.0 fcstats.bcentral.com +0.0.0.0 fi.sitestat.com +0.0.0.0 fl01.ct2.comclick.com +0.0.0.0 flycast.com +0.0.0.0 forbescollect.247realmedia.com +0.0.0.0 formalyzer.com +0.0.0.0 foxcounter.com +0.0.0.0 free-counter.5u.com +0.0.0.0 freeinvisiblecounters.com +0.0.0.0 freestats.com +0.0.0.0 freewebcounter.com +0.0.0.0 free.xxxcounter.com +0.0.0.0 fs10.fusestats.com +0.0.0.0 ft2.autonomycloud.com +0.0.0.0 gapl.hit.gemius.pl +0.0.0.0 gator.com +0.0.0.0 gcounter.hosting4u.net +0.0.0.0 gd.mlb.com +0.0.0.0 geocounter.net +0.0.0.0 gkkzngresullts.com +0.0.0.0 go-in-search.net +0.0.0.0 goldstats.com +0.0.0.0 googfle.com +0.0.0.0 googletagservices.com +0.0.0.0 gostats.com +0.0.0.0 grafix.xxxcounter.com +0.0.0.0 gtcc1.acecounter.com +0.0.0.0 g-wizzads.net +0.0.0.0 hc2.humanclick.com +0.0.0.0 hit10.hotlog.ru +0.0.0.0 hit2.hotlog.ru +0.0.0.0 hit37.chark.dk +0.0.0.0 hit37.chart.dk +0.0.0.0 hit39.chart.dk +0.0.0.0 hit5.hotlog.ru +0.0.0.0 hit8.hotlog.ru +0.0.0.0 hit.clickaider.com +0.0.0.0 hit-counter.5u.com +0.0.0.0 hit-counter.udub.com +0.0.0.0 hits.guardian.co.uk +0.0.0.0 hits.gureport.co.uk +0.0.0.0 hits.nextstat.com +0.0.0.0 hits.webstat.com +0.0.0.0 hitx.statistics.ro +0.0.0.0 hst.tradedoubler.com +0.0.0.0 htm.freelogs.com +0.0.0.0 http300.edge.ru4.com +0.0.0.0 iccee.com +0.0.0.0 idm.hit.gemius.pl +0.0.0.0 ieplugin.com +0.0.0.0 iesnare.com # See http://www.codingthewheel.com/archives/online-gambling-privacy-iesnare +0.0.0.0 ig.insightgrit.com +0.0.0.0 ih.constantcontacts.com +0.0.0.0 i.kissmetrics.com # http://www.wired.com/epicenter/2011/07/undeletable-cookie/ +0.0.0.0 ilead.itrack.it +0.0.0.0 image.masterstats.com +0.0.0.0 images1.paycounter.com +0.0.0.0 images-aud.freshmeat.net +0.0.0.0 images-aud.slashdot.org +0.0.0.0 images-aud.sourceforge.net +0.0.0.0 images.dailydiscounts.com # "spam bugs" +0.0.0.0 images.itchydawg.com +0.0.0.0 impacts.alliancehub.com # "spam bugs" +0.0.0.0 impch.tradedoubler.com +0.0.0.0 imp.clickability.com +0.0.0.0 impde.tradedoubler.com +0.0.0.0 impdk.tradedoubler.com +0.0.0.0 impes.tradedoubler.com +0.0.0.0 impfr.tradedoubler.com +0.0.0.0 impgb.tradedoubler.com +0.0.0.0 impie.tradedoubler.com +0.0.0.0 impit.tradedouble.com +0.0.0.0 impit.tradedoubler.com +0.0.0.0 impnl.tradedoubler.com +0.0.0.0 impno.tradedoubler.com +0.0.0.0 impse.tradedoubler.com +0.0.0.0 in.paycounter.com +0.0.0.0 insightfirst.com +0.0.0.0 insightxe.looksmart.com +0.0.0.0 int.sitestat.com +0.0.0.0 in.webcounter.cc +0.0.0.0 iprocollect.realmedia.com +0.0.0.0 izitracking.izimailing.com +0.0.0.0 jgoyk.cjt1.net +0.0.0.0 jkearns.freestats.com +0.0.0.0 journalism.uk.smarttargetting.com +0.0.0.0 js.cybermonitor.com +0.0.0.0 jsonlinecollect.247realmedia.com +0.0.0.0 js.revsci.net +0.0.0.0 kissmetrics.com +0.0.0.0 kqzyfj.com +0.0.0.0 kt4.kliptracker.com +0.0.0.0 leadpub.com +0.0.0.0 liapentruromania.ro +0.0.0.0 lin31.metriweb.be +0.0.0.0 linkcounter.com +0.0.0.0 linkcounter.pornosite.com +0.0.0.0 link.masterstats.com +0.0.0.0 linktrack.bravenet.com +0.0.0.0 livestats.atlanta-airport.com +#0.0.0.0 ll.a.hulu.com # Uncomment to block Hulu. +0.0.0.0 loc1.hitsprocessor.com +0.0.0.0 log1.countomat.com +0.0.0.0 log4.quintelligence.com +0.0.0.0 log999.goo.ne.jp +0.0.0.0 loga.xiti.com +0.0.0.0 log.btopenworld.com +0.0.0.0 logc146.xiti.com +0.0.0.0 logc1.xiti.com +0.0.0.0 logc22.xiti.com +0.0.0.0 logc25.xiti.com +0.0.0.0 logc31.xiti.com +0.0.0.0 log.clickstream.co.za +0.0.0.0 log.hankooki.com +0.0.0.0 logi6.xiti.com +0.0.0.0 logi7.xiti.com +0.0.0.0 logi8.xiti.com +0.0.0.0 logp3.xiti.com +0.0.0.0 logs.comics.com +0.0.0.0 logs.eresmas.com +0.0.0.0 logs.eresmas.net +0.0.0.0 log.statistici.ro +0.0.0.0 logv14.xiti.com +0.0.0.0 logv17.xiti.com +0.0.0.0 logv18.xiti.com +0.0.0.0 logv21.xiti.com +0.0.0.0 logv25.xiti.com +0.0.0.0 logv27.xiti.com +0.0.0.0 logv29.xiti.com +0.0.0.0 logv32.xiti.com +0.0.0.0 logv4.xiti.com +0.0.0.0 logv.xiti.com +0.0.0.0 luycos.com +0.0.0.0 lycoscollect.247realmedia.com +0.0.0.0 lycoscollect.realmedia.com +0.0.0.0 m1.nedstatbasic.net +0.0.0.0 m1.webstats4u.com +0.0.0.0 mailcheckisp.biz # "spam bugs" +0.0.0.0 mama128.valuehost.ru +0.0.0.0 marketscore.com +0.0.0.0 mature.xxxcounter.com +0.0.0.0 mbox5.offermatica.com +0.0.0.0 media101.sitebrand.com +0.0.0.0 media.superstats.com +0.0.0.0 mediatrack.revenue.net +0.0.0.0 metric.10best.com +0.0.0.0 metric.infoworld.com +0.0.0.0 metric.nationalgeographic.com +0.0.0.0 metric.nwsource.com +0.0.0.0 metric.olivegarden.com +0.0.0.0 metrics2.pricegrabber.com +0.0.0.0 metrics.accuweather.com +0.0.0.0 metrics.al.com +0.0.0.0 metrics.boston.com +0.0.0.0 metrics.cbc.ca +0.0.0.0 metrics.cleveland.com +0.0.0.0 metrics.cnn.com +0.0.0.0 metrics.csmonitor.com +0.0.0.0 metrics.ctv.ca +0.0.0.0 metrics.dallasnews.com +0.0.0.0 metrics.elle.com +0.0.0.0 metrics.experts-exchange.com +0.0.0.0 metrics.fandome.com +0.0.0.0 metrics.foxnews.com +0.0.0.0 metrics.gap.com +0.0.0.0 metrics.health.com +0.0.0.0 metrics.hrblock.com +0.0.0.0 metrics.ioffer.com +0.0.0.0 metrics.ireport.com +0.0.0.0 metrics.kgw.com +0.0.0.0 metrics.ktvb.com +0.0.0.0 metrics.landolakes.com +0.0.0.0 metrics.lhj.com +0.0.0.0 metrics.maxim.com +0.0.0.0 metrics.mlive.com +0.0.0.0 metrics.mms.mavenapps.net +0.0.0.0 metrics.mpora.com +0.0.0.0 metrics.mysanantonio.com +0.0.0.0 metrics.nba.com +0.0.0.0 metrics.nextgov.com +0.0.0.0 metrics.nfl.com +0.0.0.0 metrics.npr.org +0.0.0.0 metrics.oclc.org +0.0.0.0 metrics.olivegarden.com +0.0.0.0 metrics.oregonlive.com +0.0.0.0 metrics.parallels.com +0.0.0.0 metrics.performancing.com +0.0.0.0 metrics.philly.com +0.0.0.0 metrics.post-gazette.com +0.0.0.0 metrics.premiere.com +0.0.0.0 metrics.rottentomatoes.com +0.0.0.0 metrics.sephora.com +0.0.0.0 metrics.soundandvision.com +0.0.0.0 metrics.soundandvisionmag.com +0.0.0.0 metrics.sun.com +0.0.0.0 metric.starz.com +0.0.0.0 metrics.technologyreview.com +0.0.0.0 metrics.theatlantic.com +0.0.0.0 metrics.thedailybeast.com +0.0.0.0 metrics.thefa.com +0.0.0.0 metrics.thefrisky.com +0.0.0.0 metrics.thenation.com +0.0.0.0 metrics.theweathernetwork.com +#0.0.0.0 metrics.ticketmaster.com # interferes with logging in to ticketmaster.com +0.0.0.0 metrics.tmz.com +0.0.0.0 metrics.toyota.com +0.0.0.0 metrics.tulsaworld.com +0.0.0.0 metrics.washingtonpost.com +0.0.0.0 metrics.whitepages.com +0.0.0.0 metrics.womansday.com +0.0.0.0 metrics.yellowpages.com +0.0.0.0 metrics.yousendit.com +0.0.0.0 metric.thenation.com +0.0.0.0 mng1.clickalyzer.com +0.0.0.0 monster.gostats.com +0.0.0.0 mpsnare.iesnare.com # See http://www.codingthewheel.com/archives/online-gambling-privacy-iesnare +0.0.0.0 msn1.com +0.0.0.0 msnm.com +0.0.0.0 mt122.mtree.com +0.0.0.0 mtcount.channeladvisor.com +0.0.0.0 mtrcs.popcap.com +0.0.0.0 mtv.247realmedia.com +0.0.0.0 multi1.rmuk.co.uk +0.0.0.0 mvs.mediavantage.de +0.0.0.0 mvtracker.com +0.0.0.0 mystats.com +0.0.0.0 nedstat.s0.nl +0.0.0.0 nethit-free.nl +0.0.0.0 net-radar.com +0.0.0.0 network.leadpub.com +0.0.0.0 nextgenstats.com +0.0.0.0 nht-2.extreme-dm.com +0.0.0.0 nl.nedstatbasic.net +0.0.0.0 nl.sitestat.com +0.0.0.0 o.addthis.com +0.0.0.0 objects.tremormedia.com +0.0.0.0 okcounter.com +0.0.0.0 omniture.theglobeandmail.com +0.0.0.0 one.123counters.com +0.0.0.0 oss-crules.marketscore.com +0.0.0.0 oss-survey.marketscore.com +0.0.0.0 ostats.mozilla.com +0.0.0.0 other.xxxcounter.com +0.0.0.0 out.true-counter.com +0.0.0.0 p.addthis.com +0.0.0.0 partner.alerts.aol.com +0.0.0.0 partners.pantheranetwork.com +0.0.0.0 passpport.com +0.0.0.0 paxito.sitetracker.com +0.0.0.0 paycounter.com +0.0.0.0 pei-ads.thesmokingjacket.com +0.0.0.0 perso.estat.com +0.0.0.0 pf.tradedoubler.com +0.0.0.0 pings.blip.tv +0.0.0.0 pix02.revsci.net +0.0.0.0 pix03.revsci.net +0.0.0.0 pix04.revsci.net +0.0.0.0 pixel.invitemedia.com +0.0.0.0 pmg.ad-logics.com +0.0.0.0 pn2.adserver.yahoo.com +0.0.0.0 pointclicktrack.com +0.0.0.0 pong.qubitproducts.com +0.0.0.0 postclick.adcentriconline.com +0.0.0.0 postgazettecollect.247realmedia.com +0.0.0.0 precisioncounter.com +0.0.0.0 p.reuters.com +0.0.0.0 printmail.biz +0.0.0.0 prof.estat.com +0.0.0.0 pro.hit.gemius.pl +0.0.0.0 proxycfg.marketscore.com +0.0.0.0 proxy.ia2.marketscore.com +0.0.0.0 proxy.ia3.marketscore.com +0.0.0.0 proxy.ia4.marketscore.com +0.0.0.0 proxy.or3.marketscore.com +0.0.0.0 proxy.or4.marketscore.com +0.0.0.0 proxy.sj3.marketscore.com +0.0.0.0 proxy.sj4.marketscore.com +0.0.0.0 quantserve.com #: Ad Tracking, JavaScript, etc. +0.0.0.0 quareclk.com +0.0.0.0 raw.oggifinogi.com +0.0.0.0 r.clickdensity.com +0.0.0.0 remotrk.com +0.0.0.0 rightmedia.net +0.0.0.0 rightstats.com +0.0.0.0 roskatrack.roskadirect.com +0.0.0.0 rr1.xxxcounter.com +0.0.0.0 rr2.xxxcounter.com +0.0.0.0 rr3.xxxcounter.com +0.0.0.0 rr4.xxxcounter.com +0.0.0.0 rr5.xxxcounter.com +0.0.0.0 rr7.xxxcounter.com +0.0.0.0 rts.pgmediaserve.com +0.0.0.0 rts.phn.doublepimp.com +0.0.0.0 s10.histats.com +0.0.0.0 s10.sitemeter.com +0.0.0.0 s11.sitemeter.com +0.0.0.0 s12.sitemeter.com +0.0.0.0 s13.sitemeter.com +0.0.0.0 s14.sitemeter.com +0.0.0.0 s15.sitemeter.com +0.0.0.0 s16.sitemeter.com +0.0.0.0 s17.sitemeter.com +0.0.0.0 s18.sitemeter.com +0.0.0.0 s19.sitemeter.com +0.0.0.0 s1.shinystat.it +0.0.0.0 s1.thecounter.com +0.0.0.0 s20.sitemeter.com +0.0.0.0 s21.sitemeter.com +0.0.0.0 s22.sitemeter.com +0.0.0.0 s23.sitemeter.com +0.0.0.0 s24.sitemeter.com +0.0.0.0 s25.sitemeter.com +0.0.0.0 s26.sitemeter.com +0.0.0.0 s27.sitemeter.com +0.0.0.0 s28.sitemeter.com +0.0.0.0 s29.sitemeter.com +0.0.0.0 s2.statcounter.com +0.0.0.0 s2.youtube.com +0.0.0.0 s30.sitemeter.com +0.0.0.0 s31.sitemeter.com +0.0.0.0 s32.sitemeter.com +0.0.0.0 s33.sitemeter.com +0.0.0.0 s34.sitemeter.com +0.0.0.0 s35.sitemeter.com +0.0.0.0 s36.sitemeter.com +0.0.0.0 s37.sitemeter.com +0.0.0.0 s38.sitemeter.com +0.0.0.0 s39.sitemeter.com +0.0.0.0 s3.hit.stat.pl +0.0.0.0 s41.sitemeter.com +0.0.0.0 s42.sitemeter.com +0.0.0.0 s43.sitemeter.com +0.0.0.0 s44.sitemeter.com +0.0.0.0 s45.sitemeter.com +0.0.0.0 s46.sitemeter.com +0.0.0.0 s47.sitemeter.com +0.0.0.0 s48.sitemeter.com +0.0.0.0 s4.histats.com +0.0.0.0 s4.shinystat.com +0.0.0.0 s.clickability.com +0.0.0.0 scorecardresearch.com +0.0.0.0 scribe.twitter.com +0.0.0.0 scrooge.channelcincinnati.com +0.0.0.0 scrooge.channeloklahoma.com +0.0.0.0 scrooge.click10.com +0.0.0.0 scrooge.clickondetroit.com +0.0.0.0 scrooge.nbc11.com +0.0.0.0 scrooge.nbc4columbus.com +0.0.0.0 scrooge.nbc4.com +0.0.0.0 scrooge.nbcsandiego.com +0.0.0.0 scrooge.newsnet5.com +0.0.0.0 scrooge.thebostonchannel.com +0.0.0.0 scrooge.thedenverchannel.com +0.0.0.0 scrooge.theindychannel.com +0.0.0.0 scrooge.thekansascitychannel.com +0.0.0.0 scrooge.themilwaukeechannel.com +0.0.0.0 scrooge.theomahachannel.com +0.0.0.0 scrooge.wesh.com +0.0.0.0 scrooge.wftv.com +0.0.0.0 scrooge.wnbc.com +0.0.0.0 scrooge.wsoctv.com +0.0.0.0 scrooge.wtov9.com +0.0.0.0 sdc.rbistats.com +0.0.0.0 searchadv.com +0.0.0.0 sekel.ch +0.0.0.0 servedby.valuead.com +0.0.0.0 server10.opentracker.net +0.0.0.0 server11.opentracker.net +0.0.0.0 server12.opentracker.net +0.0.0.0 server13.opentracker.net +0.0.0.0 server14.opentracker.net +0.0.0.0 server15.opentracker.net +0.0.0.0 server16.opentracker.net +0.0.0.0 server17.opentracker.net +0.0.0.0 server18.opentracker.net +0.0.0.0 server1.opentracker.net +0.0.0.0 server2.opentracker.net +0.0.0.0 server3.opentracker.net +0.0.0.0 server3.web-stat.com +0.0.0.0 server4.opentracker.net +0.0.0.0 server5.opentracker.net +0.0.0.0 server6.opentracker.net +0.0.0.0 server7.opentracker.net +0.0.0.0 server8.opentracker.net +0.0.0.0 server9.opentracker.net +0.0.0.0 service.bfast.com +0.0.0.0 services.krxd.net +0.0.0.0 se.sitestat.com +0.0.0.0 sexcounter.com +0.0.0.0 seznam.hit.gemius.pl +0.0.0.0 showads.pubmatic.com +0.0.0.0 showcount.honest.com +0.0.0.0 sideshow.directtrack.com +0.0.0.0 sitestat.com +0.0.0.0 sitestats.tiscali.co.uk +0.0.0.0 sm1.sitemeter.com +0.0.0.0 sm2.sitemeter.com +0.0.0.0 sm3.sitemeter.com +0.0.0.0 sm4.sitemeter.com +0.0.0.0 sm5.sitemeter.com +0.0.0.0 sm6.sitemeter.com +0.0.0.0 sm7.sitemeter.com +0.0.0.0 sm8.sitemeter.com +0.0.0.0 sm9.sitemeter.com +0.0.0.0 smartstats.com +0.0.0.0 softcore.xxxcounter.com +0.0.0.0 sostats.mozilla.com +0.0.0.0 sovereign.sitetracker.com +0.0.0.0 spinbox.maccentral.com +0.0.0.0 spinbox.versiontracker.com +0.0.0.0 spklds.com +0.0.0.0 s.statistici.ro +0.0.0.0 s.stats.wordpress.com +0.0.0.0 ss.tiscali.com +0.0.0.0 ss.tiscali.it +0.0.0.0 st1.hit.gemius.pl +0.0.0.0 stags.peer39.net +0.0.0.0 stast2.gq.com +0.0.0.0 stat1.z-stat.com +0.0.0.0 stat3.cybermonitor.com +0.0.0.0 stat.4u.pl +0.0.0.0 stat.alibaba.com +0.0.0.0 statcounter.com +0.0.0.0 stat-counter.tass-online.ru +0.0.0.0 stat.discogs.com +0.0.0.0 static.kibboko.com +0.0.0.0 static.smni.com # Santa Monica - popunders +0.0.0.0 statik.topica.com +0.0.0.0 statistics.dynamicsitestats.com +0.0.0.0 statistics.elsevier.nl +0.0.0.0 statistics.reedbusiness.nl +0.0.0.0 statistics.theonion.com +0.0.0.0 statistik-gallup.net +0.0.0.0 stat.netmonitor.fi +0.0.0.0 stat.onestat.com +0.0.0.0 stats1.clicktracks.com +0.0.0.0 stats1.corusradio.com +0.0.0.0 stats1.in +0.0.0.0 stats.24ways.org +0.0.0.0 stats2.clicktracks.com +0.0.0.0 stats2.gourmet.com +0.0.0.0 stats2.newyorker.com +0.0.0.0 stats2.rte.ie +0.0.0.0 stats2.unrulymedia.com +0.0.0.0 stats2.vanityfair.com +0.0.0.0 stats4all.com +0.0.0.0 stats5.lightningcast.com +0.0.0.0 stats6.lightningcast.net +0.0.0.0 stats.absol.co.za +0.0.0.0 stats.adbrite.com +0.0.0.0 stats.adotube.com +0.0.0.0 stats.adultswim.com +0.0.0.0 stats.airfarewatchdog.com +0.0.0.0 stats.allliquid.com +0.0.0.0 stats.askmen.com +0.0.0.0 stats.bbc.co.uk +0.0.0.0 stats.becu.org +0.0.0.0 stats.big-boards.com +0.0.0.0 stats.blogoscoop.net +0.0.0.0 stats.bonzaii.no +0.0.0.0 stats.break.com +0.0.0.0 stats.brides.com +0.0.0.0 stats.buysellads.com +0.0.0.0 stats.cafepress.com +0.0.0.0 stats.canalblog.com +0.0.0.0 stats.cartoonnetwork.com +0.0.0.0 stats.channel4.com +0.0.0.0 stats.clickability.com +0.0.0.0 stats.concierge.com +0.0.0.0 stats.cts-bv.nl +0.0.0.0 stats.darkbluesea.com +0.0.0.0 stats.datahjaelp.net +0.0.0.0 stats.directnic.com +0.0.0.0 stats.dziennik.pl +0.0.0.0 stats.economist.com +0.0.0.0 stats.epicurious.com +0.0.0.0 statse.webtrendslive.com # Fortune.com among others +0.0.0.0 stats.examiner.com +0.0.0.0 stats.fairmont.com +0.0.0.0 stats.fastcompany.com +0.0.0.0 stats.foxcounter.com +0.0.0.0 stats.free-rein.net +0.0.0.0 stats.f-secure.com +0.0.0.0 stats.ft.com +0.0.0.0 stats.gamestop.com +0.0.0.0 stats.globesports.com +0.0.0.0 stats.groupninetyfour.com +0.0.0.0 stats.idsoft.com +0.0.0.0 stats.ign.com +0.0.0.0 stats.ilsemedia.nl +0.0.0.0 stats.independent.co.uk +0.0.0.0 stats.indexstats.com +0.0.0.0 stats.indextools.com +0.0.0.0 stats.investors.com +0.0.0.0 stats.iwebtrack.com +0.0.0.0 stats.jippii.com +0.0.0.0 stats.klsoft.com +0.0.0.0 stats.ladotstats.nl +0.0.0.0 stats.macworld.com +0.0.0.0 stats.magnify.net +0.0.0.0 stats.manticoretechnology.com +0.0.0.0 stats.mbamupdates.com +0.0.0.0 stats.millanusa.com +0.0.0.0 stats.nowpublic.com +0.0.0.0 stats.paycounter.com +0.0.0.0 stats.platinumbucks.com +0.0.0.0 stats.popscreen.com +0.0.0.0 stats.reinvigorate.net +0.0.0.0 stats.resellerratings.com +0.0.0.0 stats.revenue.net +0.0.0.0 stats.searchles.com +0.0.0.0 stats.ssa.gov +0.0.0.0 stats.superstats.com +0.0.0.0 stats.telegraph.co.uk +0.0.0.0 stats.thoughtcatalog.com +0.0.0.0 stats.townnews.com +0.0.0.0 stats.ultimate-webservices.com +0.0.0.0 stats.unionleader.com +0.0.0.0 stats.video.search.yahoo.com +0.0.0.0 stats.vodpod.com +0.0.0.0 stats.wordpress.com +0.0.0.0 stats.www.ibm.com +0.0.0.0 stats.yourminis.com +0.0.0.0 stat.webmedia.pl +0.0.0.0 stat.www.fi +0.0.0.0 stat.yellowtracker.com +0.0.0.0 stat.youku.com +0.0.0.0 stl.p.a1.traceworks.com +0.0.0.0 straighttangerine.cz.cc +0.0.0.0 st.sageanalyst.net +0.0.0.0 sugoicounter.com +0.0.0.0 superstats.com +0.0.0.0 s.youtube.com +#0.0.0.0 t2.hulu.com # Uncomment to block Hulu. +0.0.0.0 tagging.outrider.com +0.0.0.0 talkcity.realtracker.com +0.0.0.0 targetnet.com +0.0.0.0 tates.freestats.com +0.0.0.0 tcookie.usatoday.com +0.0.0.0 tcr.tynt.com # See http://daringfireball.net/2010/05/tynt_copy_paste_jerks +0.0.0.0 tgpcounter.freethumbnailgalleries.com +0.0.0.0 thecounter.com +0.0.0.0 the-counter.net +0.0.0.0 themecounter.com +0.0.0.0 the.sextracker.com +0.0.0.0 tipsurf.com +0.0.0.0 toolbarpartner.com +0.0.0.0 tools.spylog.ru +0.0.0.0 top.mail.ru +0.0.0.0 topstats.com +0.0.0.0 topstats.net +0.0.0.0 torstarcollect.247realmedia.com +0.0.0.0 track2.mybloglog.com +0.0.0.0 track.adform.com +0.0.0.0 track.adform.net +0.0.0.0 track.did-it.com +0.0.0.0 track.directleads.com +0.0.0.0 track.domainsponsor.com +0.0.0.0 track.effiliation.com +0.0.0.0 tracker.bonnint.net +0.0.0.0 tracker.clicktrade.com +0.0.0.0 tracker.idg.co.uk +0.0.0.0 tracker.mattel.com +0.0.0.0 tracker.netklix.com +0.0.0.0 tracker.tradedoubler.com +0.0.0.0 track.exclusivecpa.com +0.0.0.0 track.ft.com +0.0.0.0 track.gawker.com +0.0.0.0 track.homestead.com +#0.0.0.0 track.hulu.com # Uncomment to block Hulu. +0.0.0.0 tracking.10e20.com +0.0.0.0 tracking.adjug.com +0.0.0.0 tracking.allposters.com +0.0.0.0 tracking.foxnews.com +0.0.0.0 tracking.iol.co.za +0.0.0.0 tracking.msadcenter.msn.com +0.0.0.0 tracking.oggifinogi.com +0.0.0.0 tracking.percentmobile.com +0.0.0.0 tracking.publicidees.com +0.0.0.0 tracking.quisma.com +0.0.0.0 tracking.rangeonlinemedia.com +0.0.0.0 tracking.searchmarketing.com +0.0.0.0 tracking.summitmedia.co.uk +0.0.0.0 tracking.trafficjunky.net +0.0.0.0 tracking.trutv.com +0.0.0.0 tracking.vindicosuite.com +0.0.0.0 track.lfstmedia.com +0.0.0.0 track.mybloglog.com +0.0.0.0 track.omg2.com +0.0.0.0 track.roiservice.com +0.0.0.0 track.searchignite.com +0.0.0.0 tracksurf.daooda.com +0.0.0.0 track.webgains.com +0.0.0.0 tradedoubler.com +0.0.0.0 tradedoubler.sonvideopro.com +0.0.0.0 tr.adinterax.com +0.0.0.0 traffic-stats.streamsolutions.co.uk +0.0.0.0 trax.gamespot.com +0.0.0.0 trc.taboolasyndication.com +0.0.0.0 trk.kissmetrics.com +0.0.0.0 trk.tidaltv.com +0.0.0.0 true-counter.com +0.0.0.0 truehits1.gits.net.th +0.0.0.0 t.senaluno.com +0.0.0.0 tu.connect.wunderloop.net +0.0.0.0 tynt.com +0.0.0.0 u1817.16.spylog.com +0.0.0.0 u3102.47.spylog.com +0.0.0.0 u3305.71.spylog.com +0.0.0.0 u3608.20.spylog.com +0.0.0.0 u4056.56.spylog.com +0.0.0.0 u432.77.spylog.com +0.0.0.0 u4396.79.spylog.com +0.0.0.0 u4443.84.spylog.com +0.0.0.0 u4556.11.spylog.com +0.0.0.0 u5234.87.spylog.com +0.0.0.0 u5234.98.spylog.com +0.0.0.0 u5687.48.spylog.com +0.0.0.0 u574.07.spylog.com +0.0.0.0 u604.41.spylog.com +0.0.0.0 u6762.46.spylog.com +0.0.0.0 u6905.71.spylog.com +0.0.0.0 u7748.16.spylog.com +0.0.0.0 u810.15.spylog.com +0.0.0.0 u920.31.spylog.com +0.0.0.0 u977.40.spylog.com +0.0.0.0 udc.msn.com +0.0.0.0 uk.cqcounter.com +0.0.0.0 uk.sitestat.com +0.0.0.0 ultimatecounter.com +0.0.0.0 us.2.cqcounter.com +0.0.0.0 usa.nedstat.net +0.0.0.0 us.cqcounter.com +0.0.0.0 v1.nedstatbasic.net +0.0.0.0 v7.stats.load.com +0.0.0.0 valueclick.com +0.0.0.0 valueclick.net +0.0.0.0 vertical-stats.huffpost.com +0.0.0.0 video-stats.video.google.com +0.0.0.0 vip.clickzs.com +0.0.0.0 virtualbartendertrack.beer.com +0.0.0.0 visit.theglobeandmail.com # Visits to theglobeandmail.com +0.0.0.0 vis.sexlist.com +0.0.0.0 voken.eyereturn.com +0.0.0.0 vs.dmtracker.com +0.0.0.0 vsii.spinbox.net +0.0.0.0 vsii.spindox.net +0.0.0.0 w1.tcr112.tynt.com +0.0.0.0 warlog.info +0.0.0.0 wau.tynt.com +0.0.0.0 web1.realtracker.com +0.0.0.0 web2.realtracker.com +0.0.0.0 web3.realtracker.com +0.0.0.0 web4.realtracker.com +0.0.0.0 webanalytics.globalthoughtz.com +0.0.0.0 webbug.seatreport.com # web bugs +0.0.0.0 web-counter.5u.com +0.0.0.0 webcounter.com +0.0.0.0 webcounter.goweb.de +0.0.0.0 webcounter.together.net +0.0.0.0 webhit.aftenposten.no +0.0.0.0 webhit.afterposten.no +0.0.0.0 webmasterkai.sitetracker.com +0.0.0.0 webpdp.gator.com +0.0.0.0 webstat.channel4.com +0.0.0.0 webtrends.telenet.be +0.0.0.0 webtrends.thisis.co.uk +0.0.0.0 webtrends.townhall.com +0.0.0.0 wtnj.worldnow.com +0.0.0.0 www.0stats.com +0.0.0.0 www101.coolsavings.com +0.0.0.0 www.123count.com +0.0.0.0 www.123counter.superstats.com +0.0.0.0 www.123stat.com +0.0.0.0 www1.addfreestats.com +0.0.0.0 www1.counter.bloke.com +0.0.0.0 www.1quickclickrx.com +0.0.0.0 www1.tynt.com +0.0.0.0 www.2001-007.com +0.0.0.0 www2.addfreestats.com +0.0.0.0 www2.counter.bloke.com +0.0.0.0 www2.pagecount.com +0.0.0.0 www3.addfreestats.com +0.0.0.0 www3.click-fr.com +0.0.0.0 www3.counter.bloke.com +0.0.0.0 www.3dstats.com +0.0.0.0 www4.addfreestats.com +0.0.0.0 www4.counter.bloke.com +0.0.0.0 www5.addfreestats.com +0.0.0.0 www5.counter.bloke.com +0.0.0.0 www60.valueclick.com +0.0.0.0 www6.addfreestats.com +0.0.0.0 www6.click-fr.com +0.0.0.0 www6.counter.bloke.com +0.0.0.0 www7.addfreestats.com +0.0.0.0 www7.counter.bloke.com +0.0.0.0 www8.addfreestats.com +0.0.0.0 www8.counter.bloke.com +0.0.0.0 www9.counter.bloke.com +0.0.0.0 www.addfreecounter.com +0.0.0.0 www.addfreestats.com +0.0.0.0 www.ademails.com +0.0.0.0 www.affiliatesuccess.net +0.0.0.0 www.bar.ry2002.02-ry014.snpr.hotmx.hair.zaam.net # In spam +0.0.0.0 www.belstat.nl +0.0.0.0 www.betcounter.com +0.0.0.0 www.bigbadted.com +0.0.0.0 www.bluestreak.com +0.0.0.0 www.c1.thecounter.de +0.0.0.0 www.c2.thecounter.de +0.0.0.0 www.clickclick.com +0.0.0.0 www.clickspring.net #used by a spyware product called PurityScan +0.0.0.0 www.clixgalore.com +0.0.0.0 www.connectionlead.com +0.0.0.0 www.counter10.sextracker.be +0.0.0.0 www.counter11.sextracker.be +0.0.0.0 www.counter12.sextracker.be +0.0.0.0 www.counter13.sextracker.be +0.0.0.0 www.counter14.sextracker.be +0.0.0.0 www.counter15.sextracker.be +0.0.0.0 www.counter16.sextracker.be +0.0.0.0 www.counter1.sextracker.be +0.0.0.0 www.counter2.sextracker.be +0.0.0.0 www.counter3.sextracker.be +0.0.0.0 www.counter4all.com +0.0.0.0 www.counter4all.de +0.0.0.0 www.counter4.sextracker.be +0.0.0.0 www.counter5.sextracker.be +0.0.0.0 www.counter6.sextracker.be +0.0.0.0 www.counter7.sextracker.be +0.0.0.0 www.counter8.sextracker.be +0.0.0.0 www.counter9.sextracker.be +0.0.0.0 www.counter.bloke.com +0.0.0.0 www.counterguide.com +0.0.0.0 www.counter.sexhound.nl +0.0.0.0 www.counter.superstats.com +0.0.0.0 www.c.thecounter.de +0.0.0.0 www.cw.nu +0.0.0.0 www.directgrowthhormone.com +0.0.0.0 www.dpbolvw.net +0.0.0.0 www.dwclick.com +0.0.0.0 www.easycounter.com +0.0.0.0 www.emaildeals.biz +0.0.0.0 www.estats4all.com +0.0.0.0 www.fastcounter.linkexchange.nl +0.0.0.0 www.formalyzer.com +0.0.0.0 www.foxcounter.com +0.0.0.0 www.freestats.com +0.0.0.0 www.fxcounters.com +0.0.0.0 www.gator.com +0.0.0.0 www.googkle.com +0.0.0.0 www.googletagservices.com +0.0.0.0 www.hitstats.co.uk +0.0.0.0 www.iccee.com +0.0.0.0 www.iesnare.com # See http://www.codingthewheel.com/archives/online-gambling-privacy-iesnare +0.0.0.0 www.jellycounter.com +0.0.0.0 www.kqzyfj.com +0.0.0.0 www.leadpub.com +0.0.0.0 www.linkcounter.com +0.0.0.0 www.marketscore.com +0.0.0.0 www.megacounter.de +0.0.0.0 www.metareward.com # web bugs in spam +0.0.0.0 www.naturalgrowthstore.biz +0.0.0.0 www.nedstat.com +0.0.0.0 www.nextgenstats.com +0.0.0.0 www.ntsearch.com +0.0.0.0 www.onestat.com +0.0.0.0 www.originalicons.com # installs IE extension +0.0.0.0 www.paycounter.com +0.0.0.0 www.pointclicktrack.com +0.0.0.0 www.popuptrafic.com +0.0.0.0 www.precisioncounter.com +0.0.0.0 www.premiumsmail.net +0.0.0.0 www.printmail.biz +0.0.0.0 www.quantserve.com #: Ad Tracking, JavaScript, etc. +0.0.0.0 www.quareclk.com +0.0.0.0 www.remotrk.com +0.0.0.0 www.rightmedia.net +0.0.0.0 www.rightstats.com +0.0.0.0 www.searchadv.com +0.0.0.0 www.sekel.ch +0.0.0.0 www.shockcounter.com +0.0.0.0 www.simplecounter.net +0.0.0.0 www.specificclick.com +0.0.0.0 www.specificpop.com +0.0.0.0 www.spklds.com +0.0.0.0 www.statcount.com +0.0.0.0 www.statcounter.com +0.0.0.0 www.statsession.com +0.0.0.0 www.stattrax.com +0.0.0.0 www.stiffnetwork.com +0.0.0.0 www.testracking.com +0.0.0.0 www.thecounter.com +0.0.0.0 www.the-counter.net +0.0.0.0 www.toolbarcounter.com +0.0.0.0 www.tradedoubler.com +0.0.0.0 www.tradedoubler.com.ar +0.0.0.0 www.trafficmagnet.net # web bugs in spam +0.0.0.0 www.trafic.ro +0.0.0.0 www.trendcounter.com +0.0.0.0 www.true-counter.com +0.0.0.0 www.tynt.com +0.0.0.0 www.ultimatecounter.com +0.0.0.0 www.v61.com +0.0.0.0 www.webcounter.com +0.0.0.0 www.web-stat.com +0.0.0.0 www.webstat.com +0.0.0.0 www.whereugetxxx.com +0.0.0.0 www.xxxcounter.com +0.0.0.0 x.cb.kount.com +0.0.0.0 xcnn.com +0.0.0.0 xxxcounter.com +0.0.0.0 xyz.freelogs.com +0.0.0.0 zz.cqcounter.com +# +# + +# sites with known trojans, phishing, or other malware +0.0.0.0 05tz2e9.com +0.0.0.0 09killspyware.com +0.0.0.0 11398.onceedge.ru +0.0.0.0 2006mindfreaklike.blogspot.com # Facebook trojan +0.0.0.0 20-yrs-1.info +0.0.0.0 59-106-20-39.r-bl100.sakura.ne.jp +0.0.0.0 662bd114b7c9.onceedge.ru +0.0.0.0 a15172379.alturo-server.de +0.0.0.0 aaukqiooaseseuke.org +0.0.0.0 abetterinternet.com +0.0.0.0 abruzzoinitaly.co.uk +0.0.0.0 acglgoa.com +0.0.0.0 acim.moqhixoz.cn +0.0.0.0 adshufffle.com +0.0.0.0 adwitty.com +0.0.0.0 adwords.google.lloymlincs.com +0.0.0.0 afantispy.com +0.0.0.0 afdbande.cn +0.0.0.0 allhqpics.com # Facebook trojan +0.0.0.0 alphabirdnetwork.com +0.0.0.0 antispywareexpert.com +0.0.0.0 antivirus-online-scan5.com +0.0.0.0 antivirus-scanner8.com +0.0.0.0 antivirus-scanner.com +0.0.0.0 a.oix.com +0.0.0.0 a.oix.net +0.0.0.0 armsart.com +0.0.0.0 articlefuns.cn +0.0.0.0 articleidea.cn +0.0.0.0 asianread.com +0.0.0.0 autohipnose.com +0.0.0.0 a.webwise.com +0.0.0.0 a.webwise.net +0.0.0.0 a.webwise.org +0.0.0.0 beloysoff.ru +0.0.0.0 binsservicesonline.info +0.0.0.0 blackhat.be +0.0.0.0 blenz-me.net +0.0.0.0 bnvxcfhdgf.blogspot.com.es +0.0.0.0 b.oix.com +0.0.0.0 b.oix.net +0.0.0.0 BonusCashh.com +0.0.0.0 brunga.at # Facebook phishing attempt +0.0.0.0 bt.webwise.com +0.0.0.0 bt.webwise.net +0.0.0.0 bt.webwise.org +0.0.0.0 b.webwise.com +0.0.0.0 b.webwise.net +0.0.0.0 b.webwise.org +0.0.0.0 callawaypos.com +0.0.0.0 callbling.com +0.0.0.0 cambonanza.com +0.0.0.0 ccudl.com +0.0.0.0 changduk26.com # Facebook trojan +0.0.0.0 chelick.net # Facebook trojan +0.0.0.0 cioco-froll.com +0.0.0.0 cira.login.cqr.ssl.igotmyloverback.com +0.0.0.0 cleanchain.net +0.0.0.0 click.get-answers-fast.com +0.0.0.0 clien.net +0.0.0.0 cnbc.com-article906773.us +0.0.0.0 co8vd.cn +0.0.0.0 c.oix.com +0.0.0.0 c.oix.net +0.0.0.0 conduit.com +0.0.0.0 cra-arc-gc-ca.noads.biz +0.0.0.0 custom3hurricanedigitalmedia.com +0.0.0.0 c.webwise.com +0.0.0.0 c.webwise.net +0.0.0.0 c.webwise.org +0.0.0.0 dbios.org +0.0.0.0 dhauzja511.co.cc +0.0.0.0 dietpharmacyrx.net +0.0.0.0 download.abetterinternet.com +0.0.0.0 drc-group.net +0.0.0.0 dubstep.onedumb.com +0.0.0.0 east.05tz2e9.com +0.0.0.0 e-kasa.w8w.pl +0.0.0.0 en.likefever.org # Facebook trojan +0.0.0.0 enteryouremail.net +0.0.0.0 eviboli576.o-f.com +0.0.0.0 facebook-repto1040s2.ahlamountada.com +0.0.0.0 faceboook-replyei0ki.montadalitihad.com +0.0.0.0 facemail.com +0.0.0.0 faggotry.com +0.0.0.0 familyupport1.com +0.0.0.0 feaecebook.com +0.0.0.0 fengyixin.com +0.0.0.0 filosvybfimpsv.ru.gg +0.0.0.0 froling.bee.pl +0.0.0.0 fromru.su +0.0.0.0 ftdownload.com +0.0.0.0 fu.golikeus.net # Facebook trojan +0.0.0.0 gamelights.ru +0.0.0.0 gasasthe.freehostia.com +0.0.0.0 get-answers-fast.com +0.0.0.0 gglcash4u.info # twitter worm +0.0.0.0 girlownedbypolicelike.blogspot.com # Facebook trojan +0.0.0.0 goggle.com +0.0.0.0 greatarcadehits.com +0.0.0.0 gyros.es +0.0.0.0 h1317070.stratoserver.net +0.0.0.0 hackerz.ir +0.0.0.0 hakerzy.net +0.0.0.0 hatrecord.ru # Facebook trojan +0.0.0.0 hellwert.biz +0.0.0.0 hotchix.servepics.com +0.0.0.0 hsb-canada.com # phishing site for hsbc.ca +0.0.0.0 hsbconline.ca # phishing site for hsbc.ca +0.0.0.0 icecars.com +0.0.0.0 idea21.org +0.0.0.0 Iframecash.biz +0.0.0.0 infopaypal.com +0.0.0.0 installmac.com +0.0.0.0 ipadzu.net +0.0.0.0 ircleaner.com +0.0.0.0 itwititer.com +0.0.0.0 ity.elusmedic.ru +0.0.0.0 jajajaj-thats-you-really.com +0.0.0.0 janezk.50webs.co +0.0.0.0 jujitsu-ostrava.info +0.0.0.0 jump.ewoss.net +0.0.0.0 juste.ru # Twitter trojan +0.0.0.0 kczambians.com +0.0.0.0 keybinary.com +0.0.0.0 kirgo.at # Facebook phishing attempt +0.0.0.0 klowns4phun.com +0.0.0.0 konflow.com # Facebook trojan +0.0.0.0 kplusd.far.ru +0.0.0.0 kpremium.com +0.0.0.0 lank.ru +0.0.0.0 lighthouse2k.com +0.0.0.0 like.likewut.net +0.0.0.0 likeportal.com # Facebook trojan +0.0.0.0 likespike.com # Facebook trojan +0.0.0.0 likethislist.biz # Facebook trojan +0.0.0.0 likethis.mbosoft.com # Facebook trojan +0.0.0.0 loseweight.asdjiiw.com +0.0.0.0 lucibad.home.ro +0.0.0.0 luxcart.ro +0.0.0.0 m01.oix.com +0.0.0.0 m01.oix.net +0.0.0.0 m01.webwise.com +0.0.0.0 m01.webwise.net +0.0.0.0 m01.webwise.org +0.0.0.0 m02.oix.com +0.0.0.0 m02.oix.net +0.0.0.0 m02.webwise.com +0.0.0.0 m02.webwise.net +0.0.0.0 m02.webwise.org +0.0.0.0 mail.cyberh.fr +0.0.0.0 malware-live-pro-scanv1.com +0.0.0.0 maxi4.firstvds.ru +0.0.0.0 megasurfin.com +0.0.0.0 monkeyball.osa.pl +0.0.0.0 movies.701pages.com +0.0.0.0 mplayerdownloader.com +0.0.0.0 murcia-ban.es +0.0.0.0 mylike.co.uk # Facebook trojan +0.0.0.0 nactx.com +0.0.0.0 natashyabaydesign.com +0.0.0.0 new-dating-2012.info +0.0.0.0 new-vid-zone-1.blogspot.com.au +0.0.0.0 newwayscanner.info +0.0.0.0 novemberrainx.com +0.0.0.0 ns1.oix.com +0.0.0.0 ns1.oix.net +0.0.0.0 ns1.webwise.com +0.0.0.0 ns1.webwise.net +0.0.0.0 ns1.webwise.org +0.0.0.0 ns2.oix.com +0.0.0.0 ns2.oix.net +0.0.0.0 ns2.webwise.com +0.0.0.0 ns2.webwise.net +0.0.0.0 ns2.webwise.org +0.0.0.0 nufindings.info +0.0.0.0 office.officenet.co.kr +0.0.0.0 oix.com +0.0.0.0 oix.net +0.0.0.0 oj.likewut.net +0.0.0.0 online-antispym4.com +0.0.0.0 oo-na-na-pics.com +0.0.0.0 ordersildenafil.com +0.0.0.0 otsserver.com +0.0.0.0 outerinfo.com +0.0.0.0 paincake.yoll.net +0.0.0.0 pc-scanner16.com +0.0.0.0 personalantispy.com +0.0.0.0 phatthalung.go.th +0.0.0.0 picture-uploads.com +0.0.0.0 pilltabletsrxbargain.net +0.0.0.0 powabcyfqe.com +0.0.0.0 premium-live-scan.com +0.0.0.0 products-gold.net +0.0.0.0 proflashdata.com # Facebook trojan +0.0.0.0 protectionupdatecenter.com +0.0.0.0 pv.wantsfly.com +0.0.0.0 qip.ru +0.0.0.0 qy.corrmedic.ru +0.0.0.0 rd.alphabirdnetwork.com +0.0.0.0 rickrolling.com +0.0.0.0 roifmd.info +0.0.0.0 russian-sex.com +0.0.0.0 s4d.in +0.0.0.0 scan.antispyware-free-scanner.com +0.0.0.0 scanner.best-click-av1.info +0.0.0.0 scanner.best-protect.info +0.0.0.0 scottishstuff-online.com # Canadian bank phishing site +0.0.0.0 sc-spyware.com +0.0.0.0 search.conduit.com +0.0.0.0 securedliveuploads.com +0.0.0.0 securityandroidupdate.dinamikaprinting.com +0.0.0.0 securityscan.us +0.0.0.0 sexymarissa.net +0.0.0.0 shell.xhhow4.com +0.0.0.0 shoppstop.comood.opsource.net +0.0.0.0 shop.skin-safety.com +0.0.0.0 signin-ebay-com-ws-ebayisapi-dll-signin-webscr.ocom.pl +0.0.0.0 sinera.org +0.0.0.0 sjguild.com +0.0.0.0 smarturl.it +0.0.0.0 smile-angel.com +0.0.0.0 software-updates.co +0.0.0.0 software-wenc.co.cc +0.0.0.0 someonewhocares.com +0.0.0.0 sousay.info +0.0.0.0 start.qip.ru +0.0.0.0 superegler.net +0.0.0.0 supernaturalart.com +0.0.0.0 superprotection10.com +0.0.0.0 sverd.net +0.0.0.0 tahoesup.com +0.0.0.0 tattooshaha.info # Facebook trojan +0.0.0.0 test.ishvara-yoga.com +0.0.0.0 TheBizMeet.com +0.0.0.0 thedatesafe.com # Facebook trojan +0.0.0.0 themoneyclippodcast.com +0.0.0.0 themusicnetwork.co.uk +0.0.0.0 thinstall.abetterinternet.com +0.0.0.0 tivvitter.com +0.0.0.0 tomorrownewstoday.com # I'm not sure what it does, but it seems to be associated with a phishing attempt on Facebook +0.0.0.0 toolbarbest.biz +0.0.0.0 toolbarbucks.biz +0.0.0.0 toolbarcool.biz +0.0.0.0 toolbardollars.biz +0.0.0.0 toolbarmoney.biz +0.0.0.0 toolbarnew.biz +0.0.0.0 toolbarsale.biz +0.0.0.0 toolbarweb.biz +0.0.0.0 traffic.adwitty.com +0.0.0.0 trialreg.com +0.0.0.0 tvshowslist.com +0.0.0.0 twitter.login.kevanshome.org +0.0.0.0 twitter.secure.bzpharma.net +0.0.0.0 uawj.moqhixoz.cn +0.0.0.0 ughmvqf.spitt.ru +0.0.0.0 uqz.com +0.0.0.0 users16.jabry.com +0.0.0.0 utenti.lycos.it +0.0.0.0 vcipo.info +0.0.0.0 videos.dskjkiuw.com +0.0.0.0 videos.twitter.secure-logins01.com # twitter worm (http://mashable.com/2009/09/23/twitter-worm-dms/) +0.0.0.0 vxiframe.biz +0.0.0.0 waldenfarms.com +0.0.0.0 weblover.info +0.0.0.0 webpaypal.com +0.0.0.0 webwise.com +0.0.0.0 webwise.net +0.0.0.0 webwise.org +0.0.0.0 west.05tz2e9.com +0.0.0.0 wewillrocknow.com +0.0.0.0 willysy.com +0.0.0.0 wm.maxysearch.info +0.0.0.0 womo.corrmedic.ru +0.0.0.0 www1.bmo.com.hotfrio.com.br +0.0.0.0 www1.firesavez5.com +0.0.0.0 www1.firesavez6.com +0.0.0.0 www1.realsoft34.com +0.0.0.0 www4.gy7k.net +0.0.0.0 www.abetterinternet.com +0.0.0.0 www.adshufffle.com +0.0.0.0 www.adwords.google.lloymlincs.com +0.0.0.0 www.afantispy.com +0.0.0.0 www.akoneplatit.sk +0.0.0.0 www.allhqpics.com # Facebook trojan +0.0.0.0 www.alrpost69.com +0.0.0.0 www.anatol.com +0.0.0.0 www.articlefuns.cn +0.0.0.0 www.articleidea.cn +0.0.0.0 www.asianread.com +0.0.0.0 www.backsim.ru +0.0.0.0 www.bankofamerica.com.ok.am +0.0.0.0 www.be4life.ru +0.0.0.0 www.blenz-me.net +0.0.0.0 www.cambonanza.com +0.0.0.0 www.chelick.net # Facebook trojan +0.0.0.0 www.didata.bw +0.0.0.0 www.dietsecret.ru +0.0.0.0 www.eroyear.ru +0.0.0.0 www.exbays.com +0.0.0.0 www.faggotry.com +0.0.0.0 www.feaecebook.com +0.0.0.0 www.fictioncinema.com +0.0.0.0 www.fischereszter.hu +0.0.0.0 www.froling.bee.pl +0.0.0.0 www.gns-consola.com +0.0.0.0 www.goggle.com +0.0.0.0 www.grouphappy.com +0.0.0.0 www.hakerzy.net +0.0.0.0 www.haoyunlaid.com +0.0.0.0 www.icecars.com +0.0.0.0 www.indesignstudioinfo.com +0.0.0.0 www.infopaypal.com +0.0.0.0 www.keybinary.com +0.0.0.0 www.kinomarathon.ru +0.0.0.0 www.kpremium.com +0.0.0.0 www.likeportal.com # Facebook trojan +0.0.0.0 www.likespike.com # Facebook trojan +0.0.0.0 www.likethislist.biz # Facebook trojan +0.0.0.0 www.likethis.mbosoft.com # Facebook trojan +0.0.0.0 www.lomalindasda.org # Facebook trojan +0.0.0.0 www.lovecouple.ru +0.0.0.0 www.lovetrust.ru +0.0.0.0 www.mikras.nl +0.0.0.0 www.monkeyball.osa.pl +0.0.0.0 www.monsonis.net +0.0.0.0 www.movie-port.ru +0.0.0.0 www.mplayerdownloader.com +0.0.0.0 www.mylike.co.uk # Facebook trojan +0.0.0.0 www.mylovecards.com +0.0.0.0 www.nine2rack.in +0.0.0.0 www.novemberrainx.com +0.0.0.0 www.nu26.com +0.0.0.0 www.oix.com +0.0.0.0 www.oix.net +0.0.0.0 www.onlyfreeoffersonline.com +0.0.0.0 www.oreidofitilho.com.br +0.0.0.0 www.otsserver.com +0.0.0.0 www.pay-pal.com-cgibin-canada.4mcmeta4v.cn +0.0.0.0 www.picture-uploads.com +0.0.0.0 www.portaldimensional.com +0.0.0.0 www.poxudeli.ru +0.0.0.0 www.proflashdata.com # Facebook trojan +0.0.0.0 www.rickrolling.com +0.0.0.0 www.russian-sex.com +0.0.0.0 www.scotiaonline.scotiabank.salferreras.com +0.0.0.0 www.sdlpgift.com +0.0.0.0 www.securityscan.us +0.0.0.0 www.servertasarimbu.com +0.0.0.0 www.sexytiger.ru +0.0.0.0 www.shinilchurch.net # domain was hacked and had a trojan installed +0.0.0.0 www.sinera.org +0.0.0.0 www.someonewhocares.com +0.0.0.0 www.tanger.com.br +0.0.0.0 www.tattooshaha.info # Facebook trojan +0.0.0.0 www.te81.net +0.0.0.0 www.thedatesafe.com # Facebook trojan +0.0.0.0 www.trucktirehotline.com +0.0.0.0 www.tvshowslist.com +0.0.0.0 www.upi6.pillsstore-c.com # Facebook trojan +0.0.0.0 www.uqz.com +0.0.0.0 www.via99.org +0.0.0.0 www.videolove.clanteam.com +0.0.0.0 www.videostan.ru +0.0.0.0 www.vippotexa.ru +0.0.0.0 www.wantsfly.com +0.0.0.0 www.webpaypal.com +0.0.0.0 www.webwise.com +0.0.0.0 www.webwise.net +0.0.0.0 www.webwise.org +0.0.0.0 www.wewillrocknow.com +0.0.0.0 www.willysy.com +0.0.0.0 xfotosx01.fromru.su +0.0.0.0 xponlinescanner.com +0.0.0.0 xvrxyzba253.hotmail.ru +0.0.0.0 yrwap.cn +0.0.0.0 zarozinski.info +0.0.0.0 zettapetta.com +0.0.0.0 zfotos.fromru.su +0.0.0.0 zip.er.cz +0.0.0.0 ztrf.net +0.0.0.0 zviframe.biz +# + +# + +0.0.0.0 3ad.doubleclick.net +0.0.0.0 ad2.doubleclick.net +0.0.0.0 ad.3au.doubleclick.net +0.0.0.0 ad.ae.doubleclick.net +0.0.0.0 ad.au.doubleclick.net +0.0.0.0 ad.be.doubleclick.net +0.0.0.0 ad.br.doubleclick.net +0.0.0.0 ad.de.doubleclick.net +0.0.0.0 ad.dk.doubleclick.net +0.0.0.0 ad-emea.doubleclick.net +0.0.0.0 ad.es.doubleclick.net +0.0.0.0 ad.fi.doubleclick.net +0.0.0.0 ad.fr.doubleclick.net +0.0.0.0 ad-g.doubleclick.net +0.0.0.0 ad.it.doubleclick.net +0.0.0.0 ad.jp.doubleclick.net +0.0.0.0 ad.mo.doubleclick.net +0.0.0.0 ad.n2434.doubleclick.net +0.0.0.0 ad.nl.doubleclick.net +0.0.0.0 ad.no.doubleclick.net +0.0.0.0 ad.nz.doubleclick.net +0.0.0.0 ad.pl.doubleclick.net +0.0.0.0 ad.se.doubleclick.net +0.0.0.0 ad.sg.doubleclick.net +0.0.0.0 ad.uk.doubleclick.net +0.0.0.0 ad.ve.doubleclick.net +0.0.0.0 ad-yt-bfp.doubleclick.net +0.0.0.0 ad.za.doubleclick.net +0.0.0.0 amn.doubleclick.net +0.0.0.0 creative.cc-dt.com +0.0.0.0 doubleclick.de +0.0.0.0 doubleclick.net +0.0.0.0 ebaycn.doubleclick.net +0.0.0.0 ebaytw.doubleclick.net +0.0.0.0 exnjadgda1.doubleclick.net +0.0.0.0 exnjadgda2.doubleclick.net +0.0.0.0 exnjadgds1.doubleclick.net +0.0.0.0 exnjmdgda1.doubleclick.net +0.0.0.0 exnjmdgds1.doubleclick.net +0.0.0.0 feedads.g.doubleclick.net +0.0.0.0 fls.doubleclick.net +0.0.0.0 gd10.doubleclick.net +0.0.0.0 gd11.doubleclick.net +0.0.0.0 gd12.doubleclick.net +0.0.0.0 gd13.doubleclick.net +0.0.0.0 gd14.doubleclick.net +0.0.0.0 gd15.doubleclick.net +0.0.0.0 gd16.doubleclick.net +0.0.0.0 gd17.doubleclick.net +0.0.0.0 gd18.doubleclick.net +0.0.0.0 gd19.doubleclick.net +0.0.0.0 gd1.doubleclick.net +0.0.0.0 gd20.doubleclick.net +0.0.0.0 gd21.doubleclick.net +0.0.0.0 gd22.doubleclick.net +0.0.0.0 gd23.doubleclick.net +0.0.0.0 gd24.doubleclick.net +0.0.0.0 gd25.doubleclick.net +0.0.0.0 gd26.doubleclick.net +0.0.0.0 gd27.doubleclick.net +0.0.0.0 gd28.doubleclick.net +0.0.0.0 gd29.doubleclick.net +0.0.0.0 gd2.doubleclick.net +0.0.0.0 gd30.doubleclick.net +0.0.0.0 gd31.doubleclick.net +0.0.0.0 gd3.doubleclick.net +0.0.0.0 gd4.doubleclick.net +0.0.0.0 gd5.doubleclick.net +0.0.0.0 gd7.doubleclick.net +0.0.0.0 gd8.doubleclick.net +0.0.0.0 gd9.doubleclick.net +0.0.0.0 googleads.g.doubleclick.net +0.0.0.0 iv.doubleclick.net +0.0.0.0 ln.doubleclick.net +0.0.0.0 m1.2mdn.net +0.0.0.0 m1.ae.2mdn.net +0.0.0.0 m1.au.2mdn.net +0.0.0.0 m1.be.2mdn.net +0.0.0.0 m1.br.2mdn.net +0.0.0.0 m1.ca.2mdn.net +0.0.0.0 m1.cn.2mdn.net +0.0.0.0 m1.de.2mdn.net +0.0.0.0 m1.dk.2mdn.net +0.0.0.0 m1.doubleclick.net +0.0.0.0 m1.es.2mdn.net +0.0.0.0 m1.fi.2mdn.net +0.0.0.0 m1.fr.2mdn.net +0.0.0.0 m1.it.2mdn.net +0.0.0.0 m1.jp.2mdn.net +0.0.0.0 m1.nl.2mdn.net +0.0.0.0 m1.no.2mdn.net +0.0.0.0 m1.nz.2mdn.net +0.0.0.0 m1.pl.2mdn.net +0.0.0.0 m1.se.2mdn.net +0.0.0.0 m1.sg.2mdn.net +0.0.0.0 m1.uk.2mdn.net +0.0.0.0 m1.ve.2mdn.net +0.0.0.0 m1.za.2mdn.net +0.0.0.0 m2.ae.2mdn.net +0.0.0.0 m2.au.2mdn.net +0.0.0.0 m2.be.2mdn.net +0.0.0.0 m2.br.2mdn.net +0.0.0.0 m2.ca.2mdn.net +0.0.0.0 m2.cn.2mdn.net +0.0.0.0 m2.cn.doubleclick.net +0.0.0.0 m2.de.2mdn.net +0.0.0.0 m2.dk.2mdn.net +0.0.0.0 m2.doubleclick.net +0.0.0.0 m2.es.2mdn.net +0.0.0.0 m2.fi.2mdn.net +0.0.0.0 m2.fr.2mdn.net +0.0.0.0 m2.it.2mdn.net +0.0.0.0 m2.jp.2mdn.net +0.0.0.0 m.2mdn.net +0.0.0.0 m2.nl.2mdn.net +0.0.0.0 m2.no.2mdn.net +0.0.0.0 m2.nz.2mdn.net +0.0.0.0 m2.pl.2mdn.net +0.0.0.0 m2.se.2mdn.net +0.0.0.0 m2.sg.2mdn.net +0.0.0.0 m2.uk.2mdn.net +0.0.0.0 m2.ve.2mdn.net +0.0.0.0 m2.za.2mdn.net +0.0.0.0 m3.ae.2mdn.net +0.0.0.0 m3.au.2mdn.net +0.0.0.0 m3.be.2mdn.net +0.0.0.0 m3.br.2mdn.net +0.0.0.0 m3.ca.2mdn.net +0.0.0.0 m3.cn.2mdn.net +0.0.0.0 m3.de.2mdn.net +0.0.0.0 m3.dk.2mdn.net +0.0.0.0 m3.doubleclick.net +0.0.0.0 m3.es.2mdn.net +0.0.0.0 m3.fi.2mdn.net +0.0.0.0 m3.fr.2mdn.net +0.0.0.0 m3.it.2mdn.net +0.0.0.0 m3.jp.2mdn.net +0.0.0.0 m3.nl.2mdn.net +0.0.0.0 m3.no.2mdn.net +0.0.0.0 m3.nz.2mdn.net +0.0.0.0 m3.pl.2mdn.net +0.0.0.0 m3.se.2mdn.net +0.0.0.0 m3.sg.2mdn.net +0.0.0.0 m3.uk.2mdn.net +0.0.0.0 m3.ve.2mdn.net +0.0.0.0 m3.za.2mdn.net +0.0.0.0 m4.ae.2mdn.net +0.0.0.0 m4.au.2mdn.net +0.0.0.0 m4.be.2mdn.net +0.0.0.0 m4.br.2mdn.net +0.0.0.0 m4.ca.2mdn.net +0.0.0.0 m4.cn.2mdn.net +0.0.0.0 m4.de.2mdn.net +0.0.0.0 m4.dk.2mdn.net +0.0.0.0 m4.doubleclick.net +0.0.0.0 m4.es.2mdn.net +0.0.0.0 m4.fi.2mdn.net +0.0.0.0 m4.fr.2mdn.net +0.0.0.0 m4.it.2mdn.net +0.0.0.0 m4.jp.2mdn.net +0.0.0.0 m4.nl.2mdn.net +0.0.0.0 m4.no.2mdn.net +0.0.0.0 m4.nz.2mdn.net +0.0.0.0 m4.pl.2mdn.net +0.0.0.0 m4.se.2mdn.net +0.0.0.0 m4.sg.2mdn.net +0.0.0.0 m4.uk.2mdn.net +0.0.0.0 m4.ve.2mdn.net +0.0.0.0 m4.za.2mdn.net +0.0.0.0 m5.ae.2mdn.net +0.0.0.0 m5.au.2mdn.net +0.0.0.0 m5.be.2mdn.net +0.0.0.0 m5.br.2mdn.net +0.0.0.0 m5.ca.2mdn.net +0.0.0.0 m5.cn.2mdn.net +0.0.0.0 m5.de.2mdn.net +0.0.0.0 m5.dk.2mdn.net +0.0.0.0 m5.doubleclick.net +0.0.0.0 m5.es.2mdn.net +0.0.0.0 m5.fi.2mdn.net +0.0.0.0 m5.fr.2mdn.net +0.0.0.0 m5.it.2mdn.net +0.0.0.0 m5.jp.2mdn.net +0.0.0.0 m5.nl.2mdn.net +0.0.0.0 m5.no.2mdn.net +0.0.0.0 m5.nz.2mdn.net +0.0.0.0 m5.pl.2mdn.net +0.0.0.0 m5.se.2mdn.net +0.0.0.0 m5.sg.2mdn.net +0.0.0.0 m5.uk.2mdn.net +0.0.0.0 m5.ve.2mdn.net +0.0.0.0 m5.za.2mdn.net +0.0.0.0 m6.ae.2mdn.net +0.0.0.0 m6.au.2mdn.net +0.0.0.0 m6.be.2mdn.net +0.0.0.0 m6.br.2mdn.net +0.0.0.0 m6.ca.2mdn.net +0.0.0.0 m6.cn.2mdn.net +0.0.0.0 m6.de.2mdn.net +0.0.0.0 m6.dk.2mdn.net +0.0.0.0 m6.doubleclick.net +0.0.0.0 m6.es.2mdn.net +0.0.0.0 m6.fi.2mdn.net +0.0.0.0 m6.fr.2mdn.net +0.0.0.0 m6.it.2mdn.net +0.0.0.0 m6.jp.2mdn.net +0.0.0.0 m6.nl.2mdn.net +0.0.0.0 m6.no.2mdn.net +0.0.0.0 m6.nz.2mdn.net +0.0.0.0 m6.pl.2mdn.net +0.0.0.0 m6.se.2mdn.net +0.0.0.0 m6.sg.2mdn.net +0.0.0.0 m6.uk.2mdn.net +0.0.0.0 m6.ve.2mdn.net +0.0.0.0 m6.za.2mdn.net +0.0.0.0 m7.ae.2mdn.net +0.0.0.0 m7.au.2mdn.net +0.0.0.0 m7.be.2mdn.net +0.0.0.0 m7.br.2mdn.net +0.0.0.0 m7.ca.2mdn.net +0.0.0.0 m7.cn.2mdn.net +0.0.0.0 m7.de.2mdn.net +0.0.0.0 m7.dk.2mdn.net +0.0.0.0 m7.doubleclick.net +0.0.0.0 m7.es.2mdn.net +0.0.0.0 m7.fi.2mdn.net +0.0.0.0 m7.fr.2mdn.net +0.0.0.0 m7.it.2mdn.net +0.0.0.0 m7.jp.2mdn.net +0.0.0.0 m7.nl.2mdn.net +0.0.0.0 m7.no.2mdn.net +0.0.0.0 m7.nz.2mdn.net +0.0.0.0 m7.pl.2mdn.net +0.0.0.0 m7.se.2mdn.net +0.0.0.0 m7.sg.2mdn.net +0.0.0.0 m7.uk.2mdn.net +0.0.0.0 m7.ve.2mdn.net +0.0.0.0 m7.za.2mdn.net +0.0.0.0 m8.ae.2mdn.net +0.0.0.0 m8.au.2mdn.net +0.0.0.0 m8.be.2mdn.net +0.0.0.0 m8.br.2mdn.net +0.0.0.0 m8.ca.2mdn.net +0.0.0.0 m8.cn.2mdn.net +0.0.0.0 m8.de.2mdn.net +0.0.0.0 m8.dk.2mdn.net +0.0.0.0 m8.doubleclick.net +0.0.0.0 m8.es.2mdn.net +0.0.0.0 m8.fi.2mdn.net +0.0.0.0 m8.fr.2mdn.net +0.0.0.0 m8.it.2mdn.net +0.0.0.0 m8.jp.2mdn.net +0.0.0.0 m8.nl.2mdn.net +0.0.0.0 m8.no.2mdn.net +0.0.0.0 m8.nz.2mdn.net +0.0.0.0 m8.pl.2mdn.net +0.0.0.0 m8.se.2mdn.net +0.0.0.0 m8.sg.2mdn.net +0.0.0.0 m8.uk.2mdn.net +0.0.0.0 m8.ve.2mdn.net +0.0.0.0 m8.za.2mdn.net +0.0.0.0 m9.ae.2mdn.net +0.0.0.0 m9.au.2mdn.net +0.0.0.0 m9.be.2mdn.net +0.0.0.0 m9.br.2mdn.net +0.0.0.0 m9.ca.2mdn.net +0.0.0.0 m9.cn.2mdn.net +0.0.0.0 m9.de.2mdn.net +0.0.0.0 m9.dk.2mdn.net +0.0.0.0 m9.doubleclick.net +0.0.0.0 m9.es.2mdn.net +0.0.0.0 m9.fi.2mdn.net +0.0.0.0 m9.fr.2mdn.net +0.0.0.0 m9.it.2mdn.net +0.0.0.0 m9.jp.2mdn.net +0.0.0.0 m9.nl.2mdn.net +0.0.0.0 m9.no.2mdn.net +0.0.0.0 m9.nz.2mdn.net +0.0.0.0 m9.pl.2mdn.net +0.0.0.0 m9.se.2mdn.net +0.0.0.0 m9.sg.2mdn.net +0.0.0.0 m9.uk.2mdn.net +0.0.0.0 m9.ve.2mdn.net +0.0.0.0 m9.za.2mdn.net +0.0.0.0 m.de.2mdn.net +0.0.0.0 m.doubleclick.net +0.0.0.0 n3302ad.doubleclick.net +0.0.0.0 n3349ad.doubleclick.net +0.0.0.0 n4061ad.doubleclick.net +0.0.0.0 n4403ad.doubleclick.net +0.0.0.0 n479ad.doubleclick.net +0.0.0.0 optimize.doubleclick.net +0.0.0.0 pubads.g.doubleclick.net +0.0.0.0 rd.intl.doubleclick.net +0.0.0.0 securepubads.g.doubleclick.net +0.0.0.0 stats.g.doubleclick.net +0.0.0.0 twx.2mdn.net +0.0.0.0 twx.doubleclick.net +0.0.0.0 ukrpts.net +0.0.0.0 uunyadgda1.doubleclick.net +0.0.0.0 uunyadgds1.doubleclick.net +0.0.0.0 www.ukrpts.net +# + +# + +0.0.0.0 1up.us.intellitxt.com +0.0.0.0 5starhiphop.us.intellitxt.com +0.0.0.0 askmen2.us.intellitxt.com +0.0.0.0 bargainpda.us.intellitxt.com +0.0.0.0 businesspundit.us.intellitxt.com +0.0.0.0 canadafreepress.us.intellitxt.com +0.0.0.0 contactmusic.uk.intellitxt.com +0.0.0.0 ctv.us.intellitxt.com +0.0.0.0 designtechnica.us.intellitxt.com +0.0.0.0 devshed.us.intellitxt.com +0.0.0.0 digitaltrends.us.intellitxt.com +0.0.0.0 dnps.us.intellitxt.com +0.0.0.0 doubleviking.us.intellitxt.com +0.0.0.0 drizzydrake.us.intellitxt.com +0.0.0.0 ehow.us.intellitxt.com +0.0.0.0 entertainment.msnbc.us.intellitxt.com +0.0.0.0 examnotes.us.intellitxt.com +0.0.0.0 excite.us.intellitxt.com +0.0.0.0 experts.us.intellitxt.com +0.0.0.0 extremetech.us.intellitxt.com +0.0.0.0 ferrago.uk.intellitxt.com +0.0.0.0 filmschoolrejects.us.intellitxt.com +0.0.0.0 filmwad.us.intellitxt.com +0.0.0.0 firstshowing.us.intellitxt.com +0.0.0.0 flashmagazine.us.intellitxt.com +0.0.0.0 foxnews.us.intellitxt.com +0.0.0.0 foxtv.us.intellitxt.com +0.0.0.0 freedownloadcenter.uk.intellitxt.com +0.0.0.0 gadgets.fosfor.se.intellitxt.com +0.0.0.0 gamesradar.us.intellitxt.com +0.0.0.0 gannettbroadcast.us.intellitxt.com +0.0.0.0 gonintendo.us.intellitxt.com +0.0.0.0 gorillanation.us.intellitxt.com +0.0.0.0 hackedgadgets.us.intellitxt.com +0.0.0.0 hardcoreware.us.intellitxt.com +0.0.0.0 hardocp.us.intellitxt.com +0.0.0.0 hothardware.us.intellitxt.com +0.0.0.0 hotonlinenews.us.intellitxt.com +0.0.0.0 ign.us.intellitxt.com +0.0.0.0 images.intellitxt.com +0.0.0.0 itxt2.us.intellitxt.com +0.0.0.0 joblo.us.intellitxt.com +0.0.0.0 johnchow.us.intellitxt.com +0.0.0.0 laptopmag.us.intellitxt.com +0.0.0.0 linuxforums.us.intellitxt.com +0.0.0.0 maccity.it.intellitxt.com +0.0.0.0 macnn.us.intellitxt.com +0.0.0.0 macuser.uk.intellitxt.com +0.0.0.0 macworld.uk.intellitxt.com +0.0.0.0 metro.uk.intellitxt.com +0.0.0.0 mobile9.us.intellitxt.com +0.0.0.0 monstersandcritics.uk.intellitxt.com +0.0.0.0 moviesonline.ca.intellitxt.com +0.0.0.0 mustangevolution.us.intellitxt.com +0.0.0.0 neowin.us.intellitxt.com +0.0.0.0 newcarnet.uk.intellitxt.com +0.0.0.0 newlaunches.uk.intellitxt.com +0.0.0.0 nexys404.us.intellitxt.com +0.0.0.0 ohgizmo.us.intellitxt.com +0.0.0.0 pcadvisor.uk.intellitxt.com +0.0.0.0 pcgameshardware.de.intellitxt.com +0.0.0.0 pcmag.us.intellitxt.com +0.0.0.0 pcper.us.intellitxt.com +0.0.0.0 penton.us.intellitxt.com +0.0.0.0 physorg.uk.intellitxt.com +0.0.0.0 physorg.us.intellitxt.com +0.0.0.0 playfuls.uk.intellitxt.com +0.0.0.0 pocketlint.uk.intellitxt.com +0.0.0.0 popularmechanics.us.intellitxt.com +0.0.0.0 postchronicle.us.intellitxt.com +0.0.0.0 projectorreviews.us.intellitxt.com +0.0.0.0 psp3d.us.intellitxt.com +0.0.0.0 pspcave.uk.intellitxt.com +0.0.0.0 qj.us.intellitxt.com +0.0.0.0 rasmussenreports.us.intellitxt.com +0.0.0.0 rawstory.us.intellitxt.com +0.0.0.0 savemanny.us.intellitxt.com +0.0.0.0 sc.intellitxt.com +0.0.0.0 siliconera.us.intellitxt.com +0.0.0.0 slashphone.us.intellitxt.com +0.0.0.0 soft32.us.intellitxt.com +0.0.0.0 softpedia.uk.intellitxt.com +0.0.0.0 somethingawful.us.intellitxt.com +0.0.0.0 splashnews.uk.intellitxt.com +0.0.0.0 spymac.us.intellitxt.com +0.0.0.0 techeblog.us.intellitxt.com +0.0.0.0 technewsworld.us.intellitxt.com +0.0.0.0 technologyreview.us.intellitxt.com +0.0.0.0 techspot.us.intellitxt.com +0.0.0.0 tgdaily.us.intellitxt.com +0.0.0.0 the-gadgeteer.us.intellitxt.com +0.0.0.0 thelastboss.us.intellitxt.com +0.0.0.0 thetechzone.us.intellitxt.com +0.0.0.0 thoughtsmedia.us.intellitxt.com +0.0.0.0 tmcnet.us.intellitxt.com +0.0.0.0 tomsnetworking.us.intellitxt.com +0.0.0.0 toms.us.intellitxt.com +0.0.0.0 tribal.us.intellitxt.com # vibrantmedia.com +0.0.0.0 universetoday.us.intellitxt.com +0.0.0.0 us.intellitxt.com +0.0.0.0 warp2search.us.intellitxt.com +0.0.0.0 wi-fitechnology.uk.intellitxt.com +0.0.0.0 worldnetdaily.us.intellitxt.com +# + +# + +# Red Sheriff and imrworldwide.com -- server side tracking +0.0.0.0 devfw.imrworldwide.com +0.0.0.0 fe1-au.imrworldwide.com +0.0.0.0 fe1-fi.imrworldwide.com +0.0.0.0 fe1-it.imrworldwide.com +0.0.0.0 fe2-au.imrworldwide.com +0.0.0.0 fe3-au.imrworldwide.com +0.0.0.0 fe3-gc.imrworldwide.com +0.0.0.0 fe3-uk.imrworldwide.com +0.0.0.0 fe4-uk.imrworldwide.com +0.0.0.0 fe-au.imrworldwide.com +0.0.0.0 imrworldwide.com +0.0.0.0 lycos-eu.imrworldwide.com +0.0.0.0 ninemsn.imrworldwide.com +0.0.0.0 rc-au.imrworldwide.com +0.0.0.0 redsheriff.com +#0.0.0.0 secure-au.imrworldwide.com +0.0.0.0 secure-jp.imrworldwide.com +0.0.0.0 secure-nz.imrworldwide.com +0.0.0.0 secure-uk.imrworldwide.com +0.0.0.0 secure-us.imrworldwide.com +0.0.0.0 secure-za.imrworldwide.com +0.0.0.0 server-au.imrworldwide.com +0.0.0.0 server-br.imrworldwide.com +0.0.0.0 server-by.imrworldwide.com +0.0.0.0 server-ca.imrworldwide.com +0.0.0.0 server-de.imrworldwide.com +0.0.0.0 server-dk.imrworldwide.com +0.0.0.0 server-ee.imrworldwide.com +0.0.0.0 server-fi.imrworldwide.com +0.0.0.0 server-fr.imrworldwide.com +0.0.0.0 server-hk.imrworldwide.com +0.0.0.0 server-it.imrworldwide.com +0.0.0.0 server-jp.imrworldwide.com +0.0.0.0 server-lt.imrworldwide.com +0.0.0.0 server-lv.imrworldwide.com +0.0.0.0 server-no.imrworldwide.com +0.0.0.0 server-nz.imrworldwide.com +0.0.0.0 server-oslo.imrworldwide.com +0.0.0.0 server-pl.imrworldwide.com +0.0.0.0 server-ru.imrworldwide.com +0.0.0.0 server-se.imrworldwide.com +0.0.0.0 server-sg.imrworldwide.com +0.0.0.0 server-stockh.imrworldwide.com +0.0.0.0 server-ua.imrworldwide.com +0.0.0.0 server-uk.imrworldwide.com +0.0.0.0 server-us.imrworldwide.com +0.0.0.0 server-za.imrworldwide.com +0.0.0.0 survey1-au.imrworldwide.com +0.0.0.0 telstra.imrworldwide.com +0.0.0.0 www.imrworldwide.com +0.0.0.0 www.imrworldwide.com.au +0.0.0.0 www.redsheriff.com +# + +# + +# cydoor -- server side tracking +0.0.0.0 cydoor.com +0.0.0.0 j.2004cms.com # cydoor +0.0.0.0 jbaventures.cjt1.net +0.0.0.0 jbeet.cjt1.net +0.0.0.0 jbit.cjt1.net +0.0.0.0 jcollegehumor.cjt1.net +0.0.0.0 jcontent.bns1.net +0.0.0.0 jdownloadacc.cjt1.net +0.0.0.0 jgen10.cjt1.net +0.0.0.0 jgen11.cjt1.net +0.0.0.0 jgen12.cjt1.net +0.0.0.0 jgen13.cjt1.net +0.0.0.0 jgen14.cjt1.net +0.0.0.0 jgen15.cjt1.net +0.0.0.0 jgen16.cjt1.net +0.0.0.0 jgen17.cjt1.net +0.0.0.0 jgen18.cjt1.net +0.0.0.0 jgen19.cjt1.net +0.0.0.0 jgen1.cjt1.net +0.0.0.0 jgen20.cjt1.net +0.0.0.0 jgen21.cjt1.net +0.0.0.0 jgen22.cjt1.net +0.0.0.0 jgen23.cjt1.net +0.0.0.0 jgen24.cjt1.net +0.0.0.0 jgen25.cjt1.net +0.0.0.0 jgen26.cjt1.net +0.0.0.0 jgen27.cjt1.net +0.0.0.0 jgen28.cjt1.net +0.0.0.0 jgen29.cjt1.net +0.0.0.0 jgen2.cjt1.net +0.0.0.0 jgen30.cjt1.net +0.0.0.0 jgen31.cjt1.net +0.0.0.0 jgen32.cjt1.net +0.0.0.0 jgen33.cjt1.net +0.0.0.0 jgen34.cjt1.net +0.0.0.0 jgen35.cjt1.net +0.0.0.0 jgen36.cjt1.net +0.0.0.0 jgen37.cjt1.net +0.0.0.0 jgen38.cjt1.net +0.0.0.0 jgen39.cjt1.net +0.0.0.0 jgen3.cjt1.net +0.0.0.0 jgen40.cjt1.net +0.0.0.0 jgen41.cjt1.net +0.0.0.0 jgen42.cjt1.net +0.0.0.0 jgen43.cjt1.net +0.0.0.0 jgen44.cjt1.net +0.0.0.0 jgen45.cjt1.net +0.0.0.0 jgen46.cjt1.net +0.0.0.0 jgen47.cjt1.net +0.0.0.0 jgen48.cjt1.net +0.0.0.0 jgen49.cjt1.net +0.0.0.0 jgen4.cjt1.net +0.0.0.0 jgen5.cjt1.net +0.0.0.0 jgen6.cjt1.net +0.0.0.0 jgen7.cjt1.net +0.0.0.0 jgen8.cjt1.net +0.0.0.0 jgen9.cjt1.net +0.0.0.0 jhumour.cjt1.net +0.0.0.0 jmbi58.cjt1.net +0.0.0.0 jnova.cjt1.net +0.0.0.0 jpirate.cjt1.net +0.0.0.0 jsandboxer.cjt1.net +0.0.0.0 jumcna.cjt1.net +0.0.0.0 jwebbsense.cjt1.net +0.0.0.0 www.cydoor.com +# + +#<2o7-sites> + +# 2o7.net -- server side tracking +0.0.0.0 102.112.2o7.net +0.0.0.0 102.122.2o7.net +0.0.0.0 112.2o7.net +0.0.0.0 122.2o7.net +0.0.0.0 192.168.112.2o7.net +0.0.0.0 2o7.net +0.0.0.0 actforvictory.112.2o7.net +0.0.0.0 adbrite.112.2o7.net +0.0.0.0 adbrite.122.2o7.net +0.0.0.0 aehistory.112.2o7.net +0.0.0.0 aetv.112.2o7.net +0.0.0.0 agamgreetingscom.112.2o7.net +0.0.0.0 allbritton.122.2o7.net +0.0.0.0 americanbaby.112.2o7.net +0.0.0.0 ancestrymsn.112.2o7.net +0.0.0.0 ancestryuki.112.2o7.net +0.0.0.0 angiba.112.2o7.net +0.0.0.0 angmar.112.2o7.net +0.0.0.0 angtr.112.2o7.net +0.0.0.0 angts.112.2o7.net +0.0.0.0 angvac.112.2o7.net +0.0.0.0 anm.112.2o7.net +0.0.0.0 aolcareers.122.2o7.net +0.0.0.0 aoldlama.122.2o7.net +0.0.0.0 aoljournals.122.2o7.net +0.0.0.0 aolnsnews.122.2o7.net +0.0.0.0 aolpf.122.2o7.net +0.0.0.0 aolpolls.112.2o7.net +0.0.0.0 aolpolls.122.2o7.net +0.0.0.0 aolsearch.122.2o7.net +0.0.0.0 aolsvc.122.2o7.net +0.0.0.0 aoltmz.122.2o7.net +0.0.0.0 aolturnercnnmoney.112.2o7.net +0.0.0.0 aolturnercnnmoney.122.2o7.net +0.0.0.0 aolturnersi.122.2o7.net +0.0.0.0 aolukglobal.122.2o7.net +0.0.0.0 aolwinamp.122.2o7.net +0.0.0.0 aolwpaim.112.2o7.net +0.0.0.0 aolwpicq.122.2o7.net +0.0.0.0 aolwpmq.112.2o7.net +0.0.0.0 aolwpmqnoban.112.2o7.net +0.0.0.0 apdigitalorg.112.2o7.net +0.0.0.0 apdigitalorgovn.112.2o7.net +0.0.0.0 apnonline.112.2o7.net +#0.0.0.0 appleglobal.112.2o7.net #breaks apple.com +#0.0.0.0 applestoreus.112.2o7.net #breaks apple.com +0.0.0.0 atlassian.122.2o7.net +0.0.0.0 autobytel.112.2o7.net +0.0.0.0 autoweb.112.2o7.net +0.0.0.0 bbcnewscouk.112.2o7.net +0.0.0.0 bellca.112.2o7.net +0.0.0.0 bellglobemediapublishing.122.2o7.net +0.0.0.0 bellglovemediapublishing.122.2o7.net +0.0.0.0 bellserviceeng.112.2o7.net +0.0.0.0 betterhg.112.2o7.net +0.0.0.0 bhgmarketing.112.2o7.net +0.0.0.0 bidentonrccom.122.2o7.net +0.0.0.0 biwwltvcom.112.2o7.net +0.0.0.0 biwwltvcom.122.2o7.net +0.0.0.0 blackpress.122.2o7.net +0.0.0.0 bnkr8dev.112.2o7.net +0.0.0.0 bntbcstglobal.112.2o7.net +0.0.0.0 bosecom.112.2o7.net +0.0.0.0 brightcove.112.2o7.net +0.0.0.0 bulldog.122.2o7.net +0.0.0.0 businessweekpoc.112.2o7.net +0.0.0.0 bzresults.122.2o7.net +0.0.0.0 cablevision.112.2o7.net +0.0.0.0 canwest.112.2o7.net +0.0.0.0 canwestcom.112.2o7.net +0.0.0.0 canwestglobal.112.2o7.net +0.0.0.0 capcityadvcom.112.2o7.net +0.0.0.0 capcityadvcom.122.2o7.net +0.0.0.0 careers.112.2o7.net +0.0.0.0 cartoonnetwork.122.2o7.net +0.0.0.0 cbaol.112.2o7.net +0.0.0.0 cbc.122.2o7.net +0.0.0.0 cbcca.112.2o7.net +0.0.0.0 cbcca.122.2o7.net +0.0.0.0 cbcincinnatienquirer.112.2o7.net +0.0.0.0 cbmsn.112.2o7.net +0.0.0.0 cbs.112.2o7.net +0.0.0.0 cbsncaasports.112.2o7.net +0.0.0.0 cbsnfl.112.2o7.net +0.0.0.0 cbspgatour.112.2o7.net +0.0.0.0 cbsspln.112.2o7.net +0.0.0.0 ccrbudgetca.112.2o7.net +0.0.0.0 ccrgaviscom.112.2o7.net +0.0.0.0 cfrfa.112.2o7.net +0.0.0.0 chicagosuntimes.122.2o7.net +0.0.0.0 chumtv.122.2o7.net +0.0.0.0 classifiedscanada.112.2o7.net +0.0.0.0 classmatescom.112.2o7.net +0.0.0.0 cmpglobalvista.112.2o7.net +0.0.0.0 cnetasiapacific.122.2o7.net +0.0.0.0 cnetaustralia.122.2o7.net +0.0.0.0 cneteurope.122.2o7.net +0.0.0.0 cnetnews.112.2o7.net +0.0.0.0 cnetzdnet.112.2o7.net +0.0.0.0 cnhienid.122.2o7.net +0.0.0.0 cnhimcalesternews.122.2o7.net +0.0.0.0 cnhipicayuneitemv.112.2o7.net +0.0.0.0 cnhitribunestar.122.2o7.net +0.0.0.0 cnhitribunestara.122.2o7.net +0.0.0.0 cnhregisterherald.122.2o7.net +0.0.0.0 cnn.122.2o7.net +0.0.0.0 computerworldcom.112.2o7.net +0.0.0.0 condenast.112.2o7.net +0.0.0.0 coxnetmasterglobal.112.2o7.net +0.0.0.0 coxpalmbeachpost.112.2o7.net +0.0.0.0 csoonlinecom.112.2o7.net +0.0.0.0 ctvcrimelibrary.112.2o7.net +0.0.0.0 ctvsmokinggun.112.2o7.net +0.0.0.0 cxociocom.112.2o7.net +0.0.0.0 denverpost.112.2o7.net +0.0.0.0 diginet.112.2o7.net +0.0.0.0 digitalhomediscountptyltd.122.2o7.net +0.0.0.0 disccglobal.112.2o7.net +0.0.0.0 disccstats.112.2o7.net +0.0.0.0 dischannel.112.2o7.net +0.0.0.0 divx.112.2o7.net +0.0.0.0 dixonslnkcouk.112.2o7.net +0.0.0.0 dogpile.112.2o7.net +0.0.0.0 donval.112.2o7.net +0.0.0.0 dowjones.122.2o7.net +0.0.0.0 dreammates.112.2o7.net +0.0.0.0 eaeacom.112.2o7.net +0.0.0.0 eagamesuk.112.2o7.net +0.0.0.0 earthlnkpsplive.122.2o7.net +0.0.0.0 ebay1.112.2o7.net +0.0.0.0 ebaynonreg.112.2o7.net +0.0.0.0 ebayreg.112.2o7.net +0.0.0.0 ebayus.112.2o7.net +0.0.0.0 ebcom.112.2o7.net +0.0.0.0 ectestlampsplus1.112.2o7.net +0.0.0.0 edietsmain.112.2o7.net +0.0.0.0 edmundsinsideline.112.2o7.net +0.0.0.0 edsa.112.2o7.net +0.0.0.0 ehg-moma.hitbox.com.112.2o7.net +0.0.0.0 emc.122.2o7.net +0.0.0.0 employ22.112.2o7.net +0.0.0.0 employ26.112.2o7.net +0.0.0.0 employment.112.2o7.net +0.0.0.0 enterprisenewsmedia.122.2o7.net +0.0.0.0 epost.122.2o7.net +0.0.0.0 ewsnaples.112.2o7.net +0.0.0.0 ewstcpalm.112.2o7.net +0.0.0.0 examinercom.122.2o7.net +0.0.0.0 execulink.112.2o7.net +0.0.0.0 expedia4.112.2o7.net +0.0.0.0 expedia.ca.112.2o7.net +0.0.0.0 f2ncracker.112.2o7.net +0.0.0.0 f2nsmh.112.2o7.net +0.0.0.0 f2ntheage.112.2o7.net +0.0.0.0 faceoff.112.2o7.net +0.0.0.0 fbkmnr.112.2o7.net +0.0.0.0 forbesattache.112.2o7.net +0.0.0.0 forbesauto.112.2o7.net +0.0.0.0 forbesautos.112.2o7.net +0.0.0.0 forbescom.112.2o7.net +0.0.0.0 ford.112.2o7.net +0.0.0.0 foxcom.112.2o7.net +0.0.0.0 foxsimpsons.112.2o7.net +0.0.0.0 georgewbush.112.2o7.net +0.0.0.0 georgewbushcom.112.2o7.net +0.0.0.0 gettyimages.122.2o7.net +0.0.0.0 gjfastcompanycom.112.2o7.net +0.0.0.0 gmchevyapprentice.112.2o7.net +0.0.0.0 gmhummer.112.2o7.net +0.0.0.0 gntbcstglobal.112.2o7.net +0.0.0.0 gntbcstkxtv.112.2o7.net +0.0.0.0 gntbcstwtsp.112.2o7.net +0.0.0.0 gpaper104.112.2o7.net +0.0.0.0 gpaper105.112.2o7.net +0.0.0.0 gpaper107.112.2o7.net +0.0.0.0 gpaper108.112.2o7.net +0.0.0.0 gpaper109.112.2o7.net +0.0.0.0 gpaper110.112.2o7.net +0.0.0.0 gpaper111.112.2o7.net +0.0.0.0 gpaper112.112.2o7.net +0.0.0.0 gpaper113.112.2o7.net +0.0.0.0 gpaper114.112.2o7.net +0.0.0.0 gpaper115.112.2o7.net +0.0.0.0 gpaper116.112.2o7.net +0.0.0.0 gpaper117.112.2o7.net +0.0.0.0 gpaper118.112.2o7.net +0.0.0.0 gpaper119.112.2o7.net +0.0.0.0 gpaper120.112.2o7.net +0.0.0.0 gpaper121.112.2o7.net +0.0.0.0 gpaper122.112.2o7.net +0.0.0.0 gpaper123.112.2o7.net +0.0.0.0 gpaper124.112.2o7.net +0.0.0.0 gpaper125.112.2o7.net +0.0.0.0 gpaper126.112.2o7.net +0.0.0.0 gpaper127.112.2o7.net +0.0.0.0 gpaper128.112.2o7.net +0.0.0.0 gpaper129.112.2o7.net +0.0.0.0 gpaper131.112.2o7.net +0.0.0.0 gpaper132.112.2o7.net +0.0.0.0 gpaper133.112.2o7.net +0.0.0.0 gpaper138.112.2o7.net +0.0.0.0 gpaper139.112.2o7.net +0.0.0.0 gpaper140.112.2o7.net +0.0.0.0 gpaper141.112.2o7.net +0.0.0.0 gpaper142.112.2o7.net +0.0.0.0 gpaper144.112.2o7.net +0.0.0.0 gpaper145.112.2o7.net +0.0.0.0 gpaper147.112.2o7.net +0.0.0.0 gpaper149.112.2o7.net +0.0.0.0 gpaper151.112.2o7.net +0.0.0.0 gpaper154.112.2o7.net +0.0.0.0 gpaper156.112.2o7.net +0.0.0.0 gpaper157.112.2o7.net +0.0.0.0 gpaper158.112.2o7.net +0.0.0.0 gpaper162.112.2o7.net +0.0.0.0 gpaper164.112.2o7.net +0.0.0.0 gpaper166.112.2o7.net +0.0.0.0 gpaper167.112.2o7.net +0.0.0.0 gpaper169.112.2o7.net +0.0.0.0 gpaper170.112.2o7.net +0.0.0.0 gpaper171.112.2o7.net +0.0.0.0 gpaper172.112.2o7.net +0.0.0.0 gpaper173.112.2o7.net +0.0.0.0 gpaper174.112.2o7.net +0.0.0.0 gpaper176.112.2o7.net +0.0.0.0 gpaper177.112.2o7.net +0.0.0.0 gpaper180.112.2o7.net +0.0.0.0 gpaper183.112.2o7.net +0.0.0.0 gpaper184.112.2o7.net +0.0.0.0 gpaper191.112.2o7.net +0.0.0.0 gpaper192.112.2o7.net +0.0.0.0 gpaper193.112.2o7.net +0.0.0.0 gpaper194.112.2o7.net +0.0.0.0 gpaper195.112.2o7.net +0.0.0.0 gpaper196.112.2o7.net +0.0.0.0 gpaper197.112.2o7.net +0.0.0.0 gpaper198.112.2o7.net +0.0.0.0 gpaper202.112.2o7.net +0.0.0.0 gpaper204.112.2o7.net +0.0.0.0 gpaper205.112.2o7.net +0.0.0.0 gpaper212.112.2o7.net +0.0.0.0 gpaper214.112.2o7.net +0.0.0.0 gpaper219.112.2o7.net +0.0.0.0 gpaper223.112.2o7.net +0.0.0.0 harpo.122.2o7.net +0.0.0.0 hchrmain.112.2o7.net +0.0.0.0 heavycom.112.2o7.net +0.0.0.0 heavycom.122.2o7.net +0.0.0.0 homesclick.112.2o7.net +0.0.0.0 hostdomainpeople.112.2o7.net +0.0.0.0 hostdomainpeopleca.112.2o7.net +0.0.0.0 hostpowermedium.112.2o7.net +0.0.0.0 hpglobal.112.2o7.net +0.0.0.0 hphqglobal.112.2o7.net +0.0.0.0 hphqsearch.112.2o7.net +0.0.0.0 infomart.ca.112.2o7.net +0.0.0.0 infospace.com.112.2o7.net +0.0.0.0 intelcorpcim.112.2o7.net +0.0.0.0 intelglobal.112.2o7.net +0.0.0.0 ivillageglobal.112.2o7.net +0.0.0.0 jijsonline.122.2o7.net +0.0.0.0 jitmj4.122.2o7.net +0.0.0.0 johnlewis.112.2o7.net +0.0.0.0 journalregistercompany.122.2o7.net +0.0.0.0 kddi.122.2o7.net +0.0.0.0 krafteurope.112.2o7.net +0.0.0.0 ktva.112.2o7.net +0.0.0.0 ladieshj.112.2o7.net +0.0.0.0 laptopmag.122.2o7.net +0.0.0.0 laxnws.112.2o7.net +0.0.0.0 laxprs.112.2o7.net +0.0.0.0 laxpsd.112.2o7.net +0.0.0.0 ldsfch.112.2o7.net +0.0.0.0 leeenterprises.112.2o7.net +0.0.0.0 lenovo.112.2o7.net +0.0.0.0 logoworksdev.112.2o7.net +0.0.0.0 losu.112.2o7.net +0.0.0.0 mailtribune.112.2o7.net +0.0.0.0 maxim.122.2o7.net +0.0.0.0 maxvr.112.2o7.net +0.0.0.0 mdamarillo.112.2o7.net +0.0.0.0 mdjacksonville.112.2o7.net +0.0.0.0 mdtopeka.112.2o7.net +0.0.0.0 mdwardmore.112.2o7.net +0.0.0.0 mdwsavannah.112.2o7.net +0.0.0.0 medbroadcast.112.2o7.net +0.0.0.0 mediabistrocom.112.2o7.net +0.0.0.0 mediamatters.112.2o7.net +0.0.0.0 meetupcom.112.2o7.net +0.0.0.0 metacafe.122.2o7.net +0.0.0.0 mgjournalnow.112.2o7.net +0.0.0.0 mgtbo.112.2o7.net +0.0.0.0 mgtimesdispatch.112.2o7.net +0.0.0.0 mgwsls.112.2o7.net +0.0.0.0 mgwspa.112.2o7.net +0.0.0.0 microsoftconsumermarketing.112.2o7.net +0.0.0.0 microsofteup.112.2o7.net +0.0.0.0 microsoftwindows.112.2o7.net +0.0.0.0 midala.112.2o7.net +0.0.0.0 midar.112.2o7.net +0.0.0.0 midsen.112.2o7.net +0.0.0.0 mlbastros.112.2o7.net +0.0.0.0 mlbcolorado.112.2o7.net +0.0.0.0 mlbcom.112.2o7.net +0.0.0.0 mlbglobal08.112.2o7.net +0.0.0.0 mlbglobal.112.2o7.net +0.0.0.0 mlbhouston.112.2o7.net +0.0.0.0 mlbstlouis.112.2o7.net +0.0.0.0 mlbtoronto.112.2o7.net +0.0.0.0 mmsshopcom.112.2o7.net +0.0.0.0 mnfidnahub.112.2o7.net +0.0.0.0 mngidmn.112.2o7.net +0.0.0.0 mngirockymtnnews.112.2o7.net +0.0.0.0 mngislctrib.112.2o7.net +0.0.0.0 mngiyrkdr.112.2o7.net +0.0.0.0 mseuppremain.112.2o7.net +0.0.0.0 msnmercom.112.2o7.net +0.0.0.0 msnportal.112.2o7.net +0.0.0.0 mtvn.112.2o7.net +0.0.0.0 mtvu.112.2o7.net +0.0.0.0 mxmacromedia.112.2o7.net +0.0.0.0 myfamilyancestry.112.2o7.net +0.0.0.0 nasdaq.122.2o7.net +0.0.0.0 natgeoeditco.112.2o7.net +0.0.0.0 natgeoeditcom.112.2o7.net +0.0.0.0 natgeonews.112.2o7.net +0.0.0.0 natgeongmcom.112.2o7.net +0.0.0.0 nationalpost.112.2o7.net +0.0.0.0 nba.112.2o7.net +0.0.0.0 neber.112.2o7.net +0.0.0.0 netrp.112.2o7.net +0.0.0.0 netsdartboards.122.2o7.net +0.0.0.0 newsinteractive.112.2o7.net +0.0.0.0 newstimeslivecom.112.2o7.net +0.0.0.0 nike.112.2o7.net +0.0.0.0 nikeplus.112.2o7.net +0.0.0.0 nmanchorage.112.2o7.net +0.0.0.0 nmbrampton.112.2o7.net +0.0.0.0 nmcommancomedia.112.2o7.net +0.0.0.0 nmfresno.112.2o7.net +0.0.0.0 nmhiltonhead.112.2o7.net +0.0.0.0 nmkawartha.112.2o7.net +0.0.0.0 nmminneapolis.112.2o7.net +0.0.0.0 nmmississauga.112.2o7.net +0.0.0.0 nmnandomedia.112.2o7.net +0.0.0.0 nmraleigh.112.2o7.net +0.0.0.0 nmrockhill.112.2o7.net +0.0.0.0 nmsacramento.112.2o7.net +0.0.0.0 nmtoronto.112.2o7.net +0.0.0.0 nmtricity.112.2o7.net +0.0.0.0 nmyork.112.2o7.net +0.0.0.0 novellcom.112.2o7.net +0.0.0.0 nytbglobe.112.2o7.net +0.0.0.0 nytglobe.112.2o7.net +0.0.0.0 nythglobe.112.2o7.net +0.0.0.0 nytimesglobal.112.2o7.net +0.0.0.0 nytimesnonsampled.112.2o7.net +0.0.0.0 nytimesnoonsampled.112.2o7.net +0.0.0.0 nytmembercenter.112.2o7.net +0.0.0.0 nytrflorence.112.2o7.net +0.0.0.0 nytrgadsden.112.2o7.net +0.0.0.0 nytrgainseville.112.2o7.net +0.0.0.0 nytrhendersonville.112.2o7.net +0.0.0.0 nytrhouma.112.2o7.net +0.0.0.0 nytrlakeland.112.2o7.net +0.0.0.0 nytrsantarosa.112.2o7.net +0.0.0.0 nytrsarasota.112.2o7.net +0.0.0.0 nytrwilmington.112.2o7.net +0.0.0.0 nyttechnology.112.2o7.net +0.0.0.0 omniture.112.2o7.net +0.0.0.0 omnitureglobal.112.2o7.net +0.0.0.0 onlineindigoca.112.2o7.net +0.0.0.0 oracle.112.2o7.net +0.0.0.0 oraclecom.112.2o7.net +0.0.0.0 overstock.com.112.2o7.net +0.0.0.0 overturecomvista.112.2o7.net +0.0.0.0 paypal.112.2o7.net +0.0.0.0 poacprod.122.2o7.net +0.0.0.0 poconorecordcom.112.2o7.net +0.0.0.0 projectorpeople.112.2o7.net +0.0.0.0 publicationsunbound.112.2o7.net +0.0.0.0 pulharktheherald.112.2o7.net +0.0.0.0 pulpantagraph.112.2o7.net +0.0.0.0 rckymtnnws.112.2o7.net +0.0.0.0 recordnetcom.112.2o7.net +0.0.0.0 recordonlinecom.112.2o7.net +0.0.0.0 rey3935.112.2o7.net +0.0.0.0 rezrezwhistler.112.2o7.net +0.0.0.0 riptownmedia.122.2o7.net +0.0.0.0 rncgopcom.122.2o7.net +0.0.0.0 roxio.112.2o7.net +0.0.0.0 salesforce.122.2o7.net +0.0.0.0 santacruzsentinel.112.2o7.net +0.0.0.0 sciamglobal.112.2o7.net +0.0.0.0 scrippsbathvert.112.2o7.net +0.0.0.0 scrippsfoodnet.112.2o7.net +0.0.0.0 scrippswfts.112.2o7.net +0.0.0.0 scrippswxyz.112.2o7.net +0.0.0.0 seacoastonlinecom.112.2o7.net +0.0.0.0 searscom.112.2o7.net +0.0.0.0 smibs.112.2o7.net +0.0.0.0 smwww.112.2o7.net +0.0.0.0 sonycorporate.122.2o7.net +0.0.0.0 sonyglobal.112.2o7.net +0.0.0.0 southcoasttoday.112.2o7.net +0.0.0.0 spiketv.112.2o7.net +0.0.0.0 stpetersburgtimes.122.2o7.net +0.0.0.0 suncom.112.2o7.net +0.0.0.0 sunglobal.112.2o7.net +0.0.0.0 sunonesearch.112.2o7.net +0.0.0.0 survey.112.2o7.net +0.0.0.0 sympmsnsports.112.2o7.net +0.0.0.0 techreview.112.2o7.net +0.0.0.0 thestar.122.2o7.net +0.0.0.0 thestardev.122.2o7.net +0.0.0.0 thinkgeek.112.2o7.net +0.0.0.0 timebus2.112.2o7.net +0.0.0.0 timecom.112.2o7.net +0.0.0.0 timeew.122.2o7.net +0.0.0.0 timefortune.112.2o7.net +0.0.0.0 timehealth.112.2o7.net +0.0.0.0 timeofficepirates.122.2o7.net +0.0.0.0 timepeople.122.2o7.net +0.0.0.0 timepopsci.122.2o7.net +0.0.0.0 timerealsimple.112.2o7.net +0.0.0.0 timewarner.122.2o7.net +0.0.0.0 tmsscion.112.2o7.net +0.0.0.0 tmstoyota.112.2o7.net +0.0.0.0 tnttv.112.2o7.net +0.0.0.0 torstardigital.122.2o7.net +0.0.0.0 travidiathebrick.112.2o7.net +0.0.0.0 tribuneinteractive.122.2o7.net +0.0.0.0 usatoday1.112.2o7.net +0.0.0.0 usnews.122.2o7.net +0.0.0.0 usun.112.2o7.net +0.0.0.0 vanns.112.2o7.net +0.0.0.0 verisignwildcard.112.2o7.net +0.0.0.0 verisonwildcard.112.2o7.net +0.0.0.0 vh1com.112.2o7.net +0.0.0.0 viaatomvideo.112.2o7.net +0.0.0.0 viacomedycentralrl.112.2o7.net +0.0.0.0 viagametrailers.112.2o7.net +0.0.0.0 viamtvcom.112.2o7.net +0.0.0.0 viasyndimedia.112.2o7.net +0.0.0.0 viavh1com.112.2o7.net +0.0.0.0 viay2m.112.2o7.net +0.0.0.0 vintacom.112.2o7.net +0.0.0.0 viralvideo.112.2o7.net +0.0.0.0 walmartcom.112.2o7.net +0.0.0.0 westjet.112.2o7.net +0.0.0.0 wileydumcom.112.2o7.net +0.0.0.0 wmg.112.2o7.net +0.0.0.0 wmgmulti.112.2o7.net +0.0.0.0 workopolis.122.2o7.net +0.0.0.0 wpni.112.2o7.net +0.0.0.0 xhealthmobiletools.112.2o7.net +0.0.0.0 youtube.112.2o7.net +0.0.0.0 yrkeve.112.2o7.net +0.0.0.0 ziffdavisglobal.112.2o7.net +0.0.0.0 ziffdavispennyarcade.112.2o7.net +# + + +# ads +0.0.0.0 0101011.com +0.0.0.0 0427d7.se +0.0.0.0 0d79ed.r.axf8.net +0.0.0.0 104231.dtiblog.com +0.0.0.0 10.im.cz +0.0.0.0 123.fluxads.com +0.0.0.0 123specialgifts.com +#0.0.0.0 140cc.v.fwmrm.net #interferes with Comedy Central videos +0.0.0.0 1.adbrite.com +0.0.0.0 1.forgetstore.com +0.0.0.0 1.httpads.com +0.0.0.0 1.primaryads.com +0.0.0.0 207-87-18-203.wsmg.digex.net +0.0.0.0 247support.adtech.fr +0.0.0.0 247support.adtech.us +0.0.0.0 24ratownik.hit.gemius.pl +0.0.0.0 24trk.com +0.0.0.0 25184.hittail.com +0.0.0.0 2754.btrll.com +0.0.0.0 2912a.v.fwmrm.net +0.0.0.0 2.adbrite.com +0.0.0.0 2-art-coliseum.com +0.0.0.0 312.1d27c9b8fb.com +0.0.0.0 321cba.com +0.0.0.0 32red.it +0.0.0.0 360ads.com +0.0.0.0 3.adbrite.com +0.0.0.0 3.cennter.com +0.0.0.0 3fns.com +0.0.0.0 4.adbrite.com +0.0.0.0 4c28d6.r.axf8.net +0.0.0.0 4qinvite.4q.iperceptions.com +0.0.0.0 7500.com +0.0.0.0 76.a.boom.ro +0.0.0.0 7adpower.com +0.0.0.0 7bpeople.com +0.0.0.0 7bpeople.data.7bpeople.com +0.0.0.0 7cnbcnews.com +0.0.0.0 85103.hittail.com +0.0.0.0 8574dnj3yzjace8c8io6zr9u3n.hop.clickbank.net +0.0.0.0 888casino.com +0.0.0.0 961.com +0.0.0.0 9cf9.v.fwmrm.net +0.0.0.0 a01.gestionpub.com +0.0.0.0 a.0day.kiev.ua +0.0.0.0 a1.greenadworks.net +0.0.0.0 a1.interclick.com +0.0.0.0 a200.yieldoptimizer.com +0.0.0.0 a2.mediagra.com +0.0.0.0 a2.websponsors.com +0.0.0.0 a3.suntimes.com +0.0.0.0 a3.websponsors.com +0.0.0.0 a4.websponsors.com +0.0.0.0 a5.websponsors.com +0.0.0.0 a.admaxserver.com +0.0.0.0 a.adorika.net +0.0.0.0 a.ad.playstation.net +0.0.0.0 a.adready.com +0.0.0.0 a.ads1.msn.com +0.0.0.0 a.ads2.msn.com +0.0.0.0 a.adstome.com +0.0.0.0 aads.treehugger.com +0.0.0.0 aams1.aim4media.com +0.0.0.0 aan.amazon.com +0.0.0.0 aa-nb.marketgid.com +0.0.0.0 aa.newsblock.dt00.net +0.0.0.0 aa.newsblock.marketgid.com +0.0.0.0 a.as-eu.falkag.net +0.0.0.0 a.as-us.falkag.net +0.0.0.0 aax-us-east.amazon-adsystem.com +0.0.0.0 abcnews.footprint.net +0.0.0.0 a.boom.ro +0.0.0.0 abrogatesdv.info +0.0.0.0 abseckw.adtlgc.com +0.0.0.0 a.collective-media.net +0.0.0.0 ac.rnm.ca +0.0.0.0 actiondesk.com +0.0.0.0 actionflash.com +0.0.0.0 action.ientry.net +0.0.0.0 action.mathtag.com +0.0.0.0 action.media6degrees.com +0.0.0.0 actionsplash.com +0.0.0.0 ac.tynt.com +0.0.0.0 acvs.mediaonenetwork.net +0.0.0.0 acvsrv.mediaonenetwork.net +0.0.0.0 ad01.adonspot.com +0.0.0.0 ad01.focalink.com +0.0.0.0 ad01.mediacorpsingapore.com +0.0.0.0 ad02.focalink.com +0.0.0.0 ad03.focalink.com +0.0.0.0 ad04.focalink.com +0.0.0.0 ad05.focalink.com +0.0.0.0 ad06.focalink.com +0.0.0.0 ad07.focalink.com +0.0.0.0 ad08.focalink.com +0.0.0.0 ad09.focalink.com +0.0.0.0 ad0.haynet.com +0.0.0.0 ad101com.adbureau.net +0.0.0.0 ad10.bannerbank.ru +0.0.0.0 ad10.focalink.com +0.0.0.0 ad11.bannerbank.ru +0.0.0.0 ad11.focalink.com +0.0.0.0 ad12.bannerbank.ru +0.0.0.0 ad12.focalink.com +0.0.0.0 ad13.focalink.com +0.0.0.0 ad14.focalink.com +0.0.0.0 ad15.focalink.com +0.0.0.0 ad16.focalink.com +0.0.0.0 ad17.focalink.com +0.0.0.0 ad18.focalink.com +0.0.0.0 ad19.focalink.com +0.0.0.0 ad1.adtitan.net +0.0.0.0 ad1.bannerbank.ru +0.0.0.0 ad1.clickhype.com +0.0.0.0 ad1.emediate.dk +0.0.0.0 ad1.emediate.se +0.0.0.0 ad1.gamezone.com +0.0.0.0 ad1.hotel.com +0.0.0.0 ad1.lbn.ru +0.0.0.0 ad1.peel.com +0.0.0.0 ad1.popcap.com +0.0.0.0 ad1.yomiuri.co.jp +0.0.0.0 ad1.yourmedia.com +0.0.0.0 ad234.prbn.ru +0.0.0.0 ad2.adecn.com +0.0.0.0 ad2.bal.dotandad.com +0.0.0.0 ad2.bannerbank.ru +0.0.0.0 ad2.bannerhost.ru +0.0.0.0 ad2.bbmedia.cz +0.0.0.0 ad2.cooks.com +0.0.0.0 ad2.firehousezone.com +0.0.0.0 ad2games.com +0.0.0.0 ad2.gammae.com +0.0.0.0 ad2.hotel.com +0.0.0.0 ad2.ip.ro +0.0.0.0 ad2.lbn.ru +0.0.0.0 ad2.nationalreview.com +0.0.0.0 ad2.pamedia.com +0.0.0.0 ad2.parom.hu +0.0.0.0 ad2.peel.com +0.0.0.0 ad2.pl +0.0.0.0 ad2.pl.mediainter.net +0.0.0.0 ad2.sbisec.co.jp +0.0.0.0 ad2.smni.com +0.0.0.0 ad.360yield.com +0.0.0.0 ad3.adfarm1.adition.com +0.0.0.0 ad3.bannerbank.ru +0.0.0.0 ad3.bb.ru +0.0.0.0 ad.3dnews.ru +0.0.0.0 ad3.lbn.ru +0.0.0.0 ad3.nationalreview.com +0.0.0.0 ad3.rambler.ru +0.0.0.0 ad41.atlas.cz +0.0.0.0 ad4.adfarm1.adition.com +0.0.0.0 ad4.bannerbank.ru +0.0.0.0 ad4.lbn.ru +0.0.0.0 ad4.liverail.com +0.0.0.0 ad4.speedbit.com +0.0.0.0 ad5.bannerbank.ru +0.0.0.0 ad5.lbn.ru +0.0.0.0 ad6.bannerbank.ru +0.0.0.0 ad6.horvitznewspapers.net +0.0.0.0 ad.71i.de +0.0.0.0 ad7.bannerbank.ru +0.0.0.0 ad8.bannerbank.ru +0.0.0.0 ad9.bannerbank.ru +0.0.0.0 ad.abcnews.com +0.0.0.0 ad.aboutwebservices.com +0.0.0.0 ad.adfunky.com +0.0.0.0 ad.adition.de +0.0.0.0 ad.adition.net +0.0.0.0 ad.adlegend.com +0.0.0.0 ad.admarketplace.net +0.0.0.0 ad.adnet.biz +0.0.0.0 ad.adnet.de +0.0.0.0 ad.adnetwork.com.br +0.0.0.0 ad.adnetwork.net +0.0.0.0 ad.adorika.com +0.0.0.0 ad.adperium.com +0.0.0.0 ad.adriver.ru +0.0.0.0 ad.adserve.com +0.0.0.0 ad.adserverplus.com +0.0.0.0 ad.adsmart.net +0.0.0.0 ad.adtegrity.net +0.0.0.0 ad.adtoma.com +0.0.0.0 ad.adverticum.net +0.0.0.0 ad.advertstream.com +0.0.0.0 ad.adview.pl +0.0.0.0 ad.afilo.pl +0.0.0.0 ad.aftenposten.no +0.0.0.0 ad.aftonbladet.se +0.0.0.0 ad.afy11.net +0.0.0.0 ad.agava.tbn.ru +0.0.0.0 adagiobanner.s3.amazonaws.com +0.0.0.0 ad.agkn.com +0.0.0.0 ad.amgdgt.com +0.0.0.0 adap.tv +0.0.0.0 ad.aquamediadirect.com +0.0.0.0 ad.asv.de +0.0.0.0 ad-audit.tubemogul.com +0.0.0.0 ad.auditude.com +0.0.0.0 ad.bannerbank.ru +0.0.0.0 ad.bannerconnect.net +0.0.0.0 adblade.com +0.0.0.0 ad.bnmla.com +0.0.0.0 adbnr.ru +0.0.0.0 adbot.theonion.com +0.0.0.0 adbrite.com +0.0.0.0 adbucks.brandreachsys.com +0.0.0.0 adc2.adcentriconline.com +0.0.0.0 adcache.aftenposten.no +0.0.0.0 adcanadian.com +0.0.0.0 adcash.com +0.0.0.0 adcast.deviantart.com +0.0.0.0 adcentriconline.com +0.0.0.0 adcentric.randomseed.com +0.0.0.0 ad.cibleclick.com +0.0.0.0 ad.clickdistrict.com +0.0.0.0 adclick.hit.gemius.pl +0.0.0.0 ad.clickotmedia.com +0.0.0.0 adclient-af.lp.uol.com.br +0.0.0.0 adclient.uimserv.net +0.0.0.0 adcode.adengage.com +0.0.0.0 adcontent.gamespy.com +0.0.0.0 adcontent.reedbusiness.com +0.0.0.0 adcontent.videoegg.com +0.0.0.0 adcontroller.unicast.com +0.0.0.0 adcount.ohmynews.com +0.0.0.0 adcreative.tribuneinteractive.com +0.0.0.0 adcycle.footymad.net +0.0.0.0 adcycle.icpeurope.net +0.0.0.0 ad.dc2.adtech.de +0.0.0.0 addelivery.thestreet.com +0.0.0.0 ad.designtaxi.com +0.0.0.0 ad.deviantart.com +0.0.0.0 ad.directrev.com +0.0.0.0 addthiscdn.com +0.0.0.0 addthis.com +0.0.0.0 adecn.com +0.0.0.0 ad.egloos.com +0.0.0.0 adengine.rt.ru +0.0.0.0 ad.espn.starwave.com +0.0.0.0 ad.eurosport.com +0.0.0.0 adexpansion.com +0.0.0.0 adexprt.com +0.0.0.0 adexprt.me +0.0.0.0 adexprts.com +0.0.0.0 adext.inkclub.com +0.0.0.0 adfarm1.adition.com +0.0.0.0 adfarm.mserve.ca +0.0.0.0 adfiles.pitchforkmedia.com +0.0.0.0 ad.filmweb.pl +0.0.0.0 ad.firstadsolution.com +0.0.0.0 ad.flux.com +#0.0.0.0 adf.ly +0.0.0.0 adforce.ads.imgis.com +0.0.0.0 adforce.adtech.de +0.0.0.0 adforce.adtech.fr +0.0.0.0 adforce.adtech.us +0.0.0.0 adforce.imgis.com +0.0.0.0 adform.com +0.0.0.0 adfu.blockstackers.com +0.0.0.0 ad.funpic.de +0.0.0.0 adfusion.com +0.0.0.0 ad.garantiarkadas.com +0.0.0.0 adgardener.com +0.0.0.0 ad.gazeta.pl +0.0.0.0 ad.goo.ne.jp +0.0.0.0 adgraphics.theonion.com +0.0.0.0 ad.gra.pl +0.0.0.0 ad.gr.doubleclick.net +0.0.0.0 ad.greenmarquee.com +0.0.0.0 adgroup.naver.com +0.0.0.0 ad.hankooki.com +0.0.0.0 ad.harrenmedianetwork.com +0.0.0.0 adhearus.com +0.0.0.0 adhese.be +0.0.0.0 adhese.com +0.0.0.0 adhitzads.com +0.0.0.0 ad.horvitznewspapers.net +0.0.0.0 ad.host.bannerflow.com +0.0.0.0 ad.howstuffworks.com +0.0.0.0 adhref.pl +#0.0.0.0 ad.hulu.com # Uncomment to block Hulu. +0.0.0.0 ad.iconadserver.com +0.0.0.0 adidm.idmnet.pl +0.0.0.0 adidm.supermedia.pl +0.0.0.0 adimage.asia1.com.sg +0.0.0.0 adimage.asiaone.com +0.0.0.0 adimage.asiaone.com.sg +0.0.0.0 adimage.blm.net +0.0.0.0 adimages.earthweb.com +0.0.0.0 adimages.go.com +0.0.0.0 adimages.mp3.com +0.0.0.0 adimages.watchmygf.net +0.0.0.0 adi.mainichi.co.jp +0.0.0.0 adimg.activeadv.net +0.0.0.0 adimg.com.com +0.0.0.0 adincl.gopher.com +0.0.0.0 ad.insightexpressai.com +0.0.0.0 ad.investopedia.com +0.0.0.0 adipics.com +0.0.0.0 adireland.com +0.0.0.0 ad.ir.ru +0.0.0.0 ad.isohunt.com +0.0.0.0 adition.com +0.0.0.0 ad.iwin.com +0.0.0.0 adj10.thruport.com +0.0.0.0 adj11.thruport.com +0.0.0.0 adj12.thruport.com +0.0.0.0 adj13.thruport.com +0.0.0.0 adj14.thruport.com +0.0.0.0 adj15.thruport.com +0.0.0.0 adj16r1.thruport.com +0.0.0.0 adj16.thruport.com +0.0.0.0 adj17.thruport.com +0.0.0.0 adj18.thruport.com +0.0.0.0 adj19.thruport.com +0.0.0.0 adj1.thruport.com +0.0.0.0 adj22.thruport.com +0.0.0.0 adj23.thruport.com +0.0.0.0 adj24.thruport.com +0.0.0.0 adj25.thruport.com +0.0.0.0 adj26.thruport.com +0.0.0.0 adj27.thruport.com +0.0.0.0 adj28.thruport.com +0.0.0.0 adj29.thruport.com +0.0.0.0 adj2.thruport.com +0.0.0.0 adj30.thruport.com +0.0.0.0 adj31.thruport.com +0.0.0.0 adj32.thruport.com +0.0.0.0 adj33.thruport.com +0.0.0.0 adj34.thruport.com +0.0.0.0 adj35.thruport.com +0.0.0.0 adj36.thruport.com +0.0.0.0 adj37.thruport.com +0.0.0.0 adj38.thruport.com +0.0.0.0 adj39.thruport.com +0.0.0.0 adj3.thruport.com +0.0.0.0 adj40.thruport.com +0.0.0.0 adj41.thruport.com +0.0.0.0 adj43.thruport.com +0.0.0.0 adj44.thruport.com +0.0.0.0 adj45.thruport.com +0.0.0.0 adj46.thruport.com +0.0.0.0 adj47.thruport.com +0.0.0.0 adj48.thruport.com +0.0.0.0 adj49.thruport.com +0.0.0.0 adj4.thruport.com +0.0.0.0 adj50.thruport.com +0.0.0.0 adj51.thruport.com +0.0.0.0 adj52.thruport.com +0.0.0.0 adj53.thruport.com +0.0.0.0 adj54.thruport.com +0.0.0.0 adj55.thruport.com +0.0.0.0 adj56.thruport.com +0.0.0.0 adj5.thruport.com +0.0.0.0 adj6.thruport.com +0.0.0.0 adj7.thruport.com +0.0.0.0 adj8.thruport.com +0.0.0.0 adj9.thruport.com +0.0.0.0 ad.jamba.net +0.0.0.0 ad.jamster.ca +0.0.0.0 adjmps.com +0.0.0.0 adjuggler.net +0.0.0.0 adjuggler.yourdictionary.com +0.0.0.0 ad.kataweb.it +0.0.0.0 ad.kat.ph +0.0.0.0 adkontekst.pl +0.0.0.0 ad.krutilka.ru +0.0.0.0 ad.leadcrunch.com +0.0.0.0 ad.lgappstv.com +0.0.0.0 ad.linkexchange.com +0.0.0.0 ad.linksynergy.com +0.0.0.0 admanager1.collegepublisher.com +0.0.0.0 admanager2.broadbandpublisher.com +0.0.0.0 admanager3.collegepublisher.com +0.0.0.0 admanager.adam4adam.com +0.0.0.0 admanager.beweb.com +0.0.0.0 admanager.btopenworld.com +0.0.0.0 admanager.collegepublisher.com +0.0.0.0 adman.freeze.com +0.0.0.0 adman.in.gr +0.0.0.0 ad.mastermedia.ru +0.0.0.0 admatcher.videostrip.com #http://admatcher.videostrip.com/?puid=23940627&host=www.dumpert.nl&categories=default +0.0.0.0 admatch-syndication.mochila.com +0.0.0.0 admax.quisma.com +0.0.0.0 ad.media-servers.net +0.0.0.0 admedia.xoom.com +0.0.0.0 admeld.com +0.0.0.0 admeta.vo.llnwd.net +#0.0.0.0 adm.fwmrm.net #may interfere with nhl.com +0.0.0.0 admin.digitalacre.com +0.0.0.0 admin.hotkeys.com +0.0.0.0 admin.inq.com +0.0.0.0 admonkey.dapper.net +0.0.0.0 ad.moscowtimes.ru +0.0.0.0 adm.shacknews.com +0.0.0.0 adms.physorg.com +0.0.0.0 ad.my.doubleclick.net +0.0.0.0 ad.nate.com +0.0.0.0 adn.ebay.com +0.0.0.0 adnet.asahi.com +0.0.0.0 adnet.biz +0.0.0.0 adnet.chicago.tribune.com +0.0.0.0 adnet.com +0.0.0.0 adnet.de +0.0.0.0 ad.network60.com +0.0.0.0 adnetwork.nextgen.net +0.0.0.0 adnetwork.rovicorp.com +0.0.0.0 adnetxchange.com +0.0.0.0 adng.ascii24.com +0.0.0.0 adn.kinkydollars.com +0.0.0.0 ad.nozonedata.com +0.0.0.0 adnxs.com +0.0.0.0 adnxs.revsci.net +0.0.0.0 adobee.com +0.0.0.0 adobe.tt.omtrdc.net +0.0.0.0 adocean.pl +0.0.0.0 ad.ohmynews.com +0.0.0.0 adopt.euroclick.com +0.0.0.0 adopt.precisead.com +0.0.0.0 adotube.com +0.0.0.0 ad.parom.hu +0.0.0.0 ad.partis.si +0.0.0.0 adpepper.dk +0.0.0.0 adp.gazeta.pl +0.0.0.0 ad.ph-prt.tbn.ru +0.0.0.0 adpick.switchboard.com +0.0.0.0 ad.pravda.ru +0.0.0.0 ad.preferences.com +0.0.0.0 ad.pro-advertising.com +0.0.0.0 ad.propellerads.com +0.0.0.0 ad.prv.pl +0.0.0.0 adpulse.ads.targetnet.com +0.0.0.0 adpush.dreamscape.com +0.0.0.0 adq.nextag.com +0.0.0.0 adremote.pathfinder.com +0.0.0.0 adremote.timeinc.aol.com +0.0.0.0 adremote.timeinc.net +0.0.0.0 ad.repubblica.it +0.0.0.0 adriver.ru +0.0.0.0 adroll.com +0.0.0.0 adrotate.se +0.0.0.0 adrotator.se +0.0.0.0 ad.ru.doubleclick.net +0.0.0.0 ads01.focalink.com +0.0.0.0 ads01.hyperbanner.net +0.0.0.0 ads02.focalink.com +0.0.0.0 ads02.hyperbanner.net +0.0.0.0 ads03.focalink.com +0.0.0.0 ads03.hyperbanner.net +0.0.0.0 ads04.focalink.com +0.0.0.0 ads04.hyperbanner.net +0.0.0.0 ads05.focalink.com +0.0.0.0 ads05.hyperbanner.net +0.0.0.0 ads06.focalink.com +0.0.0.0 ads06.hyperbanner.net +0.0.0.0 ads07.focalink.com +0.0.0.0 ads07.hyperbanner.net +0.0.0.0 ads08.focalink.com +0.0.0.0 ads08.hyperbanner.net +0.0.0.0 ads09.focalink.com +0.0.0.0 ads09.hyperbanner.net +0.0.0.0 ads0.okcupid.com +0.0.0.0 ads10.focalink.com +0.0.0.0 ads10.hyperbanner.net +0.0.0.0 ads10.speedbit.com +0.0.0.0 ads10.udc.advance.net +0.0.0.0 ads11.focalink.com +0.0.0.0 ads11.hyperbanner.net +0.0.0.0 ads11.udc.advance.net +0.0.0.0 ads12.focalink.com +0.0.0.0 ads12.hyperbanner.net +0.0.0.0 ads12.udc.advance.net +0.0.0.0 ads13.focalink.com +0.0.0.0 ads13.hyperbanner.net +0.0.0.0 ads13.udc.advance.net +0.0.0.0 ads14.bpath.com +0.0.0.0 ads14.focalink.com +0.0.0.0 ads14.hyperbanner.net +0.0.0.0 ads14.udc.advance.net +0.0.0.0 ads15.bpath.com +0.0.0.0 ads15.focalink.com +0.0.0.0 ads15.hyperbanner.net +0.0.0.0 ads15.udc.advance.net +0.0.0.0 ads16.advance.net +0.0.0.0 ads16.focalink.com +0.0.0.0 ads16.hyperbanner.net +0.0.0.0 ads16.udc.advance.net +0.0.0.0 ads17.focalink.com +0.0.0.0 ads17.hyperbanner.net +0.0.0.0 ads18.focalink.com +0.0.0.0 ads18.hyperbanner.net +0.0.0.0 ads19.focalink.com +0.0.0.0 ads1.activeagent.at +0.0.0.0 ads1.ad-flow.com +0.0.0.0 ads1.admedia.ro +0.0.0.0 ads1.advance.net +0.0.0.0 ads1.advertwizard.com +0.0.0.0 ads1.ami-admin.com +0.0.0.0 ads1.canoe.ca +0.0.0.0 ads1.destructoid.com +0.0.0.0 ads1.empiretheatres.com +0.0.0.0 ads1.erotism.com +0.0.0.0 ads1.eudora.com +0.0.0.0 ads1.globeandmail.com +0.0.0.0 ads1.itadnetwork.co.uk +0.0.0.0 ads1.jev.co.za +0.0.0.0 ads1.msads.net +0.0.0.0 ads1.msn.com +0.0.0.0 ads1.perfadbrite.com.akadns.net +0.0.0.0 ads1.performancingads.com +0.0.0.0 ads1.realcities.com +0.0.0.0 ads1.revenue.net +0.0.0.0 ads1.sptimes.com +0.0.0.0 ads1.theglobeandmail.com +0.0.0.0 ads1.ucomics.com +0.0.0.0 ads1.udc.advance.net +0.0.0.0 ads1.updated.com +0.0.0.0 ads1.virtumundo.com +0.0.0.0 ads1.zdnet.com +0.0.0.0 ads20.focalink.com +0.0.0.0 ads21.focalink.com +0.0.0.0 ads22.focalink.com +0.0.0.0 ads23.focalink.com +0.0.0.0 ads24.focalink.com +0.0.0.0 ads25.focalink.com +0.0.0.0 ads2.adbrite.com +0.0.0.0 ads2.ad-flow.com +0.0.0.0 ads2.advance.net +0.0.0.0 ads2.advertwizard.com +0.0.0.0 ads2.canoe.ca +0.0.0.0 ads2.clearchannel.com +0.0.0.0 ads2.clickad.com +0.0.0.0 ads2.collegclub.com +0.0.0.0 ads2.collegeclub.com +0.0.0.0 ads2.contentabc.com +0.0.0.0 ads2.drivelinemedia.com +0.0.0.0 ads2.emeraldcoast.com +0.0.0.0 ads2.exhedra.com +0.0.0.0 ads2.firingsquad.com +0.0.0.0 ads2.gamecity.net +0.0.0.0 ads2.jubii.dk +0.0.0.0 ads2.ljworld.com +0.0.0.0 ads2.msn.com +0.0.0.0 ads2.newtimes.com +0.0.0.0 ads2.osdn.com +0.0.0.0 ads2.pittsburghlive.com +0.0.0.0 ads2.realcities.com +0.0.0.0 ads2.revenue.net +0.0.0.0 ads2.rp.pl +0.0.0.0 ads2srv.com +0.0.0.0 ads2.theglobeandmail.com +0.0.0.0 ads2.udc.advance.net +0.0.0.0 ads2.virtumundo.com +0.0.0.0 ads2.weblogssl.com +0.0.0.0 ads2.zdnet.com +0.0.0.0 ads2.zeusclicks.com +0.0.0.0 ads360.com +0.0.0.0 ads36.hyperbanner.net +0.0.0.0 ads3.ad-flow.com +0.0.0.0 ads3.adman.gr +0.0.0.0 ads3.advance.net +0.0.0.0 ads3.advertwizard.com +0.0.0.0 ads3.canoe.ca +0.0.0.0 ads3.freebannertrade.com +0.0.0.0 ads3.gamecity.net +0.0.0.0 ads3.jubii.dk +0.0.0.0 ads3.realcities.com +0.0.0.0 ads3.udc.advance.net +0.0.0.0 ads3.virtumundo.com +0.0.0.0 ads3.zdnet.com +0.0.0.0 ads4.ad-flow.com +0.0.0.0 ads4.advance.net +0.0.0.0 ads4.advertwizard.com +0.0.0.0 ads4.canoe.ca +0.0.0.0 ads4.clearchannel.com +0.0.0.0 ads4.gamecity.net +0.0.0.0 ads4homes.com +0.0.0.0 ads4.realcities.com +0.0.0.0 ads4.udc.advance.net +0.0.0.0 ads4.virtumundo.com +0.0.0.0 ads5.ad-flow.com +0.0.0.0 ads5.advance.net +0.0.0.0 ads5.advertwizard.com +0.0.0.0 ads5.canoe.ca +0.0.0.0 ads.5ci.lt +0.0.0.0 ads5.fxdepo.com +0.0.0.0 ads5.mconetwork.com +0.0.0.0 ads5.udc.advance.net +0.0.0.0 ads5.virtumundo.com +0.0.0.0 ads6.ad-flow.com +0.0.0.0 ads6.advance.net +0.0.0.0 ads6.advertwizard.com +0.0.0.0 ads6.gamecity.net +0.0.0.0 ads6.udc.advance.net +0.0.0.0 ads7.ad-flow.com +0.0.0.0 ads7.advance.net +0.0.0.0 ads7.advertwizard.com +0.0.0.0 ads.7days.ae +0.0.0.0 ads7.gamecity.net +0.0.0.0 ads7.speedbit.com +0.0.0.0 ads7.udc.advance.net +0.0.0.0 ads.8833.com +0.0.0.0 ads8.ad-flow.com +0.0.0.0 ads8.advertwizard.com +0.0.0.0 ads8.com +0.0.0.0 ads8.udc.advance.net +0.0.0.0 ads9.ad-flow.com +0.0.0.0 ads9.advertwizard.com +0.0.0.0 ads9.udc.advance.net +0.0.0.0 ads.abs-cbn.com +0.0.0.0 ads.accelerator-media.com +0.0.0.0 ads.aceweb.net +0.0.0.0 ads.activeagent.at +0.0.0.0 ads.active.com +0.0.0.0 ads.ad4game.com +0.0.0.0 ads.adap.tv +0.0.0.0 ads.adbrite.com +0.0.0.0 ads.adbroker.de +0.0.0.0 ads.adcorps.com +0.0.0.0 ads.addesktop.com +0.0.0.0 ads.addynamix.com +0.0.0.0 ads.adengage.com +0.0.0.0 ads.ad-flow.com +0.0.0.0 ads.adfox.ru +0.0.0.0 ads.adgoto.com +0.0.0.0 ads.adhall.com +0.0.0.0 ads.adhearus.com +0.0.0.0 ads.adhostingsolutions.com +0.0.0.0 ads.admarvel.com +0.0.0.0 ads.admaximize.com +0.0.0.0 adsadmin.aspentimes.com +0.0.0.0 adsadmin.corusradionetwork.com +0.0.0.0 adsadmin.vaildaily.com +0.0.0.0 ads.admonitor.net +0.0.0.0 ads.adn.com +0.0.0.0 ads.adroar.com +0.0.0.0 ads.adsag.com +0.0.0.0 ads.adsbookie.com +0.0.0.0 ads.adshareware.net +0.0.0.0 ads.adsinimages.com +0.0.0.0 ads.adsonar.com +0.0.0.0 ads.adsrvmedia.com +0.0.0.0 ads.adtegrity.net +0.0.0.0 ads.adtiger.de +0.0.0.0 ads.adultfriendfinder.com +0.0.0.0 ads.adultswim.com +0.0.0.0 ads.advance.net +0.0.0.0 ads.adverline.com +0.0.0.0 ads.adviva.net +0.0.0.0 ads.advolume.com +0.0.0.0 ads.adworldnetwork.com +0.0.0.0 ads.adx.nu +0.0.0.0 ads.adxpansion.com +0.0.0.0 ads.adxpose.com +0.0.0.0 ads.adxpose.mpire.akadns.net +0.0.0.0 ads.affiliates.match.com +0.0.0.0 ads.aftonbladet.se +0.0.0.0 ads.ah-ha.com +0.0.0.0 ads.aintitcool.com +0.0.0.0 ads.airamericaradio.com +0.0.0.0 ads.ak.facebook.com +0.0.0.0 ads.albawaba.com +0.0.0.0 ads.al.com +0.0.0.0 ads.allsites.com +0.0.0.0 ads.allvertical.com +0.0.0.0 ads.amarillo.com +0.0.0.0 ads.amateurmatch.com +0.0.0.0 ads.amazingmedia.com +0.0.0.0 ads.amgdgt.com +0.0.0.0 ads.ami-admin.com +0.0.0.0 ads.anm.co.uk +0.0.0.0 ads.anvato.com +0.0.0.0 ads.aol.com +0.0.0.0 ads.apartmenttherapy.com +0.0.0.0 ads.apn.co.nz +0.0.0.0 ads.apn.co.za +0.0.0.0 ads.appleinsider.com +0.0.0.0 ads.arcadechain.com +0.0.0.0 ads.aroundtherings.com +0.0.0.0 ads.as4x.tmcs.net +0.0.0.0 ads.as4x.tmcs.ticketmaster.ca +0.0.0.0 ads.as4x.tmcs.ticketmaster.com +0.0.0.0 ads.asia1.com +0.0.0.0 ads.asia1.com.sg +0.0.0.0 ads.aspalliance.com +0.0.0.0 ads.aspentimes.com +0.0.0.0 ads.asp.net +0.0.0.0 ads.associatedcontent.com +0.0.0.0 ads.astalavista.us +0.0.0.0 ads.atlantamotorspeedway.com +0.0.0.0 adsatt.abcnews.starwave.com +0.0.0.0 adsatt.espn.go.com +0.0.0.0 adsatt.espn.starwave.com +0.0.0.0 ads.auctionads.com +0.0.0.0 ads.auctioncity.co.nz +0.0.0.0 ads.auctions.yahoo.com +0.0.0.0 ads.augusta.com +0.0.0.0 ads.aversion2.com +0.0.0.0 ads.aws.sitepoint.com +0.0.0.0 ads.azjmp.com +0.0.0.0 ads.baazee.com +0.0.0.0 ads.bangkokpost.co.th +0.0.0.0 ads.banner.t-online.de +0.0.0.0 ads.barnonedrinks.com +0.0.0.0 ads.battle.net +0.0.0.0 ads.bauerpublishing.com +0.0.0.0 ads.baventures.com +0.0.0.0 ads.bbcworld.com +0.0.0.0 ads.bcnewsgroup.com +0.0.0.0 ads.beeb.com +0.0.0.0 ads.beliefnet.com +0.0.0.0 ads.belointeractive.com +0.0.0.0 ads.beta.itravel2000.com +0.0.0.0 ads.betanews.com +0.0.0.0 ads.bfast.com +0.0.0.0 ads.bfm.valueclick.net +0.0.0.0 ads.bianca.com +0.0.0.0 ads.bidclix.com +0.0.0.0 ads.bidvertiser.com +0.0.0.0 ads.bigcitytools.com +0.0.0.0 ads.biggerboat.com +0.0.0.0 ads.bitsonthewire.com +0.0.0.0 ads.bizhut.com +0.0.0.0 ads.blixem.nl +0.0.0.0 ads.blog.com +0.0.0.0 ads.blogherads.com +0.0.0.0 ads.bloomberg.com +0.0.0.0 ads.blp.calueclick.net +0.0.0.0 ads.blp.valueclick.net +0.0.0.0 ads.bluelithium.com +0.0.0.0 ads.bluemountain.com +0.0.0.0 ads.bonnint.net +0.0.0.0 ads.box.sk +0.0.0.0 ads.brabys.com +0.0.0.0 ads.brand.net +0.0.0.0 ads.bridgetrack.com +0.0.0.0 ads.britishexpats.com +0.0.0.0 ads.buscape.com.br +0.0.0.0 ads.businessclick.com +0.0.0.0 ads.businessweek.com +0.0.0.0 ads.calgarysun.com +0.0.0.0 ads.callofdutyblackopsforum.net +0.0.0.0 ads.camrecord.com +0.0.0.0 ads.canoe.ca +0.0.0.0 ads.cardea.se +0.0.0.0 ads.cardplayer.com +0.0.0.0 ads.carltononline.com +0.0.0.0 ads.carocean.co.uk +0.0.0.0 ads.casinocity.com +0.0.0.0 ads.catholic.org +0.0.0.0 ads.cavello.com +0.0.0.0 ads.cbc.ca +0.0.0.0 ads.cdfreaks.com +0.0.0.0 ads.cdnow.com +0.0.0.0 adscendmedia.com +0.0.0.0 ads.centraliprom.com +0.0.0.0 ads.cgchannel.com +0.0.0.0 ads.chalomumbai.com +0.0.0.0 ads.champs-elysees.com +0.0.0.0 ads.channel4.com +0.0.0.0 ads.checkm8.co.za +0.0.0.0 ads.chipcenter.com +0.0.0.0 adscholar.com +0.0.0.0 ads.chumcity.com +0.0.0.0 ads.cjonline.com +0.0.0.0 ads.clamav.net +0.0.0.0 ads.clara.net +0.0.0.0 ads.clearchannel.com +0.0.0.0 ads.cleveland.com +0.0.0.0 ads.clickability.com +0.0.0.0 ads.clickad.com.pl +0.0.0.0 ads.clickagents.com +0.0.0.0 ads.clickhouse.com +0.0.0.0 ads.clicksor.com +0.0.0.0 ads.clickthru.net +0.0.0.0 ads.clicmanager.fr +0.0.0.0 ads.clubzone.com +0.0.0.0 ads.cluster01.oasis.zmh.zope.net +0.0.0.0 ads.cmediaworld.com +0.0.0.0 ads.cmg.valueclick.net +0.0.0.0 ads.cnn.com +0.0.0.0 ads.cnngo.com +0.0.0.0 ads.cobrad.com +0.0.0.0 ads.collegclub.com +0.0.0.0 ads.collegehumor.com +0.0.0.0 ads.collegemix.com +0.0.0.0 ads.com.com +0.0.0.0 ads.comediagroup.hu +0.0.0.0 ads.comicbookresources.com +0.0.0.0 ads.contactmusic.com +0.0.0.0 ads.contentabc.com +0.0.0.0 ads.coopson.com +0.0.0.0 ads.corusradionetwork.com +0.0.0.0 ads.courierpostonline.com +0.0.0.0 ads.cpsgsoftware.com +0.0.0.0 ads.crakmedia.com +0.0.0.0 ads.crapville.com +0.0.0.0 ads.creative-serving.com +0.0.0.0 ads.crosscut.com +0.0.0.0 ads.ctvdigital.net +0.0.0.0 ads.currantbun.com +0.0.0.0 ads.cyberfight.ru +0.0.0.0 ads.cybersales.cz +0.0.0.0 ads.cybertrader.com +0.0.0.0 ads.dada.it +0.0.0.0 ads.danworld.net +0.0.0.0 adsdaq.com +0.0.0.0 ads.dbforums.com +0.0.0.0 ads.ddj.com +0.0.0.0 ads.dealnews.com +0.0.0.0 ads.democratandchronicle.com +0.0.0.0 ads.dennisnet.co.uk +0.0.0.0 ads.designboom.com +0.0.0.0 ads.designtaxi.com +0.0.0.0 ads.desmoinesregister.com +0.0.0.0 ads-de.spray.net +0.0.0.0 ads.detelefoongids.nl +0.0.0.0 ads.developershed.com +0.0.0.0 ads.deviantart.com +0.0.0.0 ads-dev.youporn.com +0.0.0.0 ads.digitalacre.com +0.0.0.0 ads.digital-digest.com +0.0.0.0 ads.digitalhealthcare.com +0.0.0.0 ads.digitalmedianet.com +0.0.0.0 ads.digitalpoint.com +0.0.0.0 ads.dimcab.com +0.0.0.0 ads.directionsmag.com +0.0.0.0 ads-direct.prodigy.net +0.0.0.0 ads.discovery.com +0.0.0.0 ads.dk +0.0.0.0 ads.doclix.com +0.0.0.0 ads.domeus.com +0.0.0.0 ads.dontpanicmedia.com +0.0.0.0 ads.dothads.com +0.0.0.0 ads.doubleviking.com +0.0.0.0 ads.drf.com +0.0.0.0 ads.drivelinemedia.com +0.0.0.0 ads.drugs.com +0.0.0.0 ads.dumpalink.com +0.0.0.0 adsearch.adkontekst.pl +0.0.0.0 adsearch.pl +0.0.0.0 adsearch.wp.pl +0.0.0.0 ads.ecircles.com +0.0.0.0 ads.economist.com +0.0.0.0 ads.ecosalon.com +0.0.0.0 ads.edirectme.com +0.0.0.0 ads.einmedia.com +0.0.0.0 ads.eircom.net +0.0.0.0 ads.emeraldcoast.com +0.0.0.0 ads.enliven.com +0.0.0.0 ad.sensismediasmart.com.au +0.0.0.0 adsentnetwork.com +0.0.0.0 adserer.ihigh.com +0.0.0.0 ads.erotism.com +0.0.0.0 adserv001.adtech.de +0.0.0.0 adserv001.adtech.fr +0.0.0.0 adserv001.adtech.us +0.0.0.0 adserv002.adtech.de +0.0.0.0 adserv002.adtech.fr +0.0.0.0 adserv002.adtech.us +0.0.0.0 adserv003.adtech.de +0.0.0.0 adserv003.adtech.fr +0.0.0.0 adserv003.adtech.us +0.0.0.0 adserv004.adtech.de +0.0.0.0 adserv004.adtech.fr +0.0.0.0 adserv004.adtech.us +0.0.0.0 adserv005.adtech.de +0.0.0.0 adserv005.adtech.fr +0.0.0.0 adserv005.adtech.us +0.0.0.0 adserv006.adtech.de +0.0.0.0 adserv006.adtech.fr +0.0.0.0 adserv006.adtech.us +0.0.0.0 adserv007.adtech.de +0.0.0.0 adserv007.adtech.fr +0.0.0.0 adserv007.adtech.us +0.0.0.0 adserv008.adtech.de +0.0.0.0 adserv008.adtech.fr +0.0.0.0 adserv008.adtech.us +0.0.0.0 adserv2.bravenet.com +0.0.0.0 adserv.aip.org +0.0.0.0 adservant.guj.de +0.0.0.0 adserv.bravenet.com +0.0.0.0 adserve5.nikkeibp.co.jp +0.0.0.0 adserve.adtoll.com +0.0.0.0 adserve.canadawidemagazines.com +0.0.0.0 adserve.city-ad.com +0.0.0.0 adserve.ehpub.com +0.0.0.0 adserve.gossipgirls.com +0.0.0.0 adserve.mizzenmedia.com +0.0.0.0 adserv.entriq.net +0.0.0.0 adserve.podaddies.com +0.0.0.0 adserve.profit-smart.com +0.0.0.0 adserver01.ancestry.com +0.0.0.0 adserver.100free.com +0.0.0.0 adserver.163.com +0.0.0.0 adserver1.adserver.com.pl +0.0.0.0 adserver1.adtech.com.tr +0.0.0.0 adserver1.backbeatmedia.com +0.0.0.0 adserver1.economist.com +0.0.0.0 adserver1.eudora.com +0.0.0.0 adserver1.harvestadsdepot.com +0.0.0.0 adserver1.hookyouup.com +0.0.0.0 adserver1-images.backbeatmedia.com +0.0.0.0 adserver1.isohunt.com +0.0.0.0 adserver1.lokitorrent.com +0.0.0.0 adserver1.mediainsight.de +0.0.0.0 adserver1.ogilvy-interactive.de +0.0.0.0 adserver1.realtracker.com +0.0.0.0 adserver1.sonymusiceurope.com +0.0.0.0 adserver1.teracent.net +0.0.0.0 adserver1.wmads.com +0.0.0.0 adserver.2618.com +0.0.0.0 adserver2.adserver.com.pl +0.0.0.0 adserver2.atman.pl +0.0.0.0 adserver2.christianitytoday.com +0.0.0.0 adserver2.condenast.co.uk +0.0.0.0 adserver2.creative.com +0.0.0.0 adserver2.eudora.com +0.0.0.0 adserver-2.ig.com.br +0.0.0.0 adserver2.mediainsight.de +0.0.0.0 adserver2.news-journalonline.com +0.0.0.0 adserver2.popdata.de +0.0.0.0 adserver2.realtracker.com +0.0.0.0 adserver2.teracent.net +0.0.0.0 adserver.3digit.de +0.0.0.0 adserver3.eudora.com +0.0.0.0 adserver-3.ig.com.br +0.0.0.0 adserver4.eudora.com +0.0.0.0 adserver-4.ig.com.br +0.0.0.0 adserver-5.ig.com.br +0.0.0.0 adserver.71i.de +0.0.0.0 adserver9.contextad.com +0.0.0.0 adserver.ad-it.dk +0.0.0.0 adserver.adreactor.com +0.0.0.0 adserver.adremedy.com +0.0.0.0 adserver.ads360.com +0.0.0.0 adserver.adserver.com.pl +0.0.0.0 adserver.adsincontext.com +0.0.0.0 adserver.adtech.de +0.0.0.0 adserver.adtech.fr +0.0.0.0 adserver.adtech.us +0.0.0.0 adserver.adtechus.com +0.0.0.0 adserver.adultfriendfinder.com +0.0.0.0 adserver.advertist.com +0.0.0.0 adserver.affiliatemg.com +0.0.0.0 adserver.affiliation.com +0.0.0.0 adserver.aim4media.com +0.0.0.0 adserver.a.in.monster.com +0.0.0.0 adserver.airmiles.ca +0.0.0.0 adserver.akqa.net +0.0.0.0 adserver.allheadlinenews.com +0.0.0.0 adserver.amnews.com +0.0.0.0 adserver.ancestry.com +0.0.0.0 adserver.anemo.com +0.0.0.0 adserver.anm.co.uk +0.0.0.0 adserver.aol.fr +0.0.0.0 adserver.archant.co.uk +0.0.0.0 adserver.artempireindustries.com +0.0.0.0 adserver.arttoday.com +0.0.0.0 adserver.atari.net +0.0.0.0 adserverb.conjelco.com +0.0.0.0 adserver.betandwin.de +0.0.0.0 adserver.billiger-surfen.de +0.0.0.0 adserver.billiger-telefonieren.de +0.0.0.0 adserver.bizland-inc.net +0.0.0.0 adserver.bluereactor.com +0.0.0.0 adserver.bluereactor.net +0.0.0.0 adserver.bluewin.ch +0.0.0.0 adserver.buttonware.com +0.0.0.0 adserver.buttonware.net +0.0.0.0 adserver.cams.com +0.0.0.0 adserver.cantv.net +0.0.0.0 adserver.cebu-online.com +0.0.0.0 adserver.cheatplanet.com +0.0.0.0 adserver.chickclick.com +0.0.0.0 adserver.click4cash.de +0.0.0.0 adserver.clubic.com +0.0.0.0 adserver.clundressed.com +0.0.0.0 adserver.co.il +0.0.0.0 adserver.colleges.com +0.0.0.0 adserver.com +0.0.0.0 adserver.comparatel.fr +0.0.0.0 adserver.com-solutions.com +0.0.0.0 adserver.conjelco.com +0.0.0.0 adserver.corusradionetwork.com +0.0.0.0 adserver.creative-asia.com +0.0.0.0 adserver.creativeinspire.com +0.0.0.0 adserver.dayrates.com +0.0.0.0 adserver.dbusiness.com +0.0.0.0 adserver.developersnetwork.com +0.0.0.0 adserver.devx.com +0.0.0.0 adserver.digitalpartners.com +0.0.0.0 adserver.digitoday.com +0.0.0.0 adserver.directforce.com +0.0.0.0 adserver.directforce.net +0.0.0.0 adserver.dnps.com +0.0.0.0 adserver.dotcommedia.de +0.0.0.0 adserver.dotmusic.com +0.0.0.0 adserver.eham.net +0.0.0.0 adserver.emapadserver.com +0.0.0.0 adserver.emporis.com +0.0.0.0 adserver.emulation64.com +0.0.0.0 adserver-espnet.sportszone.net +0.0.0.0 adserver.eudora.com +0.0.0.0 adserver.eva2000.com +0.0.0.0 adserver.expatica.nxs.nl +0.0.0.0 adserver.ezzhosting.com +0.0.0.0 adserver.filefront.com +0.0.0.0 adserver.fmpub.net +0.0.0.0 adserver.fr.adtech.de +0.0.0.0 adserver.freecity.de +0.0.0.0 adserver.freenet.de +0.0.0.0 adserver.friendfinder.com +0.0.0.0 adserver.gameparty.net +0.0.0.0 adserver.gamesquad.net +0.0.0.0 adserver.garden.com +0.0.0.0 adserver.gorillanation.com +0.0.0.0 adserver.gr +0.0.0.0 adserver.gunaxin.com +0.0.0.0 adserver.hardsextube.com +0.0.0.0 adserver.hardwareanalysis.com +0.0.0.0 adserver.harktheherald.com +0.0.0.0 adserver.harvestadsdepot.com +0.0.0.0 adserver.hellasnet.gr +0.0.0.0 adserver.hg-computer.de +0.0.0.0 adserver.hi-m.de +0.0.0.0 adserver.hispavista.com +0.0.0.0 adserver.hk.outblaze.com +0.0.0.0 adserver.home.pl +0.0.0.0 adserver.hostinteractive.com +0.0.0.0 adserver.humanux.com +0.0.0.0 adserver.hwupgrade.it +0.0.0.0 adserver.ifmagazine.com +0.0.0.0 adserver.ig.com.br +0.0.0.0 adserver.ign.com +0.0.0.0 adserver.ilounge.com +0.0.0.0 adserver.infinit.net +0.0.0.0 adserver.infotiger.com +0.0.0.0 adserver.interfree.it +0.0.0.0 adserver.inwind.it +0.0.0.0 adserver.ision.de +0.0.0.0 adserver.isonews.com +0.0.0.0 adserver.ixm.co.uk +0.0.0.0 adserver.jacotei.com.br +0.0.0.0 adserver.janes.com +0.0.0.0 adserver.janes.net +0.0.0.0 adserver.janes.org +0.0.0.0 adserver.jolt.co.uk +0.0.0.0 adserver.journalinteractive.com +0.0.0.0 adserver.juicyads.com +0.0.0.0 adserver.kcilink.com +0.0.0.0 adserver.killeraces.com +0.0.0.0 adserver.kylemedia.com +0.0.0.0 adserver.lanacion.com.ar +0.0.0.0 adserver.lanepress.com +0.0.0.0 adserver.latimes.com +0.0.0.0 adserver.legacy-network.com +0.0.0.0 adserver.libero.it +0.0.0.0 adserver.linktrader.co.uk +0.0.0.0 adserver.livejournal.com +0.0.0.0 adserver.lostreality.com +0.0.0.0 adserver.lunarpages.com +0.0.0.0 adserver.lycos.co.jp +0.0.0.0 adserver.m2kcore.com +0.0.0.0 adserver.magazyn.pl +0.0.0.0 adserver.matchcraft.com +0.0.0.0 adserver.merc.com +0.0.0.0 adserver.mindshare.de +0.0.0.0 adserver.mobsmith.com +0.0.0.0 adserver.monster.com +0.0.0.0 adserver.monstersandcritics.com +0.0.0.0 adserver.motonews.pl +0.0.0.0 adserver.myownemail.com +0.0.0.0 adserver.netcreators.nl +0.0.0.0 adserver.netshelter.net +0.0.0.0 adserver.newdigitalgroup.com +0.0.0.0 adserver.newmassmedia.net +0.0.0.0 adserver.news.com +0.0.0.0 adserver.news.com.au +0.0.0.0 adserver.news-journalonline.com +0.0.0.0 adserver.newtimes.com +0.0.0.0 adserver.ngz-network.de +0.0.0.0 adserver.nydailynews.com +0.0.0.0 adserver.nzoom.com +0.0.0.0 adserver.o2.pl +0.0.0.0 adserver.onwisconsin.com +0.0.0.0 adserver.passion.com +0.0.0.0 adserver.phatmax.net +0.0.0.0 adserver.phillyburbs.com +0.0.0.0 adserver.pl +0.0.0.0 adserver.planet-multiplayer.de +0.0.0.0 adserver.plhb.com +0.0.0.0 adserver.pollstar.com +0.0.0.0 adserver.portalofevil.com +0.0.0.0 adserver.portal.pl +0.0.0.0 adserver.portugalmail.pt +0.0.0.0 adserver.prodigy.net +0.0.0.0 adserver.proteinos.com +0.0.0.0 adserver.radio-canada.ca +0.0.0.0 adserver.ratestar.net +0.0.0.0 adserver.revver.com +0.0.0.0 adserver.ro +0.0.0.0 adserver.sabc.co.za +0.0.0.0 adserver.sabcnews.co.za +0.0.0.0 adserver.sanomawsoy.fi +0.0.0.0 adserver.scmp.com +0.0.0.0 adserver.securityfocus.com +0.0.0.0 adserver.sextracker.com +0.0.0.0 adserver.sharewareonline.com +0.0.0.0 adserver.singnet.com +0.0.0.0 adserver.sl.kharkov.ua +0.0.0.0 adserver.smashtv.com +0.0.0.0 adserver.snowball.com +0.0.0.0 adserver.softonic.com +0.0.0.0 adserver.soloserver.com +0.0.0.0 adserversolutions.com +0.0.0.0 adserver.swiatobrazu.pl +0.0.0.0 adserver.synergetic.de +0.0.0.0 adserver.telalink.net +0.0.0.0 adserver.te.pt +0.0.0.0 adserver.teracent.net +0.0.0.0 adserver.terra.com.br +0.0.0.0 adserver.terra.es +0.0.0.0 adserver.theknot.com +0.0.0.0 adserver.theonering.net +0.0.0.0 adserver.thirty4.com +0.0.0.0 adserver.thisislondon.co.uk +0.0.0.0 adserver.tilted.net +0.0.0.0 adserver.tqs.ca +0.0.0.0 adserver.track-star.com +0.0.0.0 adserver.trader.ca +0.0.0.0 adserver.trafficsyndicate.com +0.0.0.0 adserver.trb.com +0.0.0.0 adserver.tribuneinteractive.com +0.0.0.0 adserver.tsgadv.com +0.0.0.0 adserver.tulsaworld.com +0.0.0.0 adserver.tweakers.net +0.0.0.0 adserver.twitpic.com +0.0.0.0 adserver.ugo.com +0.0.0.0 adserver.ugo.nl +0.0.0.0 adserver.ukplus.co.uk +0.0.0.0 adserver.uproxx.com +0.0.0.0 adserver.usermagnet.com +0.0.0.0 adserver.van.net +0.0.0.0 adserver.virginmedia.com +0.0.0.0 adserver.virgin.net +0.0.0.0 adserver.virtualminds.nl +0.0.0.0 adserver.virtuous.co.uk +0.0.0.0 adserver.voir.ca +0.0.0.0 adserver.webads.co.uk +0.0.0.0 adserver.webads.nl +0.0.0.0 adserver.wemnet.nl +0.0.0.0 adserver.x3.hu +0.0.0.0 adserver.ya.com +0.0.0.0 adserver.yahoo.com +0.0.0.0 adserver.zaz.com.br +0.0.0.0 adserver.zeads.com +0.0.0.0 adserve.shopzilla.com +0.0.0.0 adserve.splicetoday.com +0.0.0.0 adserve.viaarena.com +0.0.0.0 adserv.free6.com +0.0.0.0 adserv.geocomm.com +0.0.0.0 adserv.iafrica.com +0.0.0.0 adservices.google.com +0.0.0.0 adservices.picadmedia.com +0.0.0.0 adservingcentral.com +0.0.0.0 adserving.cpxinteractive.com +0.0.0.0 adserv.internetfuel.com +0.0.0.0 adserv.jupiter.com +0.0.0.0 adserv.lwmn.net +0.0.0.0 adserv.maineguide.com +0.0.0.0 adserv.muchosucko.com +0.0.0.0 adserv.mywebtimes.com +0.0.0.0 adserv.pitchforkmedia.com +0.0.0.0 adserv.postbulletin.com +0.0.0.0 adserv.qconline.com +0.0.0.0 adserv.quality-channel.de +0.0.0.0 adserv.usps.com +0.0.0.0 adserwer.o2.pl +0.0.0.0 ads.espn.adsonar.com +0.0.0.0 ads.eudora.com +0.0.0.0 ads.eu.msn.com +0.0.0.0 ads.euniverseads.com +0.0.0.0 adseu.novem.pl +0.0.0.0 ads.examiner.net +0.0.0.0 ads.exhedra.com +0.0.0.0 ads.expedia.com +0.0.0.0 ads.expekt.com +0.0.0.0 ads.ezboard.com +0.0.0.0 adsfac.eu +0.0.0.0 adsfac.net +0.0.0.0 adsfac.us +0.0.0.0 ads.fairfax.com.au +0.0.0.0 ads.fark.com +0.0.0.0 ads.fayettevillenc.com +0.0.0.0 ads.filecloud.com +0.0.0.0 ads.fileindexer.com +0.0.0.0 ads.filmup.com +0.0.0.0 ads.first-response.be +0.0.0.0 ads.flabber.nl +0.0.0.0 ads.flashgames247.com +0.0.0.0 ads.fling.com +0.0.0.0 ads.floridatoday.com +0.0.0.0 ads.fool.com +0.0.0.0 ads.forbes.com +0.0.0.0 ads.forbes.net +0.0.0.0 ads.fortunecity.com +0.0.0.0 ads.fredericksburg.com +0.0.0.0 ads.freebannertrade.com +0.0.0.0 ads.freshmeat.net +0.0.0.0 ads.fresnobee.com +0.0.0.0 ads.friendfinder.com +0.0.0.0 ads.ft.com +0.0.0.0 ads.gamblinghit.com +0.0.0.0 ads.gamecity.net +0.0.0.0 ads.gamecopyworld.no +0.0.0.0 ads.gameinformer.com +0.0.0.0 ads.game.net +0.0.0.0 ads.gamershell.com +0.0.0.0 ads.gamespy.com +0.0.0.0 ads.gamespyid.com +0.0.0.0 ads.gateway.com +0.0.0.0 ads.gawker.com +0.0.0.0 ads.gettools.com +0.0.0.0 ads.gigaom.com.php5-12.websitetestlink.com +0.0.0.0 ads.globeandmail.com +0.0.0.0 ads.gmg.valueclick.net +0.0.0.0 ads.gmodules.com +0.0.0.0 ads.god.co.uk +0.0.0.0 ads.gorillanation.com +0.0.0.0 ads.gplusmedia.com +0.0.0.0 ads.granadamedia.com +0.0.0.0 ads.greenbaypressgazette.com +0.0.0.0 ads.greenvilleonline.com +0.0.0.0 ads.guardian.co.uk +0.0.0.0 ads.guardianunlimited.co.uk +0.0.0.0 ads.gunaxin.com +0.0.0.0 ads.halogennetwork.com +0.0.0.0 ads.hamptonroads.com +0.0.0.0 ads.hamtonroads.com +0.0.0.0 ads.hardwarezone.com +0.0.0.0 ads.harpers.org +0.0.0.0 ads.hbv.de +0.0.0.0 ads.hearstmags.com +0.0.0.0 ads.heartlight.org +0.0.0.0 ads.herald-mail.com +0.0.0.0 ads.heraldnet.com +0.0.0.0 ads.heraldonline.com +0.0.0.0 ads.heraldsun.com +0.0.0.0 ads.heroldonline.com +0.0.0.0 ads.he.valueclick.net +0.0.0.0 ads.hitcents.com +0.0.0.0 ads.hlwd.valueclick.net +0.0.0.0 ads.hollandsentinel.com +0.0.0.0 ads.hollywood.com +0.0.0.0 ads.hooqy.com +0.0.0.0 ads.hothardware.com +0.0.0.0 ad.showbizz.net +0.0.0.0 ads.hulu.com.edgesuite.net +#0.0.0.0 ads.hulu.com # Uncomment to block Hulu. +0.0.0.0 ads.humorbua.no +0.0.0.0 ads.i12.de +0.0.0.0 ads.i33.com +0.0.0.0 ads.iafrica.com +0.0.0.0 ads.i-am-bored.com +0.0.0.0 ads.iboost.com +0.0.0.0 ads.icq.com +0.0.0.0 ads.iforex.com +0.0.0.0 ads.ign.com +0.0.0.0 ads.illuminatednation.com +0.0.0.0 ads.imdb.com +0.0.0.0 ads.imgur.com +0.0.0.0 ads.imposibil.ro +0.0.0.0 ads.indiatimes.com +0.0.0.0 ads.indya.com +0.0.0.0 ads.indystar.com +0.0.0.0 ads.inedomedia.com +0.0.0.0 ads.inetdirectories.com +0.0.0.0 ads.inetinteractive.com +0.0.0.0 ads.infi.net +0.0.0.0 ads.infospace.com +0.0.0.0 adsinimages.com +0.0.0.0 ads.injersey.com +0.0.0.0 ads.insidehighered.com +0.0.0.0 ads.intellicast.com +0.0.0.0 ads.internic.co.il +0.0.0.0 ads.inthesidebar.com +0.0.0.0 adsintl.starwave.com +0.0.0.0 ads.iol.co.il +0.0.0.0 ads.ipowerweb.com +0.0.0.0 ads.ireport.com +0.0.0.0 ads.isat-tech.com +0.0.0.0 ads.isoftmarketing.com +0.0.0.0 ads.isum.de +0.0.0.0 ads.itv.com +0.0.0.0 ads.iwon.com +0.0.0.0 ads.jacksonville.com +0.0.0.0 ads.jeneauempire.com +0.0.0.0 ads.jetpackdigital.com +0.0.0.0 ads.jetphotos.net +0.0.0.0 ads.jewcy.com +0.0.0.0 ads.jimworld.com +0.0.0.0 ads.joetec.net +0.0.0.0 ads.jokaroo.com +0.0.0.0 ads.jornadavirtual.com.mx +0.0.0.0 ads.jossip.com +0.0.0.0 ads.jpost.com +0.0.0.0 ads.jubii.dk +0.0.0.0 ads.juicyads.com +0.0.0.0 ads.juneauempire.com +0.0.0.0 ads.jwtt3.com +0.0.0.0 ads.kazaa.com +0.0.0.0 ads.keywordblocks.com +0.0.0.0 ads.kixer.com +0.0.0.0 ads.kleinman.com +0.0.0.0 ads.kmpads.com +0.0.0.0 ads.koreanfriendfinder.com +0.0.0.0 ads.ksl.com +0.0.0.0 ad.slashgear.com +0.0.0.0 ads.leo.org +0.0.0.0 ads.lfstmedia.com +0.0.0.0 ads.lilengine.com +0.0.0.0 ads.link4ads.com +0.0.0.0 ads.linksponsor.com +0.0.0.0 ads.linktracking.net +0.0.0.0 ads.linuxjournal.com +0.0.0.0 ads.linuxsecurity.com +0.0.0.0 ads.list-universe.com +0.0.0.0 ads.live365.com +0.0.0.0 ads.ljworld.com +0.0.0.0 ads.lnkworld.com +0.0.0.0 ads.localnow.com +0.0.0.0 ads-local.sixapart.com +0.0.0.0 ads.lubbockonline.com +0.0.0.0 ads.lucidmedia.com +0.0.0.0 ads.lucidmedia.com.gslb.com +0.0.0.0 ads.lycos.com +0.0.0.0 ads.lycos-europe.com +0.0.0.0 ads.lzjl.com +0.0.0.0 ads.macnews.de +0.0.0.0 ads.macupdate.com +0.0.0.0 ads.madisonavenue.com +0.0.0.0 ads.madison.com +0.0.0.0 ads.magnetic.is +0.0.0.0 ads.mail.com +0.0.0.0 ads.mambocommunities.com +0.0.0.0 ad.sma.punto.net +0.0.0.0 ads.mariuana.it +0.0.0.0 adsmart.com +0.0.0.0 adsmart.co.uk +0.0.0.0 adsmart.net +0.0.0.0 ads.mcafee.com +0.0.0.0 ads.mdchoice.com +0.0.0.0 ads.mediamayhemcorp.com +0.0.0.0 ads.mediaodyssey.com +0.0.0.0 ads.mediaturf.net +0.0.0.0 ads.mefeedia.com +0.0.0.0 ads.megaproxy.com +0.0.0.0 ads.metblogs.com +0.0.0.0 ads.mgnetwork.com +0.0.0.0 ads.mindsetnetwork.com +0.0.0.0 ads.miniclip.com +0.0.0.0 ads.mininova.org +0.0.0.0 ads.mircx.com +0.0.0.0 ads.mixtraffic.com +0.0.0.0 ads.mlive.com +0.0.0.0 ads.mm.ap.org +0.0.0.0 ads.mndaily.com +0.0.0.0 ad.smni.com +0.0.0.0 ads.mobiledia.com +0.0.0.0 ads.mobygames.com +0.0.0.0 ads.modbee.com +0.0.0.0 ads.mofos.com +0.0.0.0 ads.money.pl +0.0.0.0 ads.monster.com +0.0.0.0 ads.mouseplanet.com +0.0.0.0 ads.movieweb.com +0.0.0.0 ads.mp3searchy.com +0.0.0.0 adsm.soush.com +0.0.0.0 ads.mt.valueclick.net +0.0.0.0 ads.mtv.uol.com.br +0.0.0.0 ads.multimania.lycos.fr +0.0.0.0 ads.musiccity.com +0.0.0.0 ads.mustangworks.com +0.0.0.0 ads.mysimon.com +0.0.0.0 ads.mytelus.com +0.0.0.0 ads.nandomedia.com +0.0.0.0 ads.nationalreview.com +0.0.0.0 ads.nativeinstruments.de +0.0.0.0 ads.neoseeker.com +0.0.0.0 ads.neowin.net +0.0.0.0 ads.nerve.com +0.0.0.0 ads.netmechanic.com +0.0.0.0 ads.networkwcs.net +0.0.0.0 ads.networldmedia.net +0.0.0.0 ads.neudesicmediagroup.com +0.0.0.0 ads.newcity.com +0.0.0.0 ads.newcitynet.com +0.0.0.0 ads.newdream.net +0.0.0.0 ads.newgrounds.com +0.0.0.0 ads.newsint.co.uk +0.0.0.0 ads.newsminerextra.com +0.0.0.0 ads.newsobserver.com +0.0.0.0 ads.newsquest.co.uk +0.0.0.0 ads.newtention.net +0.0.0.0 ads.newtimes.com +0.0.0.0 adsnew.userfriendly.org +0.0.0.0 ads.ngenuity.com +0.0.0.0 ads.ninemsn.com.au +0.0.0.0 adsniper.ru +0.0.0.0 ads.nola.com +0.0.0.0 ads.northjersey.com +0.0.0.0 ads.novem.pl +0.0.0.0 ads.nowrunning.com +0.0.0.0 ads.npr.valueclick.net +0.0.0.0 ads.ntadvice.com +0.0.0.0 ads.nudecards.com +0.0.0.0 ads.nwsource.com +0.0.0.0 ads.nwsource.com.edgesuite.net +0.0.0.0 ads.nyi.net +0.0.0.0 ads.nyjournalnews.com +0.0.0.0 ads.nypost.com +0.0.0.0 ads.nytimes.com +0.0.0.0 ads.o2.pl +0.0.0.0 adsoftware.com +0.0.0.0 adsoldier.com +0.0.0.0 ads.ole.com +0.0.0.0 ads.omaha.com +0.0.0.0 adsonar.com +0.0.0.0 adson.awempire.com +0.0.0.0 ads.onlineathens.com +0.0.0.0 ads.online.ie +0.0.0.0 ads.onvertise.com +0.0.0.0 ads.ookla.com +0.0.0.0 ads.open.pl +0.0.0.0 ads.opensubtitles.org +0.0.0.0 ads.oregonlive.com +0.0.0.0 ads.orsm.net +0.0.0.0 ads.osdn.com +0.0.0.0 ad-souk.com +0.0.0.0 adspaces.ero-advertising.com +0.0.0.0 ads.parrysound.com +0.0.0.0 ads.partner2profit.com +0.0.0.0 ads.pastemagazine.com +0.0.0.0 ads.paxnet.co.kr +0.0.0.0 ads.pcper.com +0.0.0.0 ads.pdxguide.com +0.0.0.0 ads.peel.com +0.0.0.0 ads.peninsulaclarion.com +0.0.0.0 ads.penny-arcade.com +0.0.0.0 ads.pennyweb.com +0.0.0.0 ads.people.com.cn +0.0.0.0 ads.pg.valueclick.net +0.0.0.0 ads.pheedo.com +0.0.0.0 ads.phillyburbs.com +0.0.0.0 ads.phpclasses.org +0.0.0.0 ads.pilotonline.com +0.0.0.0 adspirit.net +0.0.0.0 adspiro.pl +0.0.0.0 ads.pitchforkmedia.com +0.0.0.0 ads.pittsburghlive.com +0.0.0.0 ads.pixiq.com +0.0.0.0 ads.place1.com +0.0.0.0 ads.planet-f1.com +0.0.0.0 ads.plantyours.com +0.0.0.0 ads.pni.com +0.0.0.0 ads.pno.net +0.0.0.0 ads.poconorecord.com +0.0.0.0 ads.pointroll.com +0.0.0.0 ads.portlandmercury.com +0.0.0.0 ads.premiumnetwork.com +0.0.0.0 ads.premiumnetwork.net +0.0.0.0 ads.pressdemo.com +0.0.0.0 ads.pricescan.com +0.0.0.0 ads.primaryclick.com +0.0.0.0 ads.primeinteractive.net +0.0.0.0 ads.prisacom.com +0.0.0.0 ads.profitsdeluxe.com +0.0.0.0 ads.profootballtalk.com +0.0.0.0 ads.program3.com +0.0.0.0 ads.pro-market.net +0.0.0.0 ads.pro-market.net.edgesuite.net +0.0.0.0 ads.prospect.org +0.0.0.0 ads.pubmatic.com +0.0.0.0 ads.queendom.com +0.0.0.0 ads.quicken.com +0.0.0.0 adsr3pg.com.br +0.0.0.0 ads.rackshack.net +0.0.0.0 ads.rasmussenreports.com +0.0.0.0 ads.ratemyprofessors.com +0.0.0.0 adsrc.bankrate.com +0.0.0.0 ads.rcgroups.com +0.0.0.0 ads.rdstore.com +0.0.0.0 ads.realcastmedia.com +0.0.0.0 ads.realcities.com +0.0.0.0 ads.realmedia.de +0.0.0.0 ads.realtechnetwork.net +0.0.0.0 ads.reason.com +0.0.0.0 ads.rediff.com +0.0.0.0 ads.redorbit.com +0.0.0.0 ads.register.com +0.0.0.0 adsremote.scripps.com +0.0.0.0 adsremote.scrippsnetwork.com +0.0.0.0 ads.revenews.com +0.0.0.0 ads.revenue.net +0.0.0.0 adsrevenue.net +0.0.0.0 ads.revsci.net +0.0.0.0 ads.rim.co.uk +0.0.0.0 ads-rm.looksmart.com +0.0.0.0 ads.roanoke.com +0.0.0.0 ads.rockstargames.com +0.0.0.0 ads.rodale.com +0.0.0.0 ads.roiserver.com +0.0.0.0 ads.rondomondo.com +0.0.0.0 ads.rootzoo.com +0.0.0.0 ads.rottentomatoes.com +0.0.0.0 ads.rp-online.de +0.0.0.0 ads.ruralpress.com +0.0.0.0 adsrv2.wilmingtonstar.com +0.0.0.0 adsrv.bankrate.com +0.0.0.0 adsrv.dispatch.com +0.0.0.0 adsrv.emporis.com +0.0.0.0 adsrv.heraldtribune.com +0.0.0.0 adsrv.hpg.com.br +0.0.0.0 adsrv.iol.co.za +0.0.0.0 adsrv.lua.pl +0.0.0.0 adsrv.news.com.au +0.0.0.0 adsrvr.com +0.0.0.0 adsrv.tuscaloosanews.com +0.0.0.0 adsrv.wilmingtonstar.com +0.0.0.0 ads.sacbee.com +0.0.0.0 ads.satyamonline.com +0.0.0.0 ads.savannahnow.com +0.0.0.0 ads.scabee.com +0.0.0.0 ads.schwabtrader.com +0.0.0.0 ads.scifi.com +0.0.0.0 ads.seattletimes.com +0.0.0.0 ads.sfusion.com +0.0.0.0 ads.shizmoo.com +0.0.0.0 ads.shoppingads.com +0.0.0.0 ads.shoutfile.com +0.0.0.0 ads.sify.com +0.0.0.0 ads.simtel.com +0.0.0.0 ads.simtel.net +0.0.0.0 ads.sitemeter.com +0.0.0.0 ads.sixapart.com +0.0.0.0 adssl01.adtech.de +0.0.0.0 adssl01.adtech.fr +0.0.0.0 adssl01.adtech.us +0.0.0.0 adssl02.adtech.de +0.0.0.0 adssl02.adtech.fr +0.0.0.0 adssl02.adtech.us +0.0.0.0 ads.sl.interpals.net +0.0.0.0 ads.smartclick.com +0.0.0.0 ads.smartclicks.com +0.0.0.0 ads.smartclicks.net +0.0.0.0 ads.snowball.com +0.0.0.0 ads.socialmedia.com +0.0.0.0 ads.sohh.com +0.0.0.0 ads.somethingawful.com +0.0.0.0 ads.space.com +0.0.0.0 adsspace.net +0.0.0.0 ads.specificclick.com +0.0.0.0 ads.specificmedia.com +0.0.0.0 ads.specificpop.com +0.0.0.0 ads.sptimes.com +0.0.0.0 ads.spymac.net +0.0.0.0 ads.stackoverflow.com +0.0.0.0 ads.starbanner.com +0.0.0.0 ads.stephensmedia.com +0.0.0.0 ads.stileproject.com +0.0.0.0 ads.stupid.com +0.0.0.0 ads.sunjournal.com +0.0.0.0 ads.sup.com +0.0.0.0 ads.swiftnews.com +0.0.0.0 ads.switchboard.com +0.0.0.0 ads.teamyehey.com +0.0.0.0 ads.technoratimedia.com +0.0.0.0 ads.techtv.com +0.0.0.0 ads.techvibes.com +0.0.0.0 ads.techweb.com +0.0.0.0 ads.telegraaf.nl +0.0.0.0 ads.telegraph.co.uk +0.0.0.0 ads.the15thinternet.com +0.0.0.0 ads.theawl.com +0.0.0.0 ads.thebugs.ws +0.0.0.0 ads.thecoolhunter.net +0.0.0.0 ads.thecrimson.com +0.0.0.0 ads.thefrisky.com +0.0.0.0 ads.thegauntlet.com +0.0.0.0 ads.theglobeandmail.com +0.0.0.0 ads.theindependent.com +0.0.0.0 ads.theolympian.com +0.0.0.0 ads.thesmokinggun.com +0.0.0.0 ads.thestar.com #Toronto Star +0.0.0.0 ads.thestranger.com +0.0.0.0 ads.thewebfreaks.com +0.0.0.0 adstil.indiatimes.com +0.0.0.0 ads.timesunion.com +0.0.0.0 ads.tiscali.fr +0.0.0.0 ads.tmcs.net +0.0.0.0 ads.tnt.tv +0.0.0.0 adstogo.com +0.0.0.0 adstome.com +0.0.0.0 ads.top500.org #TOP500 SuperComputer Site +0.0.0.0 ads.top-banners.com +0.0.0.0 ads.toronto.com +0.0.0.0 ads.townhall.com +0.0.0.0 ads.track.net +0.0.0.0 ads.traderonline.com +0.0.0.0 ads.traffichaus.com +0.0.0.0 ads.trafficjunky.net +0.0.0.0 ads.traffikings.com +0.0.0.0 adstream.cardboardfish.com +0.0.0.0 adstreams.org +0.0.0.0 ads.treehugger.com +0.0.0.0 ads.tricityherald.com +0.0.0.0 ads.trinitymirror.co.uk +0.0.0.0 ads.tripod.com +0.0.0.0 ads.tripod.lycos.co.uk +0.0.0.0 ads.tripod.lycos.de +0.0.0.0 ads.tripod.lycos.es +0.0.0.0 ads.tromaville.com +0.0.0.0 ads-t.ru +0.0.0.0 ads.trutv.com +0.0.0.0 ads.tucows.com +0.0.0.0 ads.tw.adsonar.com +0.0.0.0 ads.ucomics.com +0.0.0.0 ads.uigc.net +0.0.0.0 ads.undertone.com +0.0.0.0 ads.unixathome.org +0.0.0.0 ads.update.com +0.0.0.0 ad.suprnova.org +0.0.0.0 ads.uproar.com +0.0.0.0 ads.urbandictionary.com +0.0.0.0 ads.usatoday.com +0.0.0.0 ads.us.e-planning.ne +0.0.0.0 ads.us.e-planning.net +0.0.0.0 ads.userfriendly.org +0.0.0.0 ads.v3.com +0.0.0.0 ads.v3exchange.com +0.0.0.0 ads.vaildaily.com +0.0.0.0 ads.valuead.com +0.0.0.0 ads.vegas.com +0.0.0.0 ads.veloxia.com +0.0.0.0 ads.ventivmedia.com +0.0.0.0 ads.veoh.com +0.0.0.0 ads.verkata.com +0.0.0.0 ads.vesperexchange.com +0.0.0.0 ads.vg.basefarm.net +0.0.0.0 ads.viddler.com +0.0.0.0 ads.videoadvertising.com +0.0.0.0 ads.viewlondon.co.uk +0.0.0.0 ads.virginislandsdailynews.com +0.0.0.0 ads.virtualcountries.com +0.0.0.0 ads.vnuemedia.com +0.0.0.0 adsvr.adknowledge.com +0.0.0.0 ads.vs.co +0.0.0.0 ads.vs.com +0.0.0.0 ads.wanadooregie.com +0.0.0.0 ads.warcry.com +0.0.0.0 ads.watershed-publishing.com +0.0.0.0 ads.wave.si +0.0.0.0 ads.weather.ca +0.0.0.0 ads.weather.com +0.0.0.0 ads.web21.com +0.0.0.0 ads.web.alwayson-network.com +0.0.0.0 ads.web.aol.com +0.0.0.0 ads.webattack.com +0.0.0.0 ads.web.compuserve.com +0.0.0.0 ads.webcoretech.com +0.0.0.0 ads.web.cs.com +0.0.0.0 ads.web.de +0.0.0.0 ads.webfeat.com +0.0.0.0 ads.webheat.com +0.0.0.0 ads.webhosting.info +0.0.0.0 ads.webindia123.com +0.0.0.0 ads-web.mail.com +0.0.0.0 ads.webmd.com +0.0.0.0 ads.webnet.advance.net +0.0.0.0 ads.websponsors.com +0.0.0.0 adsweb.tiscali.cz +0.0.0.0 ads.weissinc.com +0.0.0.0 ads.whaleads.com +0.0.0.0 ads.whi.co.nz +0.0.0.0 ads.winsite.com +0.0.0.0 ads.wnd.com +0.0.0.0 ads.wunderground.com +0.0.0.0 ads.x10.com +0.0.0.0 ads.x10.net +0.0.0.0 ads.x17online.com +0.0.0.0 ads.xboxic.com +0.0.0.0 ads.xbox-scene.com +0.0.0.0 ads.xposed.com +0.0.0.0 ads.xtra.ca +0.0.0.0 ads.xtra.co.nz +0.0.0.0 ads.xtramsn.co.nz +0.0.0.0 ads.yahoo.com +0.0.0.0 ads.yimg.com +0.0.0.0 ads.yimg.com.edgesuite.net +0.0.0.0 ads.yldmgrimg.net +0.0.0.0 adsyndication.msn.com +0.0.0.0 adsyndication.yelldirect.com +0.0.0.0 adsynergy.com +0.0.0.0 ads.youporn.com +0.0.0.0 ads.youtube.com +0.0.0.0 adsys.townnews.com +0.0.0.0 ads.zap2it.com +0.0.0.0 ads.zdnet.com +0.0.0.0 adtag.msn.ca +0.0.0.0 adtag.sympatico.ca +0.0.0.0 adtaily.com +0.0.0.0 adtaily.pl +0.0.0.0 ad.tbn.ru +0.0.0.0 adtcp.ru +0.0.0.0 adtech.de +0.0.0.0 ad.technoramedia.com +0.0.0.0 adtech.panthercustomer.com +0.0.0.0 adtechus.com +0.0.0.0 adtegrity.spinbox.net +0.0.0.0 adtext.pl +0.0.0.0 ad.text.tbn.ru +0.0.0.0 ad.tgdaily.com +0.0.0.0 ad.thehill.com +0.0.0.0 ad.thetyee.ca +0.0.0.0 ad.thewheelof.com +0.0.0.0 adthru.com +0.0.0.0 adtigerpl.adspirit.net +0.0.0.0 ad.tiscali.com +0.0.0.0 adtlgc.com +0.0.0.0 adtology3.com +0.0.0.0 ad.tomshardware.com +0.0.0.0 adtotal.pl +0.0.0.0 adtracking.vinden.nl +0.0.0.0 adtrader.com +0.0.0.0 ad.trafficmp.com +0.0.0.0 adtrak.net +0.0.0.0 ad.turn.com +0.0.0.0 ad.tv2.no +0.0.0.0 ad.twitchguru.com +0.0.0.0 ad.ubnm.co.kr +0.0.0.0 ad.uk.tangozebra.com +0.0.0.0 ad-uk.tiscali.com +0.0.0.0 adultadworld.com +0.0.0.0 ad.usatoday.com +0.0.0.0 adv0005.247realmedia.com +0.0.0.0 adv0035.247realmedia.com +0.0.0.0 adv.440net.com +0.0.0.0 adv.adgates.com +0.0.0.0 adv.adtotal.pl +0.0.0.0 adv.adview.pl +0.0.0.0 adv.bannercity.ru +0.0.0.0 adv.bbanner.it +0.0.0.0 adv.bookclubservices.ca +0.0.0.0 adveng.hiasys.com +0.0.0.0 adveraction.pl +0.0.0.0 advert.bayarea.com +0.0.0.0 advertise.com +0.0.0.0 advertisers.federatedmedia.net +0.0.0.0 advertising.aol.com +0.0.0.0 advertisingbay.com +0.0.0.0 advertising.bbcworldwide.com +0.0.0.0 advertising.com +0.0.0.0 advertising.gfxartist.com +0.0.0.0 advertising.hiasys.com +0.0.0.0 advertising.illinimedia.com +0.0.0.0 advertising.online-media24.de +0.0.0.0 advertising.paltalk.com +0.0.0.0 advertising.wellpack.fr +0.0.0.0 advertising.zenit.org +0.0.0.0 advertlets.com +0.0.0.0 advertpro.investorvillage.com +0.0.0.0 advertpro.sitepoint.com +0.0.0.0 adverts.digitalspy.co.uk +0.0.0.0 adverts.ecn.co.uk +0.0.0.0 adverts.freeloader.com +0.0.0.0 adverts.im4ges.com +0.0.0.0 advertstream.com +0.0.0.0 advert.uloz.to +0.0.0.0 adv.federalpost.ru +0.0.0.0 adv.gazeta.pl +0.0.0.0 advicepl.adocean.pl +0.0.0.0 adview.pl +0.0.0.0 adviva.net +0.0.0.0 adv.lampsplus.com +0.0.0.0 advmaker.ru +0.0.0.0 adv.merlin.co.il +0.0.0.0 adv.netshelter.net +0.0.0.0 adv.publy.net +0.0.0.0 adv.surinter.net +0.0.0.0 advt.webindia123.com +0.0.0.0 ad.vurts.com +0.0.0.0 adv.virgilio.it +0.0.0.0 adv.webmd.com +0.0.0.0 adv.wp.pl +0.0.0.0 adv.zapal.ru +0.0.0.0 advzilla.com +0.0.0.0 adware.kogaryu.com +0.0.0.0 adweb2.hornymatches.com +0.0.0.0 ad.webprovider.com +0.0.0.0 adw.sapo.pt +0.0.0.0 ad.wsod.com +0.0.0.0 adx.adrenalinesk.sk +0.0.0.0 adx.gainesvillesun.com +0.0.0.0 adx.gainesvillsun.com +0.0.0.0 adx.groupstate.com +0.0.0.0 adx.hendersonvillenews.com +0.0.0.0 adx.heraldtribune.com +0.0.0.0 adxpose.com +0.0.0.0 adx.starnewsonline.com +0.0.0.0 ad.xtendmedia.com +0.0.0.0 adx.theledger.com +0.0.0.0 ad.yadro.ru +0.0.0.0 ad.yieldmanager.com +0.0.0.0 adz.afterdawn.net +0.0.0.0 ad.zanox.com +0.0.0.0 adzerk.net +0.0.0.0 ad.zodera.hu +0.0.0.0 adzone.ro +0.0.0.0 adzone.stltoday.com +0.0.0.0 adzservice.theday.com +0.0.0.0 ae.goodsblock.marketgid.com +0.0.0.0 afe2.specificclick.net +0.0.0.0 afe.specificclick.net +0.0.0.0 aff.foxtab.com +0.0.0.0 affiliate.a4dtracker.com +0.0.0.0 affiliate.aol.com +0.0.0.0 affiliate.baazee.com +0.0.0.0 affiliate.cfdebt.com +0.0.0.0 affiliate.exabytes.com.my +0.0.0.0 affiliate-fr.com +0.0.0.0 affiliate.fr.espotting.com +0.0.0.0 affiliate.googleusercontent.com +0.0.0.0 affiliate.hbytracker.com +0.0.0.0 affiliate.mlntracker.com +0.0.0.0 affiliates.arvixe.com +0.0.0.0 affiliates.eblastengine.com +0.0.0.0 affiliates.genealogybank.com +0.0.0.0 affiliates.globat.com +0.0.0.0 affiliation-france.com +0.0.0.0 affimg.pop6.com +0.0.0.0 afform.co.uk +0.0.0.0 affpartners.com +0.0.0.0 aff.ringtonepartner.com +0.0.0.0 afi.adocean.pl +0.0.0.0 afilo.pl +0.0.0.0 agkn.com +0.0.0.0 aj.600z.com +0.0.0.0 ajcclassifieds.com +0.0.0.0 akaads-espn.starwave.com +0.0.0.0 aka-cdn.adtechus.com +0.0.0.0 aka-cdn-ns.adtech.de +0.0.0.0 aka-cdn-ns.adtechus.com +0.0.0.0 akamai.invitemedia.com +0.0.0.0 ak.buyservices.com +0.0.0.0 a.kerg.net +0.0.0.0 ak.maxserving.com +0.0.0.0 ako.cc +0.0.0.0 ak.p.openx.net +0.0.0.0 al1.sharethis.com +0.0.0.0 alert.police-patrol-agent.com +0.0.0.0 a.ligatus.com +0.0.0.0 a.ligatus.de +0.0.0.0 alliance.adbureau.net +0.0.0.0 all.orfr.adgtw.orangeads.fr +0.0.0.0 altfarm.mediaplex.com +0.0.0.0 amch.questionmarket.com +0.0.0.0 americansingles.click-url.com +0.0.0.0 a.mktw.net +0.0.0.0 amscdn.btrll.com +0.0.0.0 analysis.fc2.com +0.0.0.0 analytics.kwebsoft.com +0.0.0.0 analytics.percentmobile.com +0.0.0.0 analyzer51.fc2.com +0.0.0.0 ankieta-online.pl +0.0.0.0 annuaire-autosurf.com +0.0.0.0 anrtx.tacoda.net +0.0.0.0 answers.us.intellitxt.com +0.0.0.0 an.tacoda.net +0.0.0.0 an.yandex.ru +0.0.0.0 apex-ad.com +0.0.0.0 api.addthis.com +0.0.0.0 api.affinesystems.com +0.0.0.0 api-public.addthis.com +0.0.0.0 apopt.hbmediapro.com +0.0.0.0 apparelncs.com +0.0.0.0 apparel-offer.com +0.0.0.0 appdev.addthis.com +0.0.0.0 appnexus.com +0.0.0.0 apps5.oingo.com +0.0.0.0 app.scanscout.com +0.0.0.0 ap.read.mediation.pns.ap.orangeads.fr +0.0.0.0 a.prisacom.com +0.0.0.0 apx.moatads.com +0.0.0.0 a.rad.live.com +0.0.0.0 a.rad.msn.com +0.0.0.0 arbomedia.pl +0.0.0.0 arbopl.bbelements.com +0.0.0.0 arsconsole.global-intermedia.com +0.0.0.0 art-music-rewardpath.com +0.0.0.0 art-offer.com +0.0.0.0 art-offer.net +0.0.0.0 art-photo-music-premiumblvd.com +0.0.0.0 art-photo-music-rewardempire.com +0.0.0.0 art-photo-music-savingblvd.com +0.0.0.0 as1.falkag.de +0.0.0.0 as1image1.adshuffle.com +0.0.0.0 as1image2.adshuffle.com +0.0.0.0 as1.inoventiv.com +0.0.0.0 as2.falkag.de +0.0.0.0 as3.falkag.de +0.0.0.0 as4.falkag.de +0.0.0.0 as.5to1.com +0.0.0.0 asa.tynt.com +0.0.0.0 asb.tynt.com +0.0.0.0 as.casalemedia.com +0.0.0.0 as.ebz.io +0.0.0.0 asg01.casalemedia.com +0.0.0.0 asg02.casalemedia.com +0.0.0.0 asg03.casalemedia.com +0.0.0.0 asg04.casalemedia.com +0.0.0.0 asg05.casalemedia.com +0.0.0.0 asg06.casalemedia.com +0.0.0.0 asg07.casalemedia.com +0.0.0.0 asg08.casalemedia.com +0.0.0.0 asg09.casalemedia.com +0.0.0.0 asg10.casalemedia.com +0.0.0.0 asg11.casalemedia.com +0.0.0.0 asg12.casalemedia.com +0.0.0.0 asg13.casalemedia.com +0.0.0.0 ask-gps.ru +0.0.0.0 asklots.com +0.0.0.0 askmen.thruport.com +0.0.0.0 asm2.z1.adserver.com +0.0.0.0 asm3.z1.adserver.com +0.0.0.0 asn.advolution.de +0.0.0.0 asn.cunda.advolution.biz +0.0.0.0 a.ss34.on9mail.com +0.0.0.0 assets.igapi.com +0.0.0.0 assets.kixer.com +0.0.0.0 assets.percentmobile.com +0.0.0.0 as.sexad.net +0.0.0.0 asv.nuggad.net +0.0.0.0 as.vs4entertainment.com +0.0.0.0 as.webmd.com +0.0.0.0 a.tadd.react2media.com +0.0.0.0 at-adserver.alltop.com +0.0.0.0 at.campaigns.f2.com.au +0.0.0.0 at.ceofreehost.com +0.0.0.0 atdmt.com +0.0.0.0 atemda.com +0.0.0.0 athena-ads.wikia.com +0.0.0.0 at.m1.nedstatbasic.net +0.0.0.0 a.total-media.net +0.0.0.0 a.tribalfusion.com +0.0.0.0 a.triggit.com +0.0.0.0 au.adserver.yahoo.com +0.0.0.0 au.ads.link4ads.com +0.0.0.0 aud.pubmatic.com +0.0.0.0 aureate.com +0.0.0.0 auslieferung.commindo-media-ressourcen.de +0.0.0.0 austria1.adverserve.net +0.0.0.0 autocontext.begun.ru +0.0.0.0 automotive-offer.com +0.0.0.0 automotive-rewardpath.com +0.0.0.0 avcounter10.com +0.0.0.0 avpa.dzone.com +0.0.0.0 avpa.javalobby.org +0.0.0.0 a.websponsors.com +0.0.0.0 awesomevipoffers.com +0.0.0.0 awrz.net +0.0.0.0 axp.zedo.com +0.0.0.0 azcentra.app.ur.gcion.com +0.0.0.0 azoogleads.com +0.0.0.0 b1.adbrite.com +0.0.0.0 b1.azjmp.com +0.0.0.0 b2b.filecloud.me +0.0.0.0 babycenter.tt.omtrdc.net +0.0.0.0 b.ads2.msn.com +0.0.0.0 badservant.guj.de +0.0.0.0 b.am15.net +0.0.0.0 bananacashback.com +0.0.0.0 banery.acr.pl +0.0.0.0 banery.netart.pl +0.0.0.0 banery.onet.pl +0.0.0.0 banki.onet.pl +0.0.0.0 bankofamerica.tt.omtrdc.net +0.0.0.0 banman.nepsecure.co.uk +0.0.0.0 banner.1and1.co.uk +0.0.0.0 banner1.pornhost.com +0.0.0.0 banner2.inet-traffic.com +0.0.0.0 bannerads.anytimenews.com +0.0.0.0 bannerads.de +0.0.0.0 bannerads.zwire.com +0.0.0.0 banner.affactive.com +0.0.0.0 banner.betroyalaffiliates.com +0.0.0.0 banner.betwwts.com +0.0.0.0 banner.cdpoker.com +0.0.0.0 banner.clubdicecasino.com +0.0.0.0 bannerconnect.net +0.0.0.0 banner.coza.com +0.0.0.0 banner.diamondclubcasino.com +0.0.0.0 bannerdriven.ru +0.0.0.0 banner.easyspace.com +0.0.0.0 bannerfarm.ace.advertising.com +0.0.0.0 banner.free6.com # www.free6.com +0.0.0.0 bannerhost.egamingonline.com +0.0.0.0 bannerimages.0catch.com +0.0.0.0 banner.joylandcasino.com +0.0.0.0 banner.media-system.de +0.0.0.0 banner.monacogoldcasino.com +0.0.0.0 banner.newyorkcasino.com +0.0.0.0 banner.northsky.com +0.0.0.0 banner.oddcast.com +0.0.0.0 banner.orb.net +0.0.0.0 banner.piratos.de +0.0.0.0 banner.playgatecasino.com +0.0.0.0 bannerpower.com +0.0.0.0 banner.prestigecasino.com +0.0.0.0 banner.publisher.to +0.0.0.0 banner.rbc.ru +0.0.0.0 banner.relcom.ru +0.0.0.0 banners1.linkbuddies.com +0.0.0.0 banners2.castles.org +0.0.0.0 banners3.spacash.com +0.0.0.0 banners.adgoto.com +0.0.0.0 banners.adultfriendfinder.com +0.0.0.0 banners.affiliatefuel.com +0.0.0.0 banners.affiliatefuture.com +0.0.0.0 banners.aftrk.com +0.0.0.0 banners.audioholics.com +0.0.0.0 banners.blogads.com +0.0.0.0 banners.bol.se +0.0.0.0 banners.broadwayworld.com +0.0.0.0 banners.celebritybling.com +0.0.0.0 banners.crisscross.com +0.0.0.0 banners.directnic.com +0.0.0.0 banners.dnastudio.com +0.0.0.0 banners.easydns.com +0.0.0.0 banners.easysolutions.be +0.0.0.0 banners.ebay.com +0.0.0.0 banners.expressindia.com +0.0.0.0 banners.flair.be +0.0.0.0 banners.free6.com # www.free6.com +0.0.0.0 banners.fuifbeest.be +0.0.0.0 banners.globovision.com +0.0.0.0 banners.img.uol.com.br +0.0.0.0 banners.ims.nl +0.0.0.0 banners.iop.org +0.0.0.0 banners.ipotd.com +0.0.0.0 banners.japantoday.com +0.0.0.0 banners.kfmb.com +0.0.0.0 banners.ksl.com +0.0.0.0 banners.linkbuddies.com +0.0.0.0 banners.looksmart.com +0.0.0.0 banners.nbcupromotes.com +0.0.0.0 banners.netcraft.com +0.0.0.0 banners.newsru.com +0.0.0.0 banners.nextcard.com +0.0.0.0 banners.passion.com +0.0.0.0 banners.pennyweb.com +0.0.0.0 banners.primaryclick.com +0.0.0.0 banners.resultonline.com +0.0.0.0 banners.rspworldwide.com +0.0.0.0 banners.sextracker.com +0.0.0.0 banners.spiceworks.com +0.0.0.0 banners.thegridwebmaster.com +0.0.0.0 banners.thestranger.com +0.0.0.0 banners.thgimages.co.uk +0.0.0.0 banners.tribute.ca +0.0.0.0 banners.tucson.com +0.0.0.0 banners.unibet.com +0.0.0.0 bannersurvey.biz +0.0.0.0 banners.valuead.com +0.0.0.0 banners.videosecrets.com +0.0.0.0 banners.webmasterplan.com +0.0.0.0 banners.wunderground.com +0.0.0.0 banners.zbs.ru +0.0.0.0 banner.tattomedia.com +0.0.0.0 banner.techarp.com +0.0.0.0 bannert.ru +0.0.0.0 bannerus1.axelsfun.com +0.0.0.0 bannerus3.axelsfun.com +0.0.0.0 banner.usacasino.com +0.0.0.0 banniere.reussissonsensemble.fr +0.0.0.0 bans.bride.ru +0.0.0.0 banstex.com +0.0.0.0 bansys.onzin.com +0.0.0.0 bargainbeautybuys.com +0.0.0.0 barnesandnoble.bfast.com +0.0.0.0 b.as-us.falkag.net +0.0.0.0 bayoubuzz.advertserve.com +0.0.0.0 bbcdn.go.adlt.bbelements.com +0.0.0.0 bbcdn.go.adnet.bbelements.com +0.0.0.0 bbcdn.go.arbo.bbelements.com +0.0.0.0 bbcdn.go.eu.bbelements.com +0.0.0.0 bbcdn.go.ihned.bbelements.com +0.0.0.0 bbcdn.go.pl.bbelements.com +0.0.0.0 bb.crwdcntrl.net +0.0.0.0 bbnaut.bbelements.com +0.0.0.0 bc685d37-266c-488e-824e-dd95d1c0e98b.statcamp.net +0.0.0.0 bcp.crwdcntrl.net +0.0.0.0 bdnad1.bangornews.com +0.0.0.0 bdv.bidvertiser.com +0.0.0.0 beacon-3.newrelic.com +0.0.0.0 beacons.helium.com +0.0.0.0 bell.adcentriconline.com +0.0.0.0 beseenad.looksmart.com +0.0.0.0 bestgift4you.cn +0.0.0.0 bestshopperrewards.com +0.0.0.0 beta.hotkeys.com +0.0.0.0 bet-at-home.com +0.0.0.0 betterperformance.goldenopps.info +0.0.0.0 bfast.com +0.0.0.0 bidclix.net +0.0.0.0 bid.openx.net +0.0.0.0 bidsystem.com +0.0.0.0 bidtraffic.com +0.0.0.0 bidvertiser.com +0.0.0.0 bigads.guj.de +0.0.0.0 bigbrandpromotions.com +0.0.0.0 bigbrandrewards.com +0.0.0.0 biggestgiftrewards.com +0.0.0.0 billing.speedboink.com +0.0.0.0 bitburg.adtech.de +0.0.0.0 bitburg.adtech.fr +0.0.0.0 bitburg.adtech.us +0.0.0.0 bitcast-d.bitgravity.com +0.0.0.0 bizad.nikkeibp.co.jp +0.0.0.0 biz-offer.com +0.0.0.0 bizopprewards.com +0.0.0.0 blabla4u.adserver.co.il +0.0.0.0 blasphemysfhs.info +0.0.0.0 blatant8jh.info +0.0.0.0 b.liquidustv.com +0.0.0.0 blog.addthis.com +0.0.0.0 blogads.com +0.0.0.0 blogads.ebanner.nl +0.0.0.0 blogvertising.pl +0.0.0.0 bluediamondoffers.com +0.0.0.0 blu.mobileads.msn.com +0.0.0.0 bl.wavecdn.de +0.0.0.0 b.myspace.com +0.0.0.0 bn.bfast.com +0.0.0.0 bnmgr.adinjector.net +0.0.0.0 bnrs.ilm.ee +0.0.0.0 boksy.dir.onet.pl +0.0.0.0 boksy.onet.pl +0.0.0.0 bookclub-offer.com +0.0.0.0 books-media-edu-premiumblvd.com +0.0.0.0 books-media-edu-rewardempire.com +0.0.0.0 books-media-rewardpath.com +0.0.0.0 bostonsubwayoffer.com +0.0.0.0 bp.specificclick.net +0.0.0.0 b.rad.live.com +0.0.0.0 b.rad.msn.com +0.0.0.0 br.adserver.yahoo.com +0.0.0.0 brandrewardcentral.com +0.0.0.0 brandsurveypanel.com +0.0.0.0 bravo.israelinfo.ru +0.0.0.0 bravospots.com +0.0.0.0 br.naked.com +0.0.0.0 broadcast.piximedia.fr +0.0.0.0 broadent.vo.llnwd.net +0.0.0.0 brokertraffic.com +0.0.0.0 bsads.looksmart.com +#0.0.0.0 b.scorecardresearch.com # interferes with Huffington Post slideshows +0.0.0.0 bs.israelinfo.ru +0.0.0.0 bs.serving-sys.com #eyeblaster.com +0.0.0.0 bt.linkpulse.com +0.0.0.0 burns.adtech.de +0.0.0.0 burns.adtech.fr +0.0.0.0 burns.adtech.us +0.0.0.0 business-rewardpath.com +0.0.0.0 bus-offer.com +0.0.0.0 buttcandy.com +0.0.0.0 buttons.googlesyndication.com +0.0.0.0 buzzbox.buzzfeed.com +0.0.0.0 bwp.lastfm.com.com +0.0.0.0 bwp.news.com +0.0.0.0 c1.popads.net +0.0.0.0 c1.teaser-goods.ru +0.0.0.0 c1.zedo.com +0.0.0.0 c2.zedo.com +0.0.0.0 c3.zedo.com +0.0.0.0 c4.maxserving.com +0.0.0.0 c4.zedo.com +0.0.0.0 c5.zedo.com +0.0.0.0 c6.zedo.com +0.0.0.0 c7.zedo.com +0.0.0.0 c8.zedo.com +0.0.0.0 ca.adserver.yahoo.com +0.0.0.0 cache.addthiscdn.com +0.0.0.0 cache.addthis.com +0.0.0.0 cache.blogads.com +0.0.0.0 cache-dev.addthis.com +0.0.0.0 cacheserve.eurogrand.com +0.0.0.0 cacheserve.prestigecasino.com +0.0.0.0 cache.unicast.com +0.0.0.0 c.actiondesk.com +0.0.0.0 c.adroll.com +0.0.0.0 califia.imaginemedia.com +0.0.0.0 c.am10.ru +0.0.0.0 camgeil.com +0.0.0.0 campaign.iitech.dk +0.0.0.0 campaign.indieclick.com +0.0.0.0 campaigns.f2.com.au +0.0.0.0 campaigns.interclick.com +0.0.0.0 capath.com +0.0.0.0 cardgamespidersolitaire.com +0.0.0.0 cards.virtuagirlhd.com +0.0.0.0 careers.canwestad.net +0.0.0.0 careers-rewardpath.com +0.0.0.0 c.ar.msn.com +0.0.0.0 carrier.bz +0.0.0.0 car-truck-boat-bonuspath.com +0.0.0.0 car-truck-boat-premiumblvd.com +0.0.0.0 casalemedia.com +0.0.0.0 cas.clickability.com +0.0.0.0 cashback.co.uk +0.0.0.0 cashbackwow.co.uk +0.0.0.0 cashflowmarketing.com +0.0.0.0 casino770.com +0.0.0.0 c.as-us.falkag.net +0.0.0.0 catalinkcashback.com +0.0.0.0 catchvid.info +0.0.0.0 c.at.msn.com +0.0.0.0 cbanners.virtuagirlhd.com +0.0.0.0 c.be.msn.com +0.0.0.0 c.blogads.com +0.0.0.0 c.br.msn.com +0.0.0.0 c.ca.msn.com +0.0.0.0 c.casalemedia.com +0.0.0.0 ccas.clearchannel.com +0.0.0.0 c.cl.msn.com +0.0.0.0 c.de.msn.com +0.0.0.0 c.dk.msn.com +0.0.0.0 cdn1.adexprt.com +0.0.0.0 cdn1.ads.mofos.com +0.0.0.0 cdn1.eyewonder.com +0.0.0.0 cdn1.rmgserving.com +0.0.0.0 cdn1.traffichaus.com +0.0.0.0 cdn1.xlightmedia.com +0.0.0.0 cdn2.adsdk.com +0.0.0.0 cdn2.amateurmatch.com +0.0.0.0 cdn2.emediate.eu +0.0.0.0 cdn3.adexprts.com +0.0.0.0 cdn3.telemetryverification.net +0.0.0.0 cdn454.telemetryverification.net +0.0.0.0 cdn5.tribalfusion.com +0.0.0.0 cdn6.emediate.eu +0.0.0.0 cdn.adigniter.org +0.0.0.0 cdn.adnxs.com +0.0.0.0 cdnads.cam4.com +0.0.0.0 cdn.ads.ookla.com +0.0.0.0 cdn.amateurmatch.com +0.0.0.0 cdn.amgdgt.com +0.0.0.0 cdn.assets.craveonline.com +0.0.0.0 cdn.banners.scubl.com +0.0.0.0 cdn.cpmstar.com +0.0.0.0 cdn.crowdignite.com +0.0.0.0 cdn.directrev.com +0.0.0.0 cdn.eyewonder.com +0.0.0.0 cdn.go.arbo.bbelements.com +0.0.0.0 cdn.go.arbopl.bbelements.com +0.0.0.0 cdn.go.cz.bbelements.com +0.0.0.0 cdn.go.idmnet.bbelements.com +0.0.0.0 cdn.go.pol.bbelements.com +0.0.0.0 cdn.hadj7.adjuggler.net +0.0.0.0 cdn.innovid.com +0.0.0.0 cdn.krxd.net +0.0.0.0 cdn.mediative.ca +0.0.0.0 cdn.merchenta.com +0.0.0.0 cdn.mobicow.com +0.0.0.0 cdn.nearbyad.com +0.0.0.0 cdn.nsimg.net +0.0.0.0 cdn.onescreen.net +#0.0.0.0 cdns.gigya.com +0.0.0.0 cdns.mydirtyhobby.com +0.0.0.0 cdns.privatamateure.com +0.0.0.0 cdn.stat.easydate.biz +0.0.0.0 cdn.syn.verticalacuity.com +0.0.0.0 cdn.tabnak.ir +0.0.0.0 cdnt.yottos.com +0.0.0.0 cdn.udmserve.net +0.0.0.0 cdn.undertone.com +0.0.0.0 cdn.wg.uproxx.com +0.0.0.0 cdnw.ringtonepartner.com +0.0.0.0 cdn.yottos.com +0.0.0.0 cdn.zeusclicks.com +0.0.0.0 cds.adecn.com +0.0.0.0 cecash.com +0.0.0.0 ced.sascdn.com +0.0.0.0 cell-phone-giveaways.com +0.0.0.0 cellphoneincentives.com +0.0.0.0 cent.adbureau.net +0.0.0.0 c.es.msn.com +0.0.0.0 c.fi.msn.com +0.0.0.0 cf.kampyle.com +0.0.0.0 c.fr.msn.com +0.0.0.0 cgirm.greatfallstribune.com +0.0.0.0 cgm.adbureau.ne +0.0.0.0 cgm.adbureau.net +0.0.0.0 c.gr.msn.com +0.0.0.0 chainsawoffer.com +0.0.0.0 chartbeat.com +0.0.0.0 checkintocash.data.7bpeople.com +0.0.0.0 cherryhi.app.ur.gcion.com +0.0.0.0 c.hk.msn.com +0.0.0.0 chkpt.zdnet.com +0.0.0.0 choicedealz.com +0.0.0.0 choicesurveypanel.com +0.0.0.0 christianbusinessadvertising.com +0.0.0.0 c.id.msn.com +0.0.0.0 c.ie.msn.com +0.0.0.0 c.il.msn.com +0.0.0.0 c.imedia.cz +0.0.0.0 c.in.msn.com +0.0.0.0 cithingy.info +0.0.0.0 citi.bridgetrack.com +0.0.0.0 c.it.msn.com +0.0.0.0 citrix.market2lead.com +0.0.0.0 cityads.telus.net +0.0.0.0 citycash2.blogspot.com +0.0.0.0 c.jp.msn.com +0.0.0.0 cl21.v4.adaction.se +0.0.0.0 cl320.v4.adaction.se +0.0.0.0 claimfreerewards.com +0.0.0.0 clashmediausa.com +0.0.0.0 classicjack.com +0.0.0.0 c.latam.msn.com +0.0.0.0 click1.mainadv.com +0.0.0.0 click1.rbc.magna.ru +0.0.0.0 click2.rbc.magna.ru +0.0.0.0 click3.rbc.magna.ru +0.0.0.0 click4.rbc.magna.ru +0.0.0.0 clickad.eo.pl +0.0.0.0 clickarrows.com +0.0.0.0 click.avenuea.com +0.0.0.0 clickbangpop.com +0.0.0.0 clickcash.webpower.com +0.0.0.0 click.go2net.com +0.0.0.0 click.israelinfo.ru +0.0.0.0 clickit.go2net.com +0.0.0.0 clickmedia.ro +0.0.0.0 click.pulse360.com +0.0.0.0 clicks2.virtuagirl.com +0.0.0.0 clicks.adultplex.com +0.0.0.0 clicks.deskbabes.com +0.0.0.0 click-see-save.com +0.0.0.0 clicksor.com +0.0.0.0 clicksotrk.com +0.0.0.0 clicks.totemcash.com +0.0.0.0 clicks.toteme.com +0.0.0.0 clicks.virtuagirl.com +0.0.0.0 clicks.virtuagirlhd.com +0.0.0.0 clicks.virtuaguyhd.com +0.0.0.0 clicks.walla.co.il +0.0.0.0 clickthru.net +0.0.0.0 clickthrunet.net +0.0.0.0 clickthruserver.com +0.0.0.0 clickthrutraffic.com +0.0.0.0 clicktorrent.info +0.0.0.0 clipserv.adclip.com +0.0.0.0 clkads.com +0.0.0.0 clk.cloudyisland.com +0.0.0.0 clk.tradedoubler.com +0.0.0.0 clkuk.tradedoubler.com +0.0.0.0 c.lomadee.com +0.0.0.0 closeoutproductsreview.com +0.0.0.0 cluster3.adultadworld.com +0.0.0.0 cluster.adultadworld.com +0.0.0.0 cm1359.com +0.0.0.0 cmads.sv.publicus.com +0.0.0.0 cmads.us.publicus.com +0.0.0.0 cmap.am.ace.advertising.com +0.0.0.0 cmap.an.ace.advertising.com +0.0.0.0 cmap.at.ace.advertising.com +0.0.0.0 cmap.dc.ace.advertising.com +0.0.0.0 cmap.ox.ace.advertising.com +0.0.0.0 cmap.pub.ace.advertising.com +0.0.0.0 cmap.rm.ace.advertising.com +0.0.0.0 cmap.rub.ace.advertising.com +0.0.0.0 cmhtml.overture.com +0.0.0.0 cmn1lsm2.beliefnet.com +0.0.0.0 cm.npc-hearst.overture.com +0.0.0.0 cmps.mt50ad.com +0.0.0.0 cm.the-n.overture.com +0.0.0.0 c.my.msn.com +0.0.0.0 cnad1.economicoutlook.net +0.0.0.0 cnad2.economicoutlook.net +0.0.0.0 cnad3.economicoutlook.net +0.0.0.0 cnad4.economicoutlook.net +0.0.0.0 cnad5.economicoutlook.net +0.0.0.0 cnad6.economicoutlook.net +0.0.0.0 cnad7.economicoutlook.net +0.0.0.0 cnad8.economicoutlook.net +0.0.0.0 cnad9.economicoutlook.net +0.0.0.0 cnad.economicoutlook.net +0.0.0.0 cn.adserver.yahoo.com +0.0.0.0 cnf.adshuffle.com +0.0.0.0 c.ninemsn.com.au +0.0.0.0 c.nl.msn.com +0.0.0.0 c.no.msn.com +0.0.0.0 c.novostimira.biz +0.0.0.0 cnt1.xhamster.com +0.0.0.0 code2.adtlgc.com +0.0.0.0 code.adtlgc.com +0.0.0.0 collectiveads.net +0.0.0.0 col.mobileads.msn.com +0.0.0.0 comadverts.bcmpweb.co.nz +0.0.0.0 comcastresidentialservices.tt.omtrdc.net +0.0.0.0 com.cool-premiums-now.com +0.0.0.0 come-see-it-all.com +0.0.0.0 com.htmlwww.youfck.com +0.0.0.0 commerce-offer.com +0.0.0.0 commerce-rewardpath.com +0.0.0.0 commerce.www.ibm.com +0.0.0.0 common.ziffdavisinternet.com +0.0.0.0 companion.adap.tv +0.0.0.0 computer-offer.com +0.0.0.0 computer-offer.net +0.0.0.0 computers-electronics-rewardpath.com +0.0.0.0 computersncs.com +0.0.0.0 com.shc-rebates.com +0.0.0.0 connect.247media.ads.link4ads.com +0.0.0.0 consumergiftcenter.com +0.0.0.0 consumerincentivenetwork.com +0.0.0.0 consumerinfo.tt.omtrdc.net +0.0.0.0 consumer-org.com +0.0.0.0 contaxe.com +0.0.0.0 content.ad-flow.com +0.0.0.0 content.clipster.ws +0.0.0.0 content.codelnet.com +0.0.0.0 content.promoisland.net +0.0.0.0 contentsearch.de.espotting.com +0.0.0.0 content.yieldmanager.edgesuite.net +0.0.0.0 context3.kanoodle.com +0.0.0.0 context5.kanoodle.com +0.0.0.0 context.adshadow.net +0.0.0.0 contextweb.com +0.0.0.0 conv.adengage.com +0.0.0.0 conversion-pixel.invitemedia.com +0.0.0.0 cookiecontainer.blox.pl +0.0.0.0 cookie.pebblemedia.be +0.0.0.0 cookingtiprewards.com +0.0.0.0 cookonsea.com +0.0.0.0 cool-premiums.com +0.0.0.0 cool-premiums-now.com +0.0.0.0 coolpremiumsnow.com +0.0.0.0 coolsavings.com +0.0.0.0 corba.adtech.de +0.0.0.0 corba.adtech.fr +0.0.0.0 corba.adtech.us +0.0.0.0 core0.node12.top.mail.ru +0.0.0.0 core2.adtlgc.com +0.0.0.0 coreg.flashtrack.net +0.0.0.0 coreglead.co.uk +0.0.0.0 core.insightexpressai.com +0.0.0.0 core.videoegg.com +0.0.0.0 cornflakes.pathfinder.com +0.0.0.0 corusads.dserv.ca +0.0.0.0 cosmeticscentre.uk.com +0.0.0.0 count6.51yes.com +0.0.0.0 count.casino-trade.com +0.0.0.0 cover.m2y.siemens.ch +0.0.0.0 c.ph.msn.com +0.0.0.0 cpmadvisors.com +0.0.0.0 cp.promoisland.net +0.0.0.0 c.prodigy.msn.com +0.0.0.0 c.pt.msn.com +0.0.0.0 cpu.firingsquad.com +0.0.0.0 creatiby1.unicast.com +0.0.0.0 creative.adshuffle.com +0.0.0.0 creative.ak.facebook.com +0.0.0.0 creatives.livejasmin.com +0.0.0.0 creatives.rgadvert.com +0.0.0.0 creatrixads.com +0.0.0.0 crediblegfj.info +0.0.0.0 creditburner.blueadvertise.com +0.0.0.0 creditsoffer.blogspot.com +0.0.0.0 creview.adbureau.net +0.0.0.0 crosspixel.demdex.net +0.0.0.0 crowdgravity.com +0.0.0.0 crowdignite.com +0.0.0.0 c.ru.msn.com +0.0.0.0 crux.songline.com +0.0.0.0 crwdcntrl.net +0.0.0.0 c.se.msn.com +0.0.0.0 cserver.mii.instacontent.net +0.0.0.0 c.sg.msn.com +0.0.0.0 csh.actiondesk.com +0.0.0.0 csm.rotator.hadj7.adjuggler.net +0.0.0.0 cspix.media6degrees.com +0.0.0.0 cs.prd.msys.playstation.net +0.0.0.0 csr.onet.pl +0.0.0.0 ctbdev.net +0.0.0.0 c.th.msn.com +0.0.0.0 c.tr.msn.com +0.0.0.0 cts.channelintelligence.com +0.0.0.0 c.tw.msn.com +0.0.0.0 ctxtad.tribalfusion.com +0.0.0.0 c.uk.msn.com +0.0.0.0 cxoadfarm.dyndns.info +0.0.0.0 cxtad.specificmedia.com +0.0.0.0 cyber-incentives.com +0.0.0.0 cz8.clickzs.com +0.0.0.0 c.za.msn.com +0.0.0.0 cz.bbelements.com +0.0.0.0 d.101m3.com +0.0.0.0 d10.zedo.com +0.0.0.0 d11.zedo.com +0.0.0.0 d12.zedo.com +0.0.0.0 d14.zedo.com +0.0.0.0 d1.openx.org +0.0.0.0 d1ros97qkrwjf5.cloudfront.net +0.0.0.0 d1.zedo.com +0.0.0.0 d2.zedo.com +0.0.0.0 d3.zedo.com +0.0.0.0 d4.zedo.com +0.0.0.0 d5phz18u4wuww.cloudfront.net +0.0.0.0 d5.zedo.com +0.0.0.0 d6.c5.b0.a2.top.mail.ru +0.0.0.0 d6.zedo.com +0.0.0.0 d7.zedo.com +0.0.0.0 d8.zedo.com +0.0.0.0 d9.zedo.com +0.0.0.0 da.2000888.com +0.0.0.0 d.adnetxchange.com +0.0.0.0 d.adserve.com +0.0.0.0 dads.new.digg.com +0.0.0.0 d.ads.readwriteweb.com +0.0.0.0 d.agkn.com +0.0.0.0 daily-saver.com +0.0.0.0 darmowe-liczniki.info +0.0.0.0 dart.chron.com +0.0.0.0 data.flurry.com +0.0.0.0 date.ventivmedia.com +0.0.0.0 datingadvertising.com +0.0.0.0 db4.net-filter.com +0.0.0.0 dbbsrv.com +0.0.0.0 dc.sabela.com.pl +0.0.0.0 dctracking.com +0.0.0.0 de.adserver.yahoo.com +0.0.0.0 del1.phillyburbs.com +0.0.0.0 delb.mspaceads.com +0.0.0.0 delivery.adyea.com +0.0.0.0 delivery.trafficjunky.net +0.0.0.0 delivery.w00tads.com +0.0.0.0 delivery.way2traffic.com +0.0.0.0 demr.mspaceads.com +0.0.0.0 demr.opt.fimserve.com +0.0.0.0 derkeiler.com +0.0.0.0 desb.mspaceads.com +0.0.0.0 descargas2.tuvideogratis.com +0.0.0.0 designbloxlive.com +0.0.0.0 desk.mspaceads.com +0.0.0.0 desk.opt.fimserve.com +0.0.0.0 dev.adforum.com +0.0.0.0 devart.adbureau.net +0.0.0.0 devlp1.linkpulse.com +0.0.0.0 dev.sfbg.com +0.0.0.0 dgm2.com +0.0.0.0 dgmaustralia.com +0.0.0.0 dg.specificclick.net +0.0.0.0 dietoftoday.ca.pn #security risk/fake news# +0.0.0.0 diff3.smartadserver.com +0.0.0.0 dinoadserver1.roka.net +0.0.0.0 dinoadserver2.roka.net +0.0.0.0 directleads.com +0.0.0.0 directpowerrewards.com +0.0.0.0 directrev.cloudapp.net +0.0.0.0 dirtyrhino.com +0.0.0.0 discount-savings-more.com +0.0.0.0 discoverecommerce.tt.omtrdc.net +0.0.0.0 display.gestionpub.com +0.0.0.0 dist.belnk.com +0.0.0.0 divx.adbureau.net +0.0.0.0 djbanners.deadjournal.com +0.0.0.0 djugoogs.com +0.0.0.0 dk.adserver.yahoo.com +0.0.0.0 dl.ncbuy.com +0.0.0.0 dl-plugin.com +0.0.0.0 dlvr.readserver.net +0.0.0.0 dnads.directnic.com +0.0.0.0 dnps.com +0.0.0.0 dnse.linkpulse.com +0.0.0.0 dosugcz.biz +0.0.0.0 dot.wp.pl +0.0.0.0 downloadcdn.com +0.0.0.0 do-wn-lo-ad.com +0.0.0.0 downloads.larivieracasino.com +0.0.0.0 downloads.mytvandmovies.com +0.0.0.0 dqs001.adtech.de +0.0.0.0 dqs001.adtech.fr +0.0.0.0 dqs001.adtech.us +0.0.0.0 dra.amazon-adsystem.com +0.0.0.0 drowle.com +0.0.0.0 ds.contextweb.com +0.0.0.0 ds.onet.pl +0.0.0.0 ds.serving-sys.com +0.0.0.0 dt.linkpulse.com +0.0.0.0 dub.mobileads.msn.com +0.0.0.0 e0.extreme-dm.com +0.0.0.0 e1.addthis.com +0.0.0.0 e2.cdn.qnsr.com +0.0.0.0 e2.emediate.se +0.0.0.0 eads-adserving.com +0.0.0.0 ead.sharethis.com +0.0.0.0 earnmygift.com +0.0.0.0 earnpointsandgifts.com +0.0.0.0 e.as-eu.falkag.net +0.0.0.0 easyadservice.com +0.0.0.0 easyweb.tdcanadatrust.secureserver.host1.customer-identification-process.b88600d8.com +0.0.0.0 eatps.web.aol.com +0.0.0.0 eb.adbureau.net +0.0.0.0 eblastengine.upickem.net +0.0.0.0 ecomadserver.com +0.0.0.0 eddamedia.linkpulse.com +0.0.0.0 edge.bnmla.com +0.0.0.0 edge.quantserve.com +0.0.0.0 edirect.hotkeys.com +0.0.0.0 education-rewardpath.com +0.0.0.0 edu-offer.com +0.0.0.0 electronics-bonuspath.com +0.0.0.0 electronics-offer.net +0.0.0.0 electronicspresent.com +0.0.0.0 electronics-rewardpath.com +0.0.0.0 emailadvantagegroup.com +0.0.0.0 emailproductreview.com +0.0.0.0 emapadserver.com +0.0.0.0 emea-bidder.mathtag.com +0.0.0.0 engage.everyone.net +0.0.0.0 engage.speedera.net +0.0.0.0 engine2.adzerk.net +0.0.0.0 engine.4chan-ads.org +0.0.0.0 engine.adland.ru +0.0.0.0 engine.adzerk.net +0.0.0.0 engine.carbonads.com +0.0.0.0 engine.espace.netavenir.com +0.0.0.0 engine.influads.com +0.0.0.0 engine.rorer.ru +0.0.0.0 enirocode.adtlgc.com +0.0.0.0 enirodk.adtlgc.com +0.0.0.0 enn.advertserve.com +0.0.0.0 entertainment-rewardpath.com +0.0.0.0 entertainment-specials.com +0.0.0.0 es.adserver.yahoo.com +0.0.0.0 escape.insites.eu +0.0.0.0 espn.footprint.net +0.0.0.0 etad.telegraph.co.uk +0.0.0.0 etrk.asus.com +0.0.0.0 etype.adbureau.net +0.0.0.0 eu2.madsone.com +0.0.0.0 euniverseads.com +0.0.0.0 eu-pn4.adserver.yahoo.com +0.0.0.0 europe.adserver.yahoo.com +0.0.0.0 eu.xtms.net +0.0.0.0 eventtracker.videostrip.com +0.0.0.0 exclusivegiftcards.com +0.0.0.0 exits1.webquest.net +0.0.0.0 exits2.webquest.net +0.0.0.0 exponential.com +0.0.0.0 eyewonder.com +0.0.0.0 ezboard.bigbangmedia.com +0.0.0.0 falkag.net +0.0.0.0 family-offer.com +0.0.0.0 farm.plista.com +0.0.0.0 f.as-eu.falkag.net +0.0.0.0 fatcatrewards.com +0.0.0.0 fbcdn-creative-a.akamaihd.net +0.0.0.0 fbfreegifts.com +0.0.0.0 fbi.gov.id402037057-8235504608.d9680.com +0.0.0.0 fcg.casino770.com +0.0.0.0 fc.webmasterpro.de +0.0.0.0 fdimages.fairfax.com.au +0.0.0.0 feedads.googleadservices.com +0.0.0.0 feeds.videosz.com +0.0.0.0 feeds.weselltraffic.com +0.0.0.0 fei.pro-market.net +0.0.0.0 fe.lea.lycos.es +0.0.0.0 fhm.valueclick.net +0.0.0.0 fif49.info +0.0.0.0 files.adbrite.com +0.0.0.0 fin.adbureau.net +0.0.0.0 finance-offer.com +0.0.0.0 finanzmeldungen.com +0.0.0.0 finder.cox.net +0.0.0.0 fixbonus.com +0.0.0.0 floatingads.madisonavenue.com +0.0.0.0 floridat.app.ur.gcion.com +0.0.0.0 flowers-offer.com +0.0.0.0 fls-na.amazon.com +0.0.0.0 flu23.com +0.0.0.0 fmads.osdn.com +0.0.0.0 focusin.ads.targetnet.com +0.0.0.0 folloyu.com +0.0.0.0 food-drink-bonuspath.com +0.0.0.0 food-drink-rewardpath.com +0.0.0.0 foodmixeroffer.com +0.0.0.0 food-offer.com +0.0.0.0 foreignpolicy.advertserve.com +0.0.0.0 fp.uclo.net +0.0.0.0 fp.valueclick.com +0.0.0.0 fr.a2dfp.net +0.0.0.0 fr.adserver.yahoo.com +0.0.0.0 fr.classic.clickintext.net +0.0.0.0 freebiegb.co.uk +0.0.0.0 freecameraonus.com +0.0.0.0 freecameraprovider.com +0.0.0.0 freecamerasource.com +0.0.0.0 freecamerauk.co.uk +0.0.0.0 freecoolgift.com +0.0.0.0 freedesignerhandbagreviews.com +0.0.0.0 freedinnersource.com +0.0.0.0 freedvddept.com +0.0.0.0 freeelectronicscenter.com +0.0.0.0 freeelectronicsdepot.com +0.0.0.0 freeelectronicsonus.com +0.0.0.0 freeelectronicssource.com +0.0.0.0 freeentertainmentsource.com +0.0.0.0 freefoodprovider.com +0.0.0.0 freefoodsource.com +0.0.0.0 freefuelcard.com +0.0.0.0 freefuelcoupon.com +0.0.0.0 freegasonus.com +0.0.0.0 freegasprovider.com +0.0.0.0 free-gift-cards-now.com +0.0.0.0 freegiftcardsource.com +0.0.0.0 freegiftreward.com +0.0.0.0 free-gifts-comp.com +0.0.0.0 free.hotsocialz.com +0.0.0.0 freeipodnanouk.co.uk +0.0.0.0 freeipoduk.com +0.0.0.0 freeipoduk.co.uk +0.0.0.0 freelaptopgift.com +0.0.0.0 freelaptopnation.com +0.0.0.0 free-laptop-reward.com +0.0.0.0 freelaptopreward.com +0.0.0.0 freelaptopwebsites.com +0.0.0.0 freenation.com +0.0.0.0 freeoffers-toys.com +0.0.0.0 freepayasyougotopupuk.co.uk +0.0.0.0 freeplasmanation.com +0.0.0.0 freerestaurantprovider.com +0.0.0.0 freerestaurantsource.com +0.0.0.0 free-rewards.com-s.tv +0.0.0.0 freeshoppingprovider.com +0.0.0.0 freeshoppingsource.com +0.0.0.0 free.thesocialsexnetwork.com +0.0.0.0 freevideodownloadforpc.com +0.0.0.0 frontend-loadbalancer.meteorsolutions.com +0.0.0.0 fwdservice.com +0.0.0.0 fwmrm.net +0.0.0.0 g1.idg.pl +0.0.0.0 g2.gumgum.com +0.0.0.0 g3t4d5.madison.com +0.0.0.0 g4p.grt02.com +0.0.0.0 gadgeteer.pdamart.com +0.0.0.0 gam.adnxs.com +0.0.0.0 gameconsolerewards.com +0.0.0.0 games-toys-bonuspath.com +0.0.0.0 games-toys-free.com +0.0.0.0 games-toys-rewardpath.com +0.0.0.0 gate.hyperpaysys.com +0.0.0.0 gavzad.keenspot.com +0.0.0.0 gazeta.hit.gemius.pl +0.0.0.0 gazetteextra.advertserve.com +0.0.0.0 gbanners.hornymatches.com +0.0.0.0 gcads.osdn.com +0.0.0.0 gcdn.2mdn.net +0.0.0.0 gc.gcl.ru +0.0.0.0 gcir.gannett-tv.com +0.0.0.0 gcirm2.indystar.com +0.0.0.0 gcirm.argusleader.com +0.0.0.0 gcirm.argusleader.gcion.com +0.0.0.0 gcirm.battlecreekenquirer.com +0.0.0.0 gcirm.burlingtonfreepress.com +0.0.0.0 gcirm.centralohio.com +0.0.0.0 gcirm.centralohio.gcion.com +0.0.0.0 gcirm.cincinnati.com +0.0.0.0 gcirm.citizen-times.com +0.0.0.0 gcirm.clarionledger.com +0.0.0.0 gcirm.coloradoan.com +0.0.0.0 gcirm.courier-journal.com +0.0.0.0 gcirm.courierpostonline.com +0.0.0.0 gcirm.customcoupon.com +0.0.0.0 gcirm.dailyrecord.com +0.0.0.0 gcirm.delawareonline.com +0.0.0.0 gcirm.democratandchronicle.com +0.0.0.0 gcirm.desmoinesregister.com +0.0.0.0 gcirm.detnews.com +0.0.0.0 gcirm.dmp.gcion.com +0.0.0.0 gcirm.dmregister.com +0.0.0.0 gcirm.dnj.com +0.0.0.0 gcirm.flatoday.com +0.0.0.0 gcirm.gannettnetwork.com +0.0.0.0 gcirm.gannett-tv.com +0.0.0.0 gcirm.greatfallstribune.com +0.0.0.0 gcirm.greenvilleonline.com +0.0.0.0 gcirm.greenvilleonline.gcion.com +0.0.0.0 gcirm.honoluluadvertiser.gcion.com +0.0.0.0 gcirm.idahostatesman.com +0.0.0.0 gcirm.idehostatesman.com +0.0.0.0 gcirm.indystar.com +0.0.0.0 gcirm.injersey.com +0.0.0.0 gcirm.jacksonsun.com +0.0.0.0 gcirm.laregionalonline.com +0.0.0.0 gcirm.lsj.com +0.0.0.0 gcirm.montgomeryadvertiser.com +0.0.0.0 gcirm.muskogeephoenix.com +0.0.0.0 gcirm.newsleader.com +0.0.0.0 gcirm.news-press.com +0.0.0.0 gcirm.ozarksnow.com +0.0.0.0 gcirm.pensacolanewsjournal.com +0.0.0.0 gcirm.press-citizen.com +0.0.0.0 gcirm.pressconnects.com +0.0.0.0 gcirm.rgj.com +0.0.0.0 gcirm.sctimes.com +0.0.0.0 gcirm.stargazette.com +0.0.0.0 gcirm.statesmanjournal.com +0.0.0.0 gcirm.tallahassee.com +0.0.0.0 gcirm.tennessean.com +0.0.0.0 gcirm.thedailyjournal.com +0.0.0.0 gcirm.thedesertsun.com +0.0.0.0 gcirm.theithacajournal.com +0.0.0.0 gcirm.thejournalnews.com +0.0.0.0 gcirm.theolympian.com +0.0.0.0 gcirm.thespectrum.com +0.0.0.0 gcirm.tucson.com +0.0.0.0 gcirm.wisinfo.com +0.0.0.0 gde.adocean.pl +0.0.0.0 gdeee.hit.gemius.pl +0.0.0.0 gdelt.hit.gemius.pl +0.0.0.0 gdelv.hit.gemius.pl +0.0.0.0 gdyn.cnngo.com +0.0.0.0 gdyn.trutv.com +0.0.0.0 gemius.pl +0.0.0.0 geoads.osdn.com +0.0.0.0 geoloc11.geovisite.com +0.0.0.0 geo.precisionclick.com +0.0.0.0 getacool100.com +0.0.0.0 getacool500.com +0.0.0.0 getacoollaptop.com +0.0.0.0 getacooltv.com +0.0.0.0 getafreeiphone.org +0.0.0.0 getagiftonline.com +0.0.0.0 getmyfreebabystuff.com +0.0.0.0 getmyfreegear.com +0.0.0.0 getmyfreegiftcard.com +0.0.0.0 getmyfreelaptop.com +0.0.0.0 getmyfreelaptophere.com +0.0.0.0 getmyfreeplasma.com +0.0.0.0 getmylaptopfree.com +0.0.0.0 getmyplasmatv.com +0.0.0.0 getspecialgifts.com +0.0.0.0 getyour5kcredits0.blogspot.com +0.0.0.0 getyourfreecomputer.com +0.0.0.0 getyourfreetv.com +0.0.0.0 getyourgiftnow2.blogspot.com +0.0.0.0 getyourgiftnow3.blogspot.com +0.0.0.0 gg.adocean.pl +0.0.0.0 giftcardchallenge.com +0.0.0.0 giftcardsurveys.us.com +0.0.0.0 giftrewardzone.com +0.0.0.0 gifts-flowers-rewardpath.com +0.0.0.0 gimmethatreward.com +0.0.0.0 gingert.net +0.0.0.0 globalwebads.com +0.0.0.0 gmads.net +0.0.0.0 gm.preferences.com +0.0.0.0 go2.hit.gemius.pl +0.0.0.0 go.adee.bbelements.com +0.0.0.0 go.adlt.bbelements.com +0.0.0.0 go.adlv.bbelements.com +0.0.0.0 go.admulti.com +0.0.0.0 go.adnet.bbelements.com +0.0.0.0 go.arbo.bbelements.com +0.0.0.0 go.arbopl.bbelements.com +0.0.0.0 go.arboru.bbelements.com +0.0.0.0 go.bb007.bbelements.com +0.0.0.0 go.evolutionmedia.bbelements.com +0.0.0.0 go-free-gifts.com +0.0.0.0 gofreegifts.com +0.0.0.0 go.ihned.bbelements.com +0.0.0.0 go.intact.bbelements.com +0.0.0.0 go.lfstmedia.com +0.0.0.0 go.lotech.bbelements.com +0.0.0.0 goodsblock.marketgid.com +0.0.0.0 goody-garage.com +0.0.0.0 go.pl.bbelements.com +0.0.0.0 got2goshop.com +0.0.0.0 goto.trafficmultiplier.com +0.0.0.0 gozing.directtrack.com +0.0.0.0 grabbit-rabbit.com +0.0.0.0 graphics.adultfriendfinder.com +0.0.0.0 graphics.pop6.com +0.0.0.0 gratkapl.adocean.pl +0.0.0.0 gravitron.chron.com +0.0.0.0 greasypalm.com +0.0.0.0 grfx.mp3.com +0.0.0.0 groupon.pl +0.0.0.0 grz67.com +0.0.0.0 gs1.idsales.co.uk +0.0.0.0 gserv.cneteu.net +0.0.0.0 gspro.hit.gemius.pl +0.0.0.0 g.thinktarget.com +0.0.0.0 guiaconsumidor.com +0.0.0.0 guide2poker.com +0.0.0.0 guptamedianetwork.com +0.0.0.0 guru.sitescout.netdna-cdn.com +0.0.0.0 gwallet.com +0.0.0.0 gx-in-f109.1e100.net +0.0.0.0 h-afnetwww.adshuffle.com +0.0.0.0 halfords.ukrpts.net +0.0.0.0 happydiscountspecials.com +0.0.0.0 harvest176.adgardener.com +0.0.0.0 harvest284.adgardener.com +0.0.0.0 harvest285.adgardener.com +0.0.0.0 harvest.adgardener.com +0.0.0.0 hathor.eztonez.com +0.0.0.0 haynet.adbureau.net +0.0.0.0 hbads.eboz.com +0.0.0.0 hbadz.eboz.com +0.0.0.0 healthbeautyncs.com +0.0.0.0 health-beauty-rewardpath.com +0.0.0.0 health-beauty-savingblvd.com +0.0.0.0 healthclicks.co.uk +0.0.0.0 hebdotop.com +0.0.0.0 help.adtech.de +0.0.0.0 help.adtech.fr +0.0.0.0 help.adtech.us +0.0.0.0 helpint.mywebsearch.com +0.0.0.0 hightrafficads.com +0.0.0.0 himediads.com +0.0.0.0 hit4.hotlog.ru +0.0.0.0 hk.adserver.yahoo.com +0.0.0.0 hlcc.ca +0.0.0.0 holiday-gift-offers.com +0.0.0.0 holidayproductpromo.com +0.0.0.0 holidayshoppingrewards.com +0.0.0.0 home4bizstart.ru +0.0.0.0 homeelectronicproducts.com +0.0.0.0 home-garden-premiumblvd.com +0.0.0.0 home-garden-rewardempire.com +0.0.0.0 home-garden-rewardpath.com +0.0.0.0 homeimprovementonus.com +0.0.0.0 honolulu.app.ur.gcion.com +0.0.0.0 hooqy.com +0.0.0.0 host207.ewtn.com +0.0.0.0 hostedaje14.thruport.com +0.0.0.0 hosting.adjug.com +0.0.0.0 hot-daily-deal.com +0.0.0.0 hotgiftzone.com +0.0.0.0 hot-product-hangout.com +0.0.0.0 hpad.www.infoseek.co.jp +0.0.0.0 h.ppjol.com +0.0.0.0 htmlads.ru +0.0.0.0 html.centralmediaserver.com +0.0.0.0 htmlwww.youfck.com +0.0.0.0 http300.content.ru4.com +0.0.0.0 httpads.com +0.0.0.0 httpwwwadserver.com +0.0.0.0 hub.com.pl +0.0.0.0 huiwiw.hit.gemius.pl +0.0.0.0 huntingtonbank.tt.omtrdc.net +0.0.0.0 huomdgde.adocean.pl +0.0.0.0 hyperion.adtech.de +0.0.0.0 hyperion.adtech.fr +0.0.0.0 hyperion.adtech.us +0.0.0.0 i1.teaser-goods.ru +0.0.0.0 iacas.adbureau.net +0.0.0.0 iad.anm.co.uk +0.0.0.0 iadc.qwapi.com +#0.0.0.0 iadsdk.apple.com #may interfere with iTunes radio +0.0.0.0 ib.adnxs.com +0.0.0.0 ibis.lgappstv.com +0.0.0.0 i.blogads.com +0.0.0.0 i.casalemedia.com +0.0.0.0 icon.clickthru.net +0.0.0.0 id11938.luxup.ru +0.0.0.0 id5576.al21.luxup.ru +0.0.0.0 idearc.tt.omtrdc.net +0.0.0.0 idpix.media6degrees.com +0.0.0.0 ieee.adbureau.net +0.0.0.0 if.bbanner.it +0.0.0.0 iftarvakitleri.net +0.0.0.0 ih2.gamecopyworld.com +0.0.0.0 i.hotkeys.com +0.0.0.0 i.interia.pl +0.0.0.0 i.laih.com +0.0.0.0 ilinks.industrybrains.com +0.0.0.0 im.adtech.de +0.0.0.0 image2.pubmatic.com +0.0.0.0 imageads.canoe.ca +0.0.0.0 imagec08.247realmedia.com +0.0.0.0 imagec12.247realmedia.com +0.0.0.0 imagec14.247realmedia.com +0.0.0.0 imagecache2.allposters.com +0.0.0.0 imageceu1.247realmedia.com +0.0.0.0 image.click.livedoor.com +0.0.0.0 image.i1img.com +0.0.0.0 image.linkexchange.com +0.0.0.0 images2.laih.com +0.0.0.0 images3.linkwithin.com +0.0.0.0 images.ads.fairfax.com.au +0.0.0.0 images.blogads.com +0.0.0.0 images.bluetime.com +0.0.0.0 images-cdn.azoogleads.com +0.0.0.0 images.clickfinders.com +0.0.0.0 images.conduit-banners.com +0.0.0.0 images.cybereps.com +0.0.0.0 images.directtrack.com +0.0.0.0 images.emapadserver.com +0.0.0.0 imageserv.adtech.de +0.0.0.0 imageserv.adtech.fr +0.0.0.0 imageserv.adtech.us +0.0.0.0 imageserver1.thruport.com +0.0.0.0 images.jambocast.com +0.0.0.0 images.linkwithin.com +0.0.0.0 images.mbuyu.nl +0.0.0.0 images.netcomvad.com +0.0.0.0 images.newsx.cc +0.0.0.0 images.people2people.com +0.0.0.0 images.primaryads.com +0.0.0.0 images.sexlist.com +0.0.0.0 images.steamray.com +0.0.0.0 images.trafficmp.com +0.0.0.0 im.banner.t-online.de +0.0.0.0 i.media.cz +0.0.0.0 img0.ru.redtram.com +0.0.0.0 img1.ru.redtram.com +0.0.0.0 img2.ru.redtram.com +0.0.0.0 img4.cdn.adjuggler.com +0.0.0.0 img-a2.ak.imagevz.net +0.0.0.0 img.blogads.com +0.0.0.0 img-cdn.mediaplex.com +0.0.0.0 img.directtrack.com +0.0.0.0 imgg.dt00.net +0.0.0.0 imgg.marketgid.com +0.0.0.0 img.layer-ads.de +0.0.0.0 img.marketgid.com +0.0.0.0 imgn.dt00.net +0.0.0.0 imgn.dt07.com +0.0.0.0 imgn.marketgid.com +0.0.0.0 imgserv.adbutler.com +0.0.0.0 img.sn00.net +0.0.0.0 img.soulmate.com +0.0.0.0 img.xnxx.com +0.0.0.0 im.of.pl +0.0.0.0 impact.cossette-webpact.com +0.0.0.0 impbe.tradedoubler.com +0.0.0.0 imp.partner2profit.com +0.0.0.0 imppl.tradedoubler.com +0.0.0.0 impressionaffiliate.com +0.0.0.0 impressionaffiliate.mobi +0.0.0.0 impressionlead.com +0.0.0.0 impressionperformance.biz +0.0.0.0 imserv001.adtech.de +0.0.0.0 imserv001.adtech.fr +0.0.0.0 imserv001.adtech.us +0.0.0.0 imserv002.adtech.de +0.0.0.0 imserv002.adtech.fr +0.0.0.0 imserv002.adtech.us +0.0.0.0 imserv003.adtech.de +0.0.0.0 imserv003.adtech.fr +0.0.0.0 imserv003.adtech.us +0.0.0.0 imserv004.adtech.de +0.0.0.0 imserv004.adtech.fr +0.0.0.0 imserv004.adtech.us +0.0.0.0 imserv005.adtech.de +0.0.0.0 imserv005.adtech.fr +0.0.0.0 imserv005.adtech.us +0.0.0.0 imserv006.adtech.de +0.0.0.0 imserv006.adtech.fr +0.0.0.0 imserv006.adtech.us +0.0.0.0 imserv00x.adtech.de +0.0.0.0 imserv00x.adtech.fr +0.0.0.0 imserv00x.adtech.us +0.0.0.0 imssl01.adtech.de +0.0.0.0 imssl01.adtech.fr +0.0.0.0 imssl01.adtech.us +0.0.0.0 im.xo.pl +0.0.0.0 in.adserver.yahoo.com +0.0.0.0 incentivegateway.com +0.0.0.0 incentiverewardcenter.com +0.0.0.0 incentive-scene.com +0.0.0.0 indexhu.adocean.pl +0.0.0.0 infinite-ads.com +0.0.0.0 inklineglobal.com +0.0.0.0 inl.adbureau.net +0.0.0.0 input.insights.gravity.com +0.0.0.0 insightxe.pittsburghlive.com +0.0.0.0 insightxe.vtsgonline.com +0.0.0.0 ins-offer.com +0.0.0.0 installer.zutrack.com +0.0.0.0 insurance-rewardpath.com +0.0.0.0 intela.com +0.0.0.0 intelliads.com +0.0.0.0 internet.billboard.cz +0.0.0.0 intnet-offer.com +0.0.0.0 intrack.pl +0.0.0.0 invitefashion.com +0.0.0.0 ipacc1.adtech.de +0.0.0.0 ipacc1.adtech.fr +0.0.0.0 ipacc1.adtech.us +0.0.0.0 ipad2free4u.com +0.0.0.0 i.pcp001.com +0.0.0.0 ipdata.adtech.de +0.0.0.0 ipdata.adtech.fr +0.0.0.0 ipdata.adtech.us +0.0.0.0 iq001.adtech.de +0.0.0.0 iq001.adtech.fr +0.0.0.0 iq001.adtech.us +0.0.0.0 i.qitrck.com +0.0.0.0 is.casalemedia.com +0.0.0.0 i.securecontactinfo.com +0.0.0.0 isg01.casalemedia.com +0.0.0.0 isg02.casalemedia.com +0.0.0.0 isg03.casalemedia.com +0.0.0.0 isg04.casalemedia.com +0.0.0.0 isg05.casalemedia.com +0.0.0.0 isg06.casalemedia.com +0.0.0.0 isg07.casalemedia.com +0.0.0.0 isg08.casalemedia.com +0.0.0.0 isg09.casalemedia.com +0.0.0.0 i.simpli.fi +0.0.0.0 it.adserver.yahoo.com +0.0.0.0 i.total-media.net +0.0.0.0 itrackerpro.com +0.0.0.0 i.trkjmp.com +0.0.0.0 itsfree123.com +0.0.0.0 itxt.vibrantmedia.com +0.0.0.0 iwantmyfreecash.com +0.0.0.0 iwantmy-freelaptop.com +0.0.0.0 iwantmyfree-laptop.com +0.0.0.0 iwantmyfreelaptop.com +0.0.0.0 iwantmygiftcard.com +0.0.0.0 jambocast.com +0.0.0.0 jb9clfifs6.s.ad6media.fr +0.0.0.0 jcarter.spinbox.net +0.0.0.0 j.clickdensity.com +0.0.0.0 jcrew.tt.omtrdc.net +0.0.0.0 jersey-offer.com +0.0.0.0 jgedads.cjt.net +0.0.0.0 jh.revolvermaps.com +0.0.0.0 jivox.com +0.0.0.0 jl29jd25sm24mc29.com +0.0.0.0 jlinks.industrybrains.com +0.0.0.0 jmn.jangonetwork.com +0.0.0.0 join1.winhundred.com +0.0.0.0 js1.bloggerads.net +0.0.0.0 js77.neodatagroup.com +0.0.0.0 js.adlink.net +0.0.0.0 js.admngr.com +0.0.0.0 js.adscale.de +0.0.0.0 js.adserverpub.com +0.0.0.0 js.adsonar.com +0.0.0.0 jsc.dt07.net +0.0.0.0 js.goods.redtram.com +0.0.0.0 js.himediads.com +0.0.0.0 js.hotkeys.com +0.0.0.0 jsn.dt07.net +0.0.0.0 js.ru.redtram.com +0.0.0.0 js.selectornews.com +0.0.0.0 js.smi2.ru +0.0.0.0 js.tongji.linezing.com +0.0.0.0 js.zevents.com +0.0.0.0 judo.salon.com +0.0.0.0 juggler.inetinteractive.com +0.0.0.0 justwebads.com +0.0.0.0 jxliu.com +0.0.0.0 k5ads.osdn.com +0.0.0.0 kaartenhuis.nl.site-id.nl +0.0.0.0 kansas.valueclick.com +0.0.0.0 katu.adbureau.net +0.0.0.0 kazaa.adserver.co.il +0.0.0.0 kermit.macnn.com +0.0.0.0 kestrel.ospreymedialp.com +0.0.0.0 keys.dmtracker.com +0.0.0.0 keywordblocks.com +0.0.0.0 keywords.adtlgc.com +0.0.0.0 kitaramarketplace.com +0.0.0.0 kitaramedia.com +0.0.0.0 kitaratrk.com +0.0.0.0 kithrup.matchlogic.com +0.0.0.0 kixer.com +0.0.0.0 klikk.linkpulse.com +0.0.0.0 klikmoney.net +0.0.0.0 kliksaya.com +0.0.0.0 klipads.dvlabs.com +0.0.0.0 klipmart.dvlabs.com +0.0.0.0 klipmart.forbes.com +0.0.0.0 kmdl101.com +0.0.0.0 knc.lv +0.0.0.0 knight.economist.com +0.0.0.0 kona2.kontera.com +0.0.0.0 kona3.kontera.com +0.0.0.0 kona4.kontera.com +0.0.0.0 kona5.kontera.com +0.0.0.0 kona6.kontera.com +0.0.0.0 kona7.kontera.com +0.0.0.0 kona8.kontera.com +0.0.0.0 kona.kontera.com +0.0.0.0 kontera.com +0.0.0.0 kreaffiliation.com +0.0.0.0 kropka.onet.pl +0.0.0.0 kuhdi.com +0.0.0.0 l.5min.com +0.0.0.0 ladyclicks.ru +0.0.0.0 lanzar.publicidadweb.com +0.0.0.0 laptopreportcard.com +0.0.0.0 laptoprewards.com +0.0.0.0 laptoprewardsgroup.com +0.0.0.0 laptoprewardszone.com +0.0.0.0 larivieracasino.com +0.0.0.0 lasthr.info +0.0.0.0 lastmeasure.zoy.org +0.0.0.0 launch.adserver.yahoo.com +0.0.0.0 layer-ads.de +0.0.0.0 lb-adserver.ig.com.br +0.0.0.0 ld1.criteo.com +0.0.0.0 ld2.criteo.com +0.0.0.0 ldglob01.adtech.de +0.0.0.0 ldglob01.adtech.fr +0.0.0.0 ldglob01.adtech.us +0.0.0.0 ldglob02.adtech.de +0.0.0.0 ldglob02.adtech.fr +0.0.0.0 ldglob02.adtech.us +0.0.0.0 ldimage01.adtech.de +0.0.0.0 ldimage01.adtech.fr +0.0.0.0 ldimage01.adtech.us +0.0.0.0 ldimage02.adtech.de +0.0.0.0 ldimage02.adtech.fr +0.0.0.0 ldimage02.adtech.us +0.0.0.0 ldserv01.adtech.de +0.0.0.0 ldserv01.adtech.fr +0.0.0.0 ldserv01.adtech.us +0.0.0.0 ldserv02.adtech.de +0.0.0.0 ldserv02.adtech.fr +0.0.0.0 ldserv02.adtech.us +0.0.0.0 le1er.net +0.0.0.0 leadback.advertising.com +0.0.0.0 leader.linkexchange.com +0.0.0.0 lead.program3.com +0.0.0.0 leadsynaptic.go2jump.org +0.0.0.0 learning-offer.com +0.0.0.0 legal-rewardpath.com +0.0.0.0 leisure-offer.com +0.0.0.0 lg.brandreachsys.com +0.0.0.0 liberty.gedads.com +0.0.0.0 link2me.ru +0.0.0.0 link4ads.com +0.0.0.0 linktracker.angelfire.com +0.0.0.0 linuxpark.adtech.de +0.0.0.0 linuxpark.adtech.fr +0.0.0.0 linuxpark.adtech.us +0.0.0.0 liquidad.narrowcastmedia.com +0.0.0.0 live-cams-1.livejasmin.com +0.0.0.0 livingnet.adtech.de +0.0.0.0 ll.atdmt.com +0.0.0.0 l.linkpulse.com +0.0.0.0 lnads.osdn.com +0.0.0.0 load.exelator.com +0.0.0.0 load.focalex.com +0.0.0.0 loading321.com +0.0.0.0 loadm.exelator.com +0.0.0.0 local.promoisland.net +0.0.0.0 logc252.xiti.com +0.0.0.0 log.feedjit.com +0.0.0.0 login.linkpulse.com +0.0.0.0 log.olark.com +0.0.0.0 looksmartcollect.247realmedia.com +0.0.0.0 louisvil.app.ur.gcion.com +0.0.0.0 louisvil.ur.gcion.com +0.0.0.0 lp1.linkpulse.com +0.0.0.0 lp4.linkpulse.com +0.0.0.0 lpcloudsvr405.com +0.0.0.0 lstats.qip.ru +0.0.0.0 lt.andomedia.com +0.0.0.0 lt.angelfire.com +0.0.0.0 lucky-day-uk.com +0.0.0.0 luxup.ru +0.0.0.0 lw1.gamecopyworld.com +0.0.0.0 lw2.gamecopyworld.com +0.0.0.0 lycos.247realmedia.com +0.0.0.0 l.yieldmanager.net +0.0.0.0 m1.emea.2mdn.net.edgesuite.net +0.0.0.0 m2.sexgarantie.nl +0.0.0.0 m3.2mdn.net +0.0.0.0 macaddictads.snv.futurenet.com +0.0.0.0 macads.net +0.0.0.0 mackeeperapp1.zeobit.com +0.0.0.0 mad2.brandreachsys.com +0.0.0.0 m.adbridge.de +0.0.0.0 mads.aol.com +0.0.0.0 mads.cnet.com +0.0.0.0 mail.radar.imgsmail.ru +0.0.0.0 manage001.adtech.de +0.0.0.0 manage001.adtech.fr +0.0.0.0 manage001.adtech.us +0.0.0.0 manager.rovion.com +0.0.0.0 manuel.theonion.com +0.0.0.0 marketgid.com +0.0.0.0 marketing.888.com +0.0.0.0 marketing-rewardpath.com +0.0.0.0 marriottinternationa.tt.omtrdc.net +0.0.0.0 mastertracks.be +0.0.0.0 matomy.adk2.co +0.0.0.0 matrix.mediavantage.de +0.0.0.0 maxadserver.corusradionetwork.com +0.0.0.0 maxads.ruralpress.com +0.0.0.0 maxbounty.com +0.0.0.0 maximumpcads.imaginemedia.com +0.0.0.0 maxmedia.sgaonline.com +0.0.0.0 maxserving.com +0.0.0.0 mb01.com +0.0.0.0 mbox2.offermatica.com +0.0.0.0 mbox9.offermatica.com +0.0.0.0 mds.centrport.net +0.0.0.0 media2021.videostrip.com +0.0.0.0 media2.adshuffle.com +0.0.0.0 media2.legacy.com +0.0.0.0 media2.travelzoo.com +0.0.0.0 media4021.videostrip.com #http://media4021.videostrip.com/dev8/0/000/449/0000449408.mp4 +0.0.0.0 media5021.videostrip.com #http://media5021.videostrip.com/dev14/0/000/363/0000363146.mp4 +0.0.0.0 media6021.videostrip.com +0.0.0.0 media6.sitebrand.com +0.0.0.0 media.888.com +0.0.0.0 media.adcentriconline.com +0.0.0.0 media.adrcdn.com +0.0.0.0 media.adrevolver.com +0.0.0.0 media.adrime.com +0.0.0.0 media.adshadow.net +0.0.0.0 media.b.lead.program3.com +0.0.0.0 media.bonnint.net +0.0.0.0 mediacharger.com +0.0.0.0 media.contextweb.com +0.0.0.0 media.elb-kind.de +0.0.0.0 media.espace-plus.net +0.0.0.0 media.fairlink.ru +0.0.0.0 mediafr.247realmedia.com +0.0.0.0 media.funpic.de +0.0.0.0 medialand.relax.ru +0.0.0.0 media.markethealth.com +0.0.0.0 media.naked.com +0.0.0.0 media.nk-net.pl +0.0.0.0 media.ontarionorth.com +0.0.0.0 media.popuptraffic.com +0.0.0.0 mediapst.adbureau.net +0.0.0.0 mediapst-images.adbureau.net +0.0.0.0 mediative.ca +0.0.0.0 mediative.com +0.0.0.0 media.trafficfactory.biz +0.0.0.0 media.trafficjunky.net +0.0.0.0 mediauk.247realmedia.com +0.0.0.0 media.ventivmedia.com +0.0.0.0 media.viwii.net +0.0.0.0 medical-offer.com +0.0.0.0 medical-rewardpath.com +0.0.0.0 medleyads.com +0.0.0.0 medrx.sensis.com.au +0.0.0.0 megapanel.gem.pl +0.0.0.0 mercury.bravenet.com +0.0.0.0 messagent.duvalguillaume.com +0.0.0.0 messagia.adcentric.proximi-t.com +0.0.0.0 meter-svc.nytimes.com +0.0.0.0 metrics.natmags.co.uk +0.0.0.0 metrics.sfr.fr +0.0.0.0 metrics.target.com +0.0.0.0 m.fr.a2dfp.net +0.0.0.0 m.friendlyduck.com +0.0.0.0 mf.sitescout.com +0.0.0.0 mg.dt00.net +0.0.0.0 mgid.com +0.0.0.0 mhlnk.com +0.0.0.0 mi.adinterax.com +0.0.0.0 microsof.wemfbox.ch +0.0.0.0 mightymagoo.com +0.0.0.0 mii-image.adjuggler.com +0.0.0.0 mini.videostrip.com +0.0.0.0 mirror.pointroll.com +0.0.0.0 mjxads.internet.com +0.0.0.0 mjx.ads.nwsource.com +0.0.0.0 mklik.gazeta.pl +0.0.0.0 mktg-offer.com +0.0.0.0 mlntracker.com +0.0.0.0 mm.admob.com +0.0.0.0 mm.chitika.net +0.0.0.0 mob.adwhirl.com +0.0.0.0 mobileads.msn.com +0.0.0.0 mobile.juicyads.com +0.0.0.0 mobularity.com +0.0.0.0 mochibot.com +0.0.0.0 mojofarm.mediaplex.com +0.0.0.0 moneyraid.com +0.0.0.0 monstersandcritics.advertserve.com +0.0.0.0 morefreecamsecrets.com +0.0.0.0 morevisits.info +0.0.0.0 motd.pinion.gg +0.0.0.0 movieads.imgs.sapo.pt +0.0.0.0 mp3playersource.com +0.0.0.0 mp.tscapeplay.com +0.0.0.0 msn.allyes.com +0.0.0.0 msnbe-hp.metriweb.be +0.0.0.0 msn-cdn.effectivemeasure.net +0.0.0.0 msn.oewabox.at +0.0.0.0 msn.tns-cs.net +0.0.0.0 msn.uvwbox.de +0.0.0.0 msn.wrating.com +0.0.0.0 mt58.mtree.com +0.0.0.0 m.tribalfusion.com +0.0.0.0 mu-in-f167.1e100.net +0.0.0.0 multi.xnxx.com +0.0.0.0 mvonline.com +0.0.0.0 mx.adserver.yahoo.com +0.0.0.0 myao.adocean.pl +0.0.0.0 my.blueadvertise.com +0.0.0.0 mycashback.co.uk +0.0.0.0 mycelloffer.com +0.0.0.0 mychoicerewards.com +0.0.0.0 myexclusiverewards.com +0.0.0.0 myfreedinner.com +0.0.0.0 myfreegifts.co.uk +0.0.0.0 myfreemp3player.com +0.0.0.0 mygiftcardcenter.com +0.0.0.0 mygiftresource.com +0.0.0.0 mygreatrewards.com +0.0.0.0 myoffertracking.com +0.0.0.0 my-reward-channel.com +0.0.0.0 my-rewardsvault.com +0.0.0.0 myseostats.com +0.0.0.0 myusersonline.com +0.0.0.0 myyearbookdigital.checkm8.com +0.0.0.0 n4g.us.intellitxt.com +0.0.0.0 n4p.ru.redtram.com +0.0.0.0 nationalissuepanel.com +0.0.0.0 nationalpost.adperfect.com +0.0.0.0 nationalsurveypanel.com +0.0.0.0 nbads.com +0.0.0.0 nbc.adbureau.net +0.0.0.0 nbimg.dt00.net +0.0.0.0 nb.netbreak.com.au +0.0.0.0 nc.ru.redtram.com +0.0.0.0 nctracking.com +0.0.0.0 nd1.gamecopyworld.com +0.0.0.0 nearbyad.com +0.0.0.0 needadvertising.com +0.0.0.0 netads.hotwired.com +0.0.0.0 netadsrv.iworld.com +0.0.0.0 netads.sohu.com +0.0.0.0 netcomm.spinbox.net +0.0.0.0 netpalnow.com +0.0.0.0 netshelter.adtrix.com +0.0.0.0 netspiderads2.indiatimes.com +0.0.0.0 netsponsors.com +0.0.0.0 networkads.net +0.0.0.0 network-ca.247realmedia.com +0.0.0.0 network.realmedia.com +0.0.0.0 network.realtechnetwork.net +0.0.0.0 newads.cmpnet.com +0.0.0.0 newadserver.interfree.it +0.0.0.0 new-ads.eurogamer.net +0.0.0.0 newbs.hutz.co.il +0.0.0.0 news6health.com +0.0.0.0 newsblock.marketgid.com +0.0.0.0 new.smartcontext.pl +0.0.0.0 newssourceoftoday.com #security risk/fake news# +0.0.0.0 newt1.adultadworld.com +0.0.0.0 newt1.adultworld.com +0.0.0.0 ng3.ads.warnerbros.com +0.0.0.0 ngads.smartage.com +0.0.0.0 nitrous.exitfuel.com +0.0.0.0 nitrous.internetfuel.com +0.0.0.0 nivendas.net +0.0.0.0 nkcache.brandreachsys.com +0.0.0.0 nl.adserver.yahoo.com +0.0.0.0 no.adserver.yahoo.com +0.0.0.0 nospartenaires.com +0.0.0.0 nothing-but-value.com +0.0.0.0 novafinanza.com +0.0.0.0 novem.onet.pl +0.0.0.0 nrads.1host.co.il +0.0.0.0 nrkno.linkpulse.com +0.0.0.0 ns1.lalibco.com +0.0.0.0 ns1.primeinteractive.net +0.0.0.0 ns2.hitbox.com +0.0.0.0 ns2.lalibco.com +0.0.0.0 ns2.primeinteractive.net +0.0.0.0 nsads4.us.publicus.com +0.0.0.0 nsads.hotwired.com +0.0.0.0 nsads.us.publicus.com +0.0.0.0 nspmotion.com +0.0.0.0 ns-vip1.hitbox.com +0.0.0.0 ns-vip2.hitbox.com +0.0.0.0 ns-vip3.hitbox.com +0.0.0.0 ntbanner.digitalriver.com +0.0.0.0 nx-adv0005.247realmedia.com +0.0.0.0 nxs.kidcolez.cn +0.0.0.0 nxtscrn.adbureau.net +0.0.0.0 nysubwayoffer.com +0.0.0.0 nytadvertising.nytimes.com +0.0.0.0 o0.winfuture.de +0.0.0.0 o1.qnsr.com +0.0.0.0 o2.eyereturn.com +0.0.0.0 oads.cracked.com +0.0.0.0 oamsrhads.us.publicus.com +0.0.0.0 oas-1.rmuk.co.uk +0.0.0.0 oasads.whitepages.com +0.0.0.0 oasc02023.247realmedia.com +0.0.0.0 oasc02.247realmedia.com +0.0.0.0 oasc03.247realmedia.com +0.0.0.0 oasc04.247.realmedia.com +0.0.0.0 oasc05050.247realmedia.com +0.0.0.0 oasc05.247realmedia.com +0.0.0.0 oasc16.247realmedia.com +0.0.0.0 oascenral.phoenixnewtimes.com +0.0.0.0 oascentral.videodome.com +0.0.0.0 oas.dn.se +0.0.0.0 oas-eu.247realmedia.com +0.0.0.0 oas.heise.de +0.0.0.0 oasis2.advfn.com +0.0.0.0 oasis.411affiliates.ca +0.0.0.0 oasis.nysun.com +0.0.0.0 oasis.promon.cz +0.0.0.0 oasis.realbeer.com +0.0.0.0 oasis.zmh.zope.com +0.0.0.0 oasis.zmh.zope.net +0.0.0.0 oasn03.247realmedia.com +0.0.0.0 oassis.zmh.zope.com +0.0.0.0 objects.abcvisiteurs.com +0.0.0.0 objects.designbloxlive.com +0.0.0.0 obozua.adocean.pl +0.0.0.0 observer.advertserve.com +0.0.0.0 obs.nnm2.ru +0.0.0.0 offers.impower.com +0.0.0.0 offerx.co.uk +0.0.0.0 oinadserve.com +0.0.0.0 old-darkroast.adknowledge.com +0.0.0.0 ometrics.warnerbros.com +0.0.0.0 onclickads.net +0.0.0.0 online1.webcams.com +0.0.0.0 onlineads.magicvalley.com +0.0.0.0 onlinebestoffers.net +0.0.0.0 onocollect.247realmedia.com +0.0.0.0 open.4info.net +0.0.0.0 openadext.tf1.fr +0.0.0.0 openad.infobel.com +0.0.0.0 openads.dimcab.com +0.0.0.0 openads.friendfinder.com +0.0.0.0 openads.nightlifemagazine.ca +0.0.0.0 openads.smithmag.net +0.0.0.0 openads.zeads.com +0.0.0.0 openad.travelnow.com +0.0.0.0 opentable.tt.omtrdc.net +0.0.0.0 openx2.fotoflexer.com +0.0.0.0 openx.adfactor.nl +0.0.0.0 openx.coolconcepts.nl +0.0.0.0 openx.shinyads.com +0.0.0.0 openxxx.viragemedia.com +0.0.0.0 optimized-by.rubiconproject.com +0.0.0.0 optimized.by.vitalads.net +0.0.0.0 optimize.indieclick.com +0.0.0.0 optimzedby.rmxads.com +0.0.0.0 orange.weborama.fr +0.0.0.0 ordie.adbureau.net +0.0.0.0 origin.chron.com +0.0.0.0 out.popads.net +0.0.0.0 overflow.adsoftware.com +0.0.0.0 overlay.ringtonematcher.com +0.0.0.0 overstock.tt.omtrdc.net +0.0.0.0 ox-d.hbr.org +0.0.0.0 ox-d.hulkshare.com +0.0.0.0 ox-d.hypeads.org +0.0.0.0 ox-d.zenoviagroup.com +0.0.0.0 ox.eurogamer.net +0.0.0.0 ox-i.zenoviagroup.com +0.0.0.0 ozonemedia.adbureau.net +0.0.0.0 oz.valueclick.com +0.0.0.0 oz.valueclick.ne.jp +0.0.0.0 p0rnuha.com +0.0.0.0 p1.adhitzads.com +0.0.0.0 pagead1.googlesyndication.com +0.0.0.0 pagead2.googlesyndication.com +0.0.0.0 pagead3.googlesyndication.com +0.0.0.0 pagead.googlesyndication.com +0.0.0.0 pages.etology.com +0.0.0.0 paime.com +0.0.0.0 panel.adtify.pl +0.0.0.0 paperg.com +0.0.0.0 partner01.oingo.com +0.0.0.0 partner02.oingo.com +0.0.0.0 partner03.oingo.com +0.0.0.0 partner.ah-ha.com +0.0.0.0 partner.ceneo.pl +0.0.0.0 partner.join.com.ua +0.0.0.0 partner.magna.ru +0.0.0.0 partner.pobieraczek.pl +0.0.0.0 partners.sprintrade.com +0.0.0.0 partners.webmasterplan.com +0.0.0.0 partner.wapacz.pl +0.0.0.0 partner.wapster.pl +0.0.0.0 pathforpoints.com +0.0.0.0 paulsnetwork.com +0.0.0.0 pbid.pro-market.net +0.0.0.0 pb.tynt.com +0.0.0.0 pcads.ru +0.0.0.0 pei-ads.playboy.com +0.0.0.0 people-choice-sites.com +0.0.0.0 personalcare-offer.com +0.0.0.0 personalcashbailout.com +0.0.0.0 pg2.solution.weborama.fr +0.0.0.0 ph-ad01.focalink.com +0.0.0.0 ph-ad02.focalink.com +0.0.0.0 ph-ad03.focalink.com +0.0.0.0 ph-ad04.focalink.com +0.0.0.0 ph-ad05.focalink.com +0.0.0.0 ph-ad06.focalink.com +0.0.0.0 ph-ad07.focalink.com +0.0.0.0 ph-ad08.focalink.com +0.0.0.0 ph-ad09.focalink.com +0.0.0.0 ph-ad10.focalink.com +0.0.0.0 ph-ad11.focalink.com +0.0.0.0 ph-ad12.focalink.com +0.0.0.0 ph-ad13.focalink.com +0.0.0.0 ph-ad14.focalink.com +0.0.0.0 ph-ad15.focalink.com +0.0.0.0 ph-ad16.focalink.com +0.0.0.0 ph-ad17.focalink.com +0.0.0.0 ph-ad18.focalink.com +0.0.0.0 ph-ad19.focalink.com +0.0.0.0 ph-ad20.focalink.com +0.0.0.0 ph-ad21.focalink.com +0.0.0.0 ph-cdn.effectivemeasure.net +0.0.0.0 phoenixads.co.in +0.0.0.0 photobucket.adnxs.com +0.0.0.0 photos0.pop6.com +0.0.0.0 photos1.pop6.com +0.0.0.0 photos2.pop6.com +0.0.0.0 photos3.pop6.com +0.0.0.0 photos4.pop6.com +0.0.0.0 photos5.pop6.com +0.0.0.0 photos6.pop6.com +0.0.0.0 photos7.pop6.com +0.0.0.0 photos8.pop6.com +0.0.0.0 photos.daily-deals.analoganalytics.com +0.0.0.0 photos.pop6.com +0.0.0.0 phpads.astalavista.us +0.0.0.0 phpads.cnpapers.com +0.0.0.0 phpads.flipcorp.com +0.0.0.0 phpads.foundrymusic.com +0.0.0.0 phpads.i-merge.net +0.0.0.0 phpads.macbidouille.com +0.0.0.0 phpadsnew.gamefolk.de +0.0.0.0 phpadsnew.wn.com +0.0.0.0 php.fark.com +0.0.0.0 pick-savings.com +0.0.0.0 p.ic.tynt.com +0.0.0.0 pink.habralab.ru +0.0.0.0 pix01.revsci.net +0.0.0.0 pix521.adtech.de +0.0.0.0 pix521.adtech.fr +0.0.0.0 pix521.adtech.us +0.0.0.0 pix522.adtech.de +0.0.0.0 pix522.adtech.fr +0.0.0.0 pix522.adtech.us +0.0.0.0 pixel.everesttech.net +0.0.0.0 pixel.mathtag.com +0.0.0.0 pixel.quantserve.com +0.0.0.0 pixel.sitescout.com +0.0.0.0 plasmatv4free.com +0.0.0.0 plasmatvreward.com +0.0.0.0 playlink.pl +0.0.0.0 playtime.tubemogul.com +0.0.0.0 pl.bbelements.com +0.0.0.0 pmstrk.mercadolivre.com.br +0.0.0.0 pntm.adbureau.net +0.0.0.0 pntm-images.adbureau.net +0.0.0.0 pol.bbelements.com +0.0.0.0 politicalopinionsurvey.com +0.0.0.0 pool.pebblemedia.adhese.com +0.0.0.0 popadscdn.net +0.0.0.0 popclick.net +0.0.0.0 poponclick.com +0.0.0.0 popunder.adsrevenue.net +0.0.0.0 popunder.paypopup.com +0.0.0.0 popupclick.ru +0.0.0.0 popupdomination.com +0.0.0.0 popup.matchmaker.com +0.0.0.0 popups.ad-logics.com +0.0.0.0 popups.infostart.com +0.0.0.0 postmasterdirect.com +0.0.0.0 post.rmbn.ru +0.0.0.0 pp.free.fr +0.0.0.0 p.profistats.net +0.0.0.0 p.publico.es +0.0.0.0 premium.ascensionweb.com +0.0.0.0 premiumholidayoffers.com +0.0.0.0 premiumproductsonline.com +0.0.0.0 premium-reward-club.com +0.0.0.0 prexyone.appspot.com +0.0.0.0 primetime.ad.primetime.net +0.0.0.0 privitize.com +0.0.0.0 prizes.co.uk +0.0.0.0 productopinionpanel.com +0.0.0.0 productresearchpanel.com +0.0.0.0 producttestpanel.com +0.0.0.0 profile.uproxx.com +0.0.0.0 promo.awempire.com +0.0.0.0 promo.easy-dating.org +0.0.0.0 promos.fling.com +0.0.0.0 promote-bz.net +0.0.0.0 promotion.partnercash.com +0.0.0.0 proximityads.flipcorp.com +0.0.0.0 proxy.blogads.com +0.0.0.0 ptrads.mp3.com +0.0.0.0 pubdirecte.com +0.0.0.0 pubimgs.sapo.pt +0.0.0.0 publiads.com +0.0.0.0 publicidades.redtotalonline.com +0.0.0.0 publicis.adcentriconline.com +0.0.0.0 publish.bonzaii.no +0.0.0.0 publishers.adscholar.com +0.0.0.0 publishers.bidtraffic.com +0.0.0.0 publishers.brokertraffic.com +0.0.0.0 publishing.kalooga.com +0.0.0.0 pub.sapo.pt +0.0.0.0 pubshop.img.uol.com.br +0.0.0.0 purgecolon.net +0.0.0.0 px10.net +0.0.0.0 q.azcentral.com +0.0.0.0 q.b.h.cltomedia.info +0.0.0.0 qip.magna.ru +0.0.0.0 qitrck.com +0.0.0.0 quickbrowsersearch.com +0.0.0.0 r1-ads.ace.advertising.com +0.0.0.0 r.ace.advertising.com +0.0.0.0 radaronline.advertserve.com +0.0.0.0 r.admob.com +0.0.0.0 rad.msn.com +0.0.0.0 rads.stackoverflow.com +0.0.0.0 ravel-rewardpath.com +0.0.0.0 rb.burstway.com +0.0.0.0 rb.newsru.com +0.0.0.0 rbqip.pochta.ru +0.0.0.0 rc.asci.freenet.de +0.0.0.0 rc.bt.ilsemedia.nl +0.0.0.0 rccl.bridgetrack.com +0.0.0.0 rcdna.gwallet.com +0.0.0.0 r.chitika.net +0.0.0.0 rc.hotkeys.com +0.0.0.0 rcm-images.amazon.com +0.0.0.0 rcm-it.amazon.it +0.0.0.0 rc.rlcdn.com +0.0.0.0 rc.wl.webads.nl +0.0.0.0 realads.realmedia.com +0.0.0.0 realgfsbucks.com +0.0.0.0 realmedia-a800.d4p.net # Scientific American +0.0.0.0 realmedia.advance.net +0.0.0.0 recreation-leisure-rewardpath.com +0.0.0.0 red01.as-eu.falkag.net +0.0.0.0 red01.as-us.falkag.net +0.0.0.0 red02.as-eu.falkag.net +0.0.0.0 red02.as-us.falkag.net +0.0.0.0 red03.as-eu.falkag.net +0.0.0.0 red03.as-us.falkag.net +0.0.0.0 red04.as-eu.falkag.net +0.0.0.0 red04.as-us.falkag.net +0.0.0.0 red.as-eu.falkag.net +0.0.0.0 red.as-us.falkag.net +0.0.0.0 redherring.ngadcenter.net +0.0.0.0 redirect.click2net.com +0.0.0.0 redirect.hotkeys.com +0.0.0.0 reduxads.valuead.com +0.0.0.0 reg.coolsavings.com +0.0.0.0 regflow.com +0.0.0.0 regie.espace-plus.net +0.0.0.0 regio.adlink.de +0.0.0.0 reklama.onet.pl +0.0.0.0 reklamy.sfd.pl +0.0.0.0 re.kontera.com +0.0.0.0 rek.www.wp.pl +0.0.0.0 relestar.com +0.0.0.0 remotead.cnet.com +0.0.0.0 report02.adtech.de +0.0.0.0 report02.adtech.fr +0.0.0.0 report02.adtech.us +0.0.0.0 reporter001.adtech.de +0.0.0.0 reporter001.adtech.fr +0.0.0.0 reporter001.adtech.us +0.0.0.0 reporter.adtech.de +0.0.0.0 reporter.adtech.fr +0.0.0.0 reporter.adtech.us +0.0.0.0 reportimage.adtech.de +0.0.0.0 reportimage.adtech.fr +0.0.0.0 reportimage.adtech.us +0.0.0.0 resolvingserver.com +0.0.0.0 resources.infolinks.com +0.0.0.0 restaurantcom.tt.omtrdc.net +0.0.0.0 reverso.refr.adgtw.orangeads.fr +0.0.0.0 revsci.net +0.0.0.0 rewardblvd.com +0.0.0.0 rewardhotspot.com +0.0.0.0 rewardsflow.com +0.0.0.0 rhads.sv.publicus.com +0.0.0.0 rh.revolvermaps.com +0.0.0.0 richmedia.yimg.com +0.0.0.0 ridepush.com +0.0.0.0 ringtonepartner.com +0.0.0.0 rmbn.ru +0.0.0.0 rmedia.boston.com +0.0.0.0 rmm1u.checkm8.com +0.0.0.0 rms.admeta.com +0.0.0.0 ro.bbelements.com +0.0.0.0 romepartners.com +0.0.0.0 roosevelt.gjbig.com +0.0.0.0 rosettastone.tt.omtrdc.net +0.0.0.0 rotabanner100.utro.ru +0.0.0.0 rotabanner468.utro.ru +0.0.0.0 rotate.infowars.com +0.0.0.0 rotator.adjuggler.com +0.0.0.0 rotator.juggler.inetinteractive.com +0.0.0.0 rotobanner468.utro.ru +0.0.0.0 rovion.com +0.0.0.0 rpc.trafficfactory.biz +0.0.0.0 rp.hit.gemius.pl +0.0.0.0 r.reklama.biz +0.0.0.0 rscounter10.com +0.0.0.0 rsense-ad.realclick.co.kr +0.0.0.0 rss.buysellads.com +0.0.0.0 rt2.infolinks.com +0.0.0.0 rt3.infolinks.com +0.0.0.0 rtb.pclick.yahoo.com +0.0.0.0 rtb.tubemogul.com +0.0.0.0 rtr.innovid.com +0.0.0.0 rts.sparkstudios.com +0.0.0.0 r.turn.com +0.0.0.0 ru.bbelements.com +0.0.0.0 ru.redtram.com +0.0.0.0 russ-shalavy.ru +0.0.0.0 rv.adcpx.v1.de.eusem.adaos-ads.net +0.0.0.0 rya.rockyou.com +0.0.0.0 s0b.bluestreak.com +0.0.0.0 s1.buysellads.com +0.0.0.0 s1.cz.adocean.pl +0.0.0.0 s1.gratkapl.adocean.pl +0.0.0.0 s2.buysellads.com +0.0.0.0 s3.buysellads.com +0.0.0.0 s5.addthis.com +0.0.0.0 s7.addthis.com +0.0.0.0 s.admulti.com +0.0.0.0 sad.sharethis.com +0.0.0.0 safe.hyperpaysys.com +0.0.0.0 safenyplanet.in +0.0.0.0 salesforcecom.tt.omtrdc.net +0.0.0.0 s.amazon-adsystem.com +0.0.0.0 samsung3.solution.weborama.fr +0.0.0.0 s.as-us.falkag.net +0.0.0.0 sat-city-ads.com +0.0.0.0 s.atemda.com +0.0.0.0 saturn.tiser.com.au +0.0.0.0 save-plan.com +0.0.0.0 savings-specials.com +0.0.0.0 savings-time.com +0.0.0.0 s.boom.ro +0.0.0.0 schoorsteen.geenstijl.nl +0.0.0.0 schumacher.adtech.de +0.0.0.0 schumacher.adtech.fr +0.0.0.0 schumacher.adtech.us +0.0.0.0 schwab.tt.omtrdc.net +0.0.0.0 s.clicktale.net +0.0.0.0 scoremygift.com +0.0.0.0 screen-mates.com +0.0.0.0 script.banstex.com +0.0.0.0 script.crsspxl.com +0.0.0.0 scripts.verticalacuity.com +0.0.0.0 scr.kliksaya.com +0.0.0.0 s.di.com.pl +0.0.0.0 se.adserver.yahoo.com +0.0.0.0 search.addthis.com +0.0.0.0 search.freeonline.com +0.0.0.0 search.keywordblocks.com +0.0.0.0 search.netseer.com +0.0.0.0 searchportal.information.com +0.0.0.0 searchwe.com +0.0.0.0 seasonalsamplerspecials.com +0.0.0.0 sec.hit.gemius.pl +0.0.0.0 secimage.adtech.de +0.0.0.0 secimage.adtech.fr +0.0.0.0 secimage.adtech.us +0.0.0.0 secserv.adtech.de +0.0.0.0 secserv.adtech.fr +0.0.0.0 secserv.adtech.us +0.0.0.0 secure.ace-tag.advertising.com +0.0.0.0 secure.addthis.com +0.0.0.0 secureads.ft.com +0.0.0.0 secure.bidvertiserr.com +0.0.0.0 securecontactinfo.com +0.0.0.0 secure.gaug.es +0.0.0.0 secure.img-cdn.mediaplex.com +0.0.0.0 securerunner.com +0.0.0.0 secure.webconnect.net +0.0.0.0 seduction-zone.com +0.0.0.0 sel.as-eu.falkag.net +0.0.0.0 sel.as-us.falkag.net +0.0.0.0 select001.adtech.de +0.0.0.0 select001.adtech.fr +0.0.0.0 select001.adtech.us +0.0.0.0 select002.adtech.de +0.0.0.0 select002.adtech.fr +0.0.0.0 select002.adtech.us +0.0.0.0 select003.adtech.de +0.0.0.0 select003.adtech.fr +0.0.0.0 select003.adtech.us +0.0.0.0 select004.adtech.de +0.0.0.0 select004.adtech.fr +0.0.0.0 select004.adtech.us +0.0.0.0 sergarius.popunder.ru +0.0.0.0 serv2.ad-rotator.com +0.0.0.0 serv.ad-rotator.com +0.0.0.0 servads.aip.org +0.0.0.0 serv.adspeed.com +0.0.0.0 servedbyadbutler.com +0.0.0.0 servedby.adcombination.com +0.0.0.0 servedby.advertising.com +0.0.0.0 servedby.flashtalking.com +0.0.0.0 servedby.netshelter.net +0.0.0.0 servedby.precisionclick.com +0.0.0.0 serve.freegaypix.com +0.0.0.0 serve.popads.net +0.0.0.0 serve.prestigecasino.com +0.0.0.0 server01.popupmoney.com +0.0.0.0 server2.as5000.com +0.0.0.0 server2.mediajmp.com +0.0.0.0 server3.yieldmanaged.com +0.0.0.0 server.as5000.com +0.0.0.0 server.bittads.com +0.0.0.0 server.cpmstar.com +0.0.0.0 server.popads.net +0.0.0.0 server-ssl.yieldmanaged.com +0.0.0.0 service001.adtech.de +0.0.0.0 service001.adtech.fr +0.0.0.0 service001.adtech.us +0.0.0.0 service002.adtech.de +0.0.0.0 service002.adtech.fr +0.0.0.0 service002.adtech.us +0.0.0.0 service003.adtech.de +0.0.0.0 service003.adtech.fr +0.0.0.0 service003.adtech.us +0.0.0.0 service004.adtech.fr +0.0.0.0 service004.adtech.us +0.0.0.0 service00x.adtech.de +0.0.0.0 service00x.adtech.fr +0.0.0.0 service00x.adtech.us +0.0.0.0 service.adtech.de +0.0.0.0 service.adtech.fr +0.0.0.0 service.adtech.us +0.0.0.0 services1.adtech.de +0.0.0.0 services1.adtech.fr +0.0.0.0 services1.adtech.us +0.0.0.0 services.adtech.de +0.0.0.0 services.adtech.fr +0.0.0.0 services.adtech.us +0.0.0.0 serving.plexop.net +0.0.0.0 sexpartnerx.com +0.0.0.0 sexsponsors.com +0.0.0.0 sexzavod.com +0.0.0.0 sfads.osdn.com +0.0.0.0 s.flite.com +0.0.0.0 sg.adserver.yahoo.com +0.0.0.0 sgs001.adtech.de +0.0.0.0 sgs001.adtech.fr +0.0.0.0 sgs001.adtech.us +0.0.0.0 sh4sure-images.adbureau.net +0.0.0.0 shareasale.com +0.0.0.0 sharebar.addthiscdn.com +0.0.0.0 share-server.com +0.0.0.0 shc-rebates.com +0.0.0.0 shinystat.shiny.it +0.0.0.0 shopperpromotions.com +0.0.0.0 shopping-offer.com +0.0.0.0 shoppingsiterewards.com +0.0.0.0 shops-malls-rewardpath.com +0.0.0.0 shoptosaveenergy.com +0.0.0.0 showads1000.pubmatic.com +0.0.0.0 showadsak.pubmatic.com +0.0.0.0 sifomedia.citypaketet.se +0.0.0.0 signup.advance.net +0.0.0.0 si.hit.gemius.pl +0.0.0.0 simg.zedo.com +0.0.0.0 simpleads.net +0.0.0.0 simpli.fi +0.0.0.0 s.innovid.com +0.0.0.0 sixapart.adbureau.net +0.0.0.0 sizzle-savings.com +0.0.0.0 skgde.adocean.pl +0.0.0.0 skill.skilljam.com +0.0.0.0 slider.plugrush.com +0.0.0.0 smartadserver +0.0.0.0 smartadserver.com +0.0.0.0 smart.besonders.ru +0.0.0.0 smartclip.com +0.0.0.0 smartclip.net +0.0.0.0 smartcontext.pl +0.0.0.0 smartinit.webads.nl +0.0.0.0 smart-scripts.com +0.0.0.0 smartshare.lgtvsdp.com +0.0.0.0 s.media-imdb.com +0.0.0.0 s.megaclick.com +0.0.0.0 smile.modchipstore.com +0.0.0.0 smm.sitescout.com +0.0.0.0 s.moatads.com +0.0.0.0 smokersopinionpoll.com +0.0.0.0 smsmovies.net +0.0.0.0 snaps.vidiemi.com +0.0.0.0 sn.baventures.com +0.0.0.0 snip.answers.com +0.0.0.0 snipjs.answcdn.com +0.0.0.0 sochr.com +0.0.0.0 social.bidsystem.com +0.0.0.0 softlinkers.popunder.ru +0.0.0.0 sokrates.adtech.de +0.0.0.0 sokrates.adtech.fr +0.0.0.0 sokrates.adtech.us +0.0.0.0 sol.adbureau.net +0.0.0.0 sol-images.adbureau.net +0.0.0.0 solitairetime.com +0.0.0.0 solution.weborama.fr +0.0.0.0 somethingawful.crwdcntrl.net +0.0.0.0 sonycomputerentertai.tt.omtrdc.net +0.0.0.0 soongu.info +0.0.0.0 spanids.dictionary.com +0.0.0.0 spanids.thesaurus.com +0.0.0.0 spc.cekfmeoejdbfcfichgbfcgjf.vast2as3.glammedia-pubnet.northamerica.telemetryverification.net +0.0.0.0 spe.atdmt.com +0.0.0.0 specialgiftrewards.com +0.0.0.0 specialoffers.aol.com +0.0.0.0 specialonlinegifts.com +0.0.0.0 specials-rewardpath.com +0.0.0.0 speedboink.com +0.0.0.0 speedclicks.ero-advertising.com +0.0.0.0 speed.pointroll.com # Microsoft +0.0.0.0 spinbox.com +0.0.0.0 spinbox.consumerreview.com +0.0.0.0 spinbox.freedom.com +0.0.0.0 spinbox.macworld.com +0.0.0.0 spinbox.techtracker.com +0.0.0.0 spin.spinbox.net +0.0.0.0 sponsor1.com +0.0.0.0 sponsors.behance.com +0.0.0.0 sponsors.ezgreen.com +0.0.0.0 sponsorships.net +0.0.0.0 sports-bonuspath.com +0.0.0.0 sports-fitness-rewardpath.com +0.0.0.0 sports-offer.com +0.0.0.0 sports-offer.net +0.0.0.0 sports-premiumblvd.com +0.0.0.0 spotxchange.com +0.0.0.0 s.ppjol.net +0.0.0.0 sq2trk2.com +0.0.0.0 srs.targetpoint.com +0.0.0.0 srv.juiceadv.com +0.0.0.0 ssads.osdn.com +0.0.0.0 s.skimresources.com +0.0.0.0 sso.canada.com +0.0.0.0 staging.snip.answers.com +0.0.0.0 stampen.adtlgc.com +0.0.0.0 stampen.linkpulse.com +0.0.0.0 stampscom.tt.omtrdc.net +0.0.0.0 stanzapub.advertserve.com +0.0.0.0 star-advertising.com +0.0.0.0 stat.blogads.com +0.0.0.0 stat.dealtime.com +0.0.0.0 stat.ebuzzing.com +0.0.0.0 static1.influads.com +0.0.0.0 static.2mdn.net +0.0.0.0 static.admaximize.com +0.0.0.0 staticads.btopenworld.com +0.0.0.0 static.adsonar.com +0.0.0.0 static.adtaily.pl +0.0.0.0 static.adzerk.net +0.0.0.0 static.aff-landing-tmp.foxtab.com +0.0.0.0 staticb.mydirtyhobby.com +0.0.0.0 static.carbonads.com +0.0.0.0 static.clicktorrent.info +0.0.0.0 static.creatives.livejasmin.com +0.0.0.0 static.doubleclick.net +0.0.0.0 static.everyone.net +0.0.0.0 static.exoclick.com +0.0.0.0 static.fastpic.ru +0.0.0.0 static.firehunt.com +0.0.0.0 static.fmpub.net +0.0.0.0 static.freenet.de +0.0.0.0 static.groupy.co.nz +0.0.0.0 static.hitfarm.com +0.0.0.0 static.ifa.camads.net +0.0.0.0 static.l3.cdn.adbucks.com +0.0.0.0 static.l3.cdn.adsucks.com +0.0.0.0 static.plista.com +0.0.0.0 static.plugrush.com +0.0.0.0 static.pulse360.com +0.0.0.0 static.scanscout.com +0.0.0.0 static.vpptechnologies.com +0.0.0.0 static.way2traffic.com +0.0.0.0 statistik-gallup.dk +0.0.0.0 stats2.dooyoo.com +0.0.0.0 stats.askmoses.com +0.0.0.0 stats.buzzparadise.com +0.0.0.0 stats.jtvnw.net +0.0.0.0 stats.shopify.com +0.0.0.0 status.addthis.com +0.0.0.0 st.blogads.com +0.0.0.0 s.tcimg.com +0.0.0.0 st.marketgid.com +0.0.0.0 stocker.bonnint.net +0.0.0.0 storage.softure.com +0.0.0.0 storage.trafic.ro +0.0.0.0 streamate.com +0.0.0.0 stts.rbc.ru +0.0.0.0 st.valueclick.com +0.0.0.0 su.addthis.com +0.0.0.0 subtracts.userplane.com +0.0.0.0 sudokuwhiz.com +0.0.0.0 sunmaker.com +0.0.0.0 superbrewards.com +0.0.0.0 support.sweepstakes.com +0.0.0.0 supremeadsonline.com +0.0.0.0 suresafe1.adsovo.com +0.0.0.0 surplus-suppliers.com +0.0.0.0 surveycentral.directinsure.info +0.0.0.0 surveymonkeycom.tt.omtrdc.net +0.0.0.0 surveypass.com +0.0.0.0 susi.adtech.fr +0.0.0.0 susi.adtech.us +0.0.0.0 svd2.adtlgc.com +0.0.0.0 svd.adtlgc.com +0.0.0.0 sview.avenuea.com +0.0.0.0 sweetsforfree.com +0.0.0.0 symbiosting.com +0.0.0.0 synad2.nuffnang.com.cn +0.0.0.0 synad.nuffnang.com.sg +0.0.0.0 syncaccess.net +0.0.0.0 sync.mathtag.com +0.0.0.0 syndicated.mondominishows.com +0.0.0.0 syndication.exoclick.com +0.0.0.0 syndication.traffichaus.com +0.0.0.0 syn.verticalacuity.com +0.0.0.0 sysadmin.map24.com +0.0.0.0 t1.adserver.com +0.0.0.0 t4.liverail.com +0.0.0.0 t-ads.adap.tv +0.0.0.0 tag1.webabacus.com +0.0.0.0 tag.admeld.com +0.0.0.0 tag.contextweb.com +0.0.0.0 tag.regieci.com +0.0.0.0 tags.bluekai.com +0.0.0.0 tags.hypeads.org +0.0.0.0 tag.webcompteur.com +0.0.0.0 tag.yieldoptimizer.com +0.0.0.0 taloussanomat.linkpulse.com +0.0.0.0 tap2-cdn.rubiconproject.com +0.0.0.0 tbtrack.zutrack.com +0.0.0.0 tcadops.ca +0.0.0.0 tcimg.com +0.0.0.0 t.cpmadvisors.com +0.0.0.0 tdameritrade.tt.omtrdc.net +0.0.0.0 tdc.advertorials.dk +0.0.0.0 tdkads.ads.dk +0.0.0.0 techreview.adbureau.net +0.0.0.0 techreview-images.adbureau.net +0.0.0.0 teeser.ru +0.0.0.0 te.kontera.com +0.0.0.0 tel.geenstijl.nl +0.0.0.0 textads.madisonavenue.com +0.0.0.0 textad.traficdublu.ro +0.0.0.0 text-link-ads.com +0.0.0.0 text-link-ads.ientry.com +0.0.0.0 text-link-ads-inventory.com +0.0.0.0 textsrv.com +0.0.0.0 tf.nexac.com +0.0.0.0 tgpmanager.com +0.0.0.0 the-binary-trader.biz +0.0.0.0 the-path-gateway.com +0.0.0.0 thepiratetrader.com +0.0.0.0 the-smart-stop.com +0.0.0.0 theuploadbusiness.com +0.0.0.0 theuseful.com +0.0.0.0 theuseful.net +0.0.0.0 thinknyc.eu-adcenter.net +0.0.0.0 thinktarget.com +0.0.0.0 thinlaptoprewards.com +0.0.0.0 this.content.served.by.adshuffle.com +0.0.0.0 thoughtfully-free.com +0.0.0.0 thruport.com +0.0.0.0 tmp3.nexac.com +0.0.0.0 tmsads.tribune.com +0.0.0.0 tmx.technoratimedia.com +0.0.0.0 tn.adserve.com +0.0.0.0 toads.osdn.com +0.0.0.0 tons-to-see.com +0.0.0.0 toolbar.adperium.com +0.0.0.0 top100-images.rambler.ru +0.0.0.0 top1site.3host.com +0.0.0.0 top5.mail.ru +0.0.0.0 topbrandrewards.com +0.0.0.0 topconsumergifts.com +0.0.0.0 topdemaroc.com +0.0.0.0 topica.advertserve.com +0.0.0.0 top.list.ru +0.0.0.0 toplist.throughput.de +0.0.0.0 topmarketcenter.com +0.0.0.0 touche.adcentric.proximi-t.com +0.0.0.0 tower.adexpedia.com +0.0.0.0 toy-offer.com +0.0.0.0 toy-offer.net +0.0.0.0 tpads.ovguide.com +0.0.0.0 tpc.googlesyndication.com +0.0.0.0 tps30.doubleverify.com +0.0.0.0 tps31.doubleverify.com +0.0.0.0 track.adbooth.net +0.0.0.0 trackadvertising.net +0.0.0.0 track-apmebf.cj.akadns.net +0.0.0.0 track.bigbrandpromotions.com +0.0.0.0 track.e7r.com.br +0.0.0.0 trackers.1st-affiliation.fr +0.0.0.0 tracking.craktraffic.com +0.0.0.0 tracking.edvisors.com +0.0.0.0 tracking.eurowebaffiliates.com +0.0.0.0 tracking.joker.com +0.0.0.0 tracking.keywordmax.com +0.0.0.0 tracking.veoxa.com +0.0.0.0 track.omgpl.com +0.0.0.0 track.the-members-section.com +0.0.0.0 track.vscash.com +0.0.0.0 tradearabia.advertserve.com +0.0.0.0 tradefx.advertserve.com +0.0.0.0 trafficbee.com +0.0.0.0 trafficrevenue.net +0.0.0.0 traffictraders.com +0.0.0.0 traffprofit.com +0.0.0.0 trafmag.com +0.0.0.0 trafsearchonline.com +0.0.0.0 traktum.com +0.0.0.0 travel-leisure-bonuspath.com +0.0.0.0 travel-leisure-premiumblvd.com +0.0.0.0 traveller-offer.com +0.0.0.0 traveller-offer.net +0.0.0.0 travelncs.com +0.0.0.0 trekmedia.net +0.0.0.0 trendnews.com +0.0.0.0 trk.alskeip.com +0.0.0.0 trk.etrigue.com +0.0.0.0 trk.yadomedia.com +0.0.0.0 trustsitesite.com +0.0.0.0 trvlnet.adbureau.net +0.0.0.0 trvlnet-images.adbureau.net +0.0.0.0 tr.wl.webads.nl +0.0.0.0 tsms-ad.tsms.com +0.0.0.0 tste.ivillage.com +0.0.0.0 tste.mcclatchyinteractive.com +0.0.0.0 tste.startribune.com +0.0.0.0 ttarget.adbureau.net +0.0.0.0 ttuk.offers4u.mobi +0.0.0.0 turnerapac.d1.sc.omtrdc.net +0.0.0.0 tv2no.linkpulse.com +0.0.0.0 tvshowsnow.tvmax.hop.clickbank.net +0.0.0.0 tw.adserver.yahoo.com +0.0.0.0 twnads.weather.ca # Canadian Weather Network +0.0.0.0 uac.advertising.com +0.0.0.0 u-ads.adap.tv +0.0.0.0 uav.tidaltv.com +0.0.0.0 uc.csc.adserver.yahoo.com +0.0.0.0 uedata.amazon.com +0.0.0.0 uelbdc74fn.s.ad6media.fr +0.0.0.0 uf2.svrni.ca +0.0.0.0 ugo.eu-adcenter.net +0.0.0.0 ui.ppjol.com +0.0.0.0 uk.adserver.yahoo.com +0.0.0.0 uleadstrk.com +0.0.0.0 ultimatefashiongifts.com +0.0.0.0 ultrabestportal.com +0.0.0.0 um.simpli.fi +0.0.0.0 undertonenetworks.com +0.0.0.0 uole.ad.uol.com.br +0.0.0.0 u.openx.net +0.0.0.0 upload.adtech.de +0.0.0.0 upload.adtech.fr +0.0.0.0 upload.adtech.us +0.0.0.0 uproar.com +0.0.0.0 uproar.fortunecity.com +0.0.0.0 upsellit.com +0.0.0.0 us.adserver.yahoo.com +0.0.0.0 usads.vibrantmedia.com +0.0.0.0 usatoday.app.ur.gcion.com +0.0.0.0 usatravel-specials.com +0.0.0.0 usatravel-specials.net +0.0.0.0 us-choicevalue.com +0.0.0.0 usemax.de +0.0.0.0 usr.marketgid.com +0.0.0.0 us-topsites.com +0.0.0.0 ut.addthis.com +0.0.0.0 utarget.ru +0.0.0.0 utils.media-general.com +0.0.0.0 utils.mediageneral.com +0.0.0.0 vad.adbasket.net +0.0.0.0 vads.adbrite.com +0.0.0.0 van.ads.link4ads.com +0.0.0.0 vast.bp3845260.btrll.com +0.0.0.0 vast.bp3846806.btrll.com +0.0.0.0 vast.bp3846885.btrll.com +0.0.0.0 vast.tubemogul.com +0.0.0.0 vclick.adbrite.com +0.0.0.0 venus.goclick.com +0.0.0.0 ve.tscapeplay.com +0.0.0.0 v.fwmrm.net +0.0.0.0 vibrantmedia.com +0.0.0.0 videocop.com +0.0.0.0 videoegg.adbureau.net +0.0.0.0 video-game-rewards-central.com +0.0.0.0 videogamerewardscentral.com +0.0.0.0 videos.fleshlight.com +0.0.0.0 videoslots.888.com +0.0.0.0 videos.video-loader.com +0.0.0.0 view.atdmt.com #This may interfere with downloading from Microsoft, MSDN and TechNet websites. +0.0.0.0 view.avenuea.com +0.0.0.0 view.binlayer.com +0.0.0.0 view.iballs.a1.avenuea.com +0.0.0.0 view.jamba.de +0.0.0.0 view.netrams.com +0.0.0.0 views.m4n.nl +0.0.0.0 viglink.com +0.0.0.0 viglink.pgpartner.com +0.0.0.0 villagevoicecollect.247realmedia.com +0.0.0.0 vip1.tw.adserver.yahoo.com +0.0.0.0 vipfastmoney.com +0.0.0.0 vk.18sexporn.ru +0.0.0.0 vmcsatellite.com +0.0.0.0 vmix.adbureau.net +0.0.0.0 vms.boldchat.com +0.0.0.0 vnu.eu-adcenter.net +0.0.0.0 vodafoneit.solution.weborama.fr +0.0.0.0 vp.tscapeplay.com +0.0.0.0 vu.veoxa.com +0.0.0.0 vzarabotke.ru +0.0.0.0 w100.am15.net +0.0.0.0 w10.am15.net +0.0.0.0 w10.centralmediaserver.com +0.0.0.0 w11.am15.net +0.0.0.0 w11.centralmediaserver.com +0.0.0.0 w12.am15.net +0.0.0.0 w13.am15.net +0.0.0.0 w14.am15.net +0.0.0.0 w15.am15.net +0.0.0.0 w16.am15.net +0.0.0.0 w17.am15.net +0.0.0.0 w18.am15.net +0.0.0.0 w19.am15.net +0.0.0.0 w1.am15.net +0.0.0.0 w1.webcompteur.com +0.0.0.0 w20.am15.net +0.0.0.0 w21.am15.net +0.0.0.0 w22.am15.net +0.0.0.0 w23.am15.net +0.0.0.0 w24.am15.net +0.0.0.0 w25.am15.net +0.0.0.0 w26.am15.net +0.0.0.0 w27.am15.net +0.0.0.0 w28.am15.net +0.0.0.0 w29.am15.net +0.0.0.0 w2.am15.net +0.0.0.0 w30.am15.net +0.0.0.0 w31.am15.net +0.0.0.0 w32.am15.net +0.0.0.0 w33.am15.net +0.0.0.0 w34.am15.net +0.0.0.0 w35.am15.net +0.0.0.0 w36.am15.net +0.0.0.0 w37.am15.net +0.0.0.0 w38.am15.net +0.0.0.0 w39.am15.net +0.0.0.0 w3.am15.net +0.0.0.0 w40.am15.net +0.0.0.0 w41.am15.net +0.0.0.0 w42.am15.net +0.0.0.0 w43.am15.net +0.0.0.0 w44.am15.net +0.0.0.0 w45.am15.net +0.0.0.0 w46.am15.net +0.0.0.0 w47.am15.net +0.0.0.0 w48.am15.net +0.0.0.0 w49.am15.net +0.0.0.0 w4.am15.net +0.0.0.0 w50.am15.net +0.0.0.0 w51.am15.net +0.0.0.0 w52.am15.net +0.0.0.0 w53.am15.net +0.0.0.0 w54.am15.net +0.0.0.0 w55.am15.net +0.0.0.0 w56.am15.net +0.0.0.0 w57.am15.net +0.0.0.0 w58.am15.net +0.0.0.0 w59.am15.net +0.0.0.0 w5.am15.net +0.0.0.0 w60.am15.net +0.0.0.0 w61.am15.net +0.0.0.0 w62.am15.net +0.0.0.0 w63.am15.net +0.0.0.0 w64.am15.net +0.0.0.0 w65.am15.net +0.0.0.0 w66.am15.net +0.0.0.0 w67.am15.net +0.0.0.0 w68.am15.net +0.0.0.0 w69.am15.net +0.0.0.0 w6.am15.net +0.0.0.0 w70.am15.net +0.0.0.0 w71.am15.net +0.0.0.0 w72.am15.net +0.0.0.0 w73.am15.net +0.0.0.0 w74.am15.net +0.0.0.0 w75.am15.net +0.0.0.0 w76.am15.net +0.0.0.0 w77.am15.net +0.0.0.0 w78.am15.net +0.0.0.0 w79.am15.net +0.0.0.0 w7.am15.net +0.0.0.0 w80.am15.net +0.0.0.0 w81.am15.net +0.0.0.0 w82.am15.net +0.0.0.0 w83.am15.net +0.0.0.0 w84.am15.net +0.0.0.0 w85.am15.net +0.0.0.0 w86.am15.net +0.0.0.0 w87.am15.net +0.0.0.0 w88.am15.net +0.0.0.0 w89.am15.net +0.0.0.0 w8.am15.net +0.0.0.0 w90.am15.net +0.0.0.0 w91.am15.net +0.0.0.0 w92.am15.net +0.0.0.0 w93.am15.net +0.0.0.0 w94.am15.net +0.0.0.0 w95.am15.net +0.0.0.0 w96.am15.net +0.0.0.0 w97.am15.net +0.0.0.0 w98.am15.net +0.0.0.0 w99.am15.net +0.0.0.0 w9.am15.net +0.0.0.0 wahoha.com +0.0.0.0 warp.crystalad.com +0.0.0.0 wdm29.com +0.0.0.0 web1b.netreflector.com +0.0.0.0 web.adblade.com +0.0.0.0 webads.bizservers.com +0.0.0.0 webads.nl +0.0.0.0 webcompteur.com +0.0.0.0 webhosting-ads.home.pl +0.0.0.0 webmdcom.tt.omtrdc.net +0.0.0.0 web.nyc.ads.juno.co +0.0.0.0 webservices-rewardpath.com +0.0.0.0 websurvey.spa-mr.com +0.0.0.0 wegetpaid.net +0.0.0.0 w.ic.tynt.com +0.0.0.0 widget3.linkwithin.com +0.0.0.0 widget5.linkwithin.com +0.0.0.0 widget.crowdignite.com +0.0.0.0 widget.plugrush.com +0.0.0.0 widgets.outbrain.com +0.0.0.0 widgets.tcimg.com +0.0.0.0 wigetmedia.com +0.0.0.0 wikiforosh.ir +0.0.0.0 williamhill.es +0.0.0.0 wmedia.rotator.hadj7.adjuggler.net +0.0.0.0 wordplaywhiz.com +0.0.0.0 work-offer.com +0.0.0.0 worry-free-savings.com +0.0.0.0 wppluginspro.com +0.0.0.0 ws.addthis.com +0.0.0.0 wtp101.com +0.0.0.0 ww251.smartadserver.com +0.0.0.0 wwbtads.com +0.0.0.0 www10.ad.tomshardware.com +0.0.0.0 www10.glam.com +0.0.0.0 www10.indiads.com +0.0.0.0 www10.paypopup.com +0.0.0.0 www11.ad.tomshardware.com +0.0.0.0 www123.glam.com +0.0.0.0 www.123specialgifts.com +0.0.0.0 www12.ad.tomshardware.com +0.0.0.0 www12.glam.com +0.0.0.0 www13.ad.tomshardware.com +0.0.0.0 www13.glam.com +0.0.0.0 www14.ad.tomshardware.com +0.0.0.0 www15.ad.tomshardware.com +0.0.0.0 www17.glam.com +0.0.0.0 www18.glam.com +0.0.0.0 www1.adireland.com +0.0.0.0 www1.ad.tomshardware.com +0.0.0.0 www1.bannerspace.com +0.0.0.0 www1.belboon.de +0.0.0.0 www1.clicktorrent.info +0.0.0.0 www1.mpnrs.com +0.0.0.0 www1.popinads.com +0.0.0.0 www1.safenyplanet.in +0.0.0.0 www210.paypopup.com +0.0.0.0 www211.paypopup.com +0.0.0.0 www212.paypopup.com +0.0.0.0 www213.paypopup.com +0.0.0.0 www.247realmedia.com +0.0.0.0 www24a.glam.com +0.0.0.0 www24.glam.com +0.0.0.0 www25a.glam.com +0.0.0.0 www25.glam.com +0.0.0.0 www2.adireland.com +0.0.0.0 www2.adserverpub.com +0.0.0.0 www2.ad.tomshardware.com +0.0.0.0 www.2-art-coliseum.com +0.0.0.0 www2.bannerspace.com +0.0.0.0 www2.glam.com +0.0.0.0 www30a1.glam.com +0.0.0.0 www30a1-orig.glam.com +0.0.0.0 www30a2-orig.glam.com +0.0.0.0 www30a3.glam.com +0.0.0.0 www30a3-orig.glam.com +0.0.0.0 www30a7.glam.com +0.0.0.0 www30.glam.com +0.0.0.0 www30l2.glam.com +0.0.0.0 www30t1-orig.glam.com +0.0.0.0 www.321cba.com +0.0.0.0 www35f.glam.com +0.0.0.0 www35jm.glam.com +0.0.0.0 www35t.glam.com +0.0.0.0 www.360ads.com +0.0.0.0 www3.addthis.com +0.0.0.0 www3.adireland.com +0.0.0.0 www3.ad.tomshardware.com +0.0.0.0 www3.bannerspace.com +0.0.0.0 www3.game-advertising-online.com +0.0.0.0 www.3qqq.net +0.0.0.0 www.3turtles.com +0.0.0.0 www.404errorpage.com +0.0.0.0 www4.ad.tomshardware.com +0.0.0.0 www4.bannerspace.com +0.0.0.0 www4.glam.com +0.0.0.0 www4.smartadserver.com +0.0.0.0 www5.ad.tomshardware.com +0.0.0.0 www5.bannerspace.com +0.0.0.0 www.5thavenue.com +0.0.0.0 www6.ad.tomshardware.com +0.0.0.0 www6.bannerspace.com +0.0.0.0 www74.valueclick.com +0.0.0.0 www.7500.com +0.0.0.0 www7.ad.tomshardware.com +0.0.0.0 www7.bannerspace.com +0.0.0.0 www.7bpeople.com +0.0.0.0 www.7cnbcnews.com +0.0.0.0 www.805m.com +0.0.0.0 www81.valueclick.com +0.0.0.0 www.888casino.com +0.0.0.0 www.888.com +0.0.0.0 www.888poker.com +0.0.0.0 www8.ad.tomshardware.com +0.0.0.0 www8.bannerspace.com +0.0.0.0 www.961.com +0.0.0.0 www9.ad.tomshardware.com +0.0.0.0 www9.paypopup.com +0.0.0.0 www.abrogatesdv.info +0.0.0.0 www.actiondesk.com +0.0.0.0 www.action.ientry.net +0.0.0.0 www.adbanner.gr +0.0.0.0 www.adbrite.com +0.0.0.0 www.adcanadian.com +0.0.0.0 www.adcash.com +0.0.0.0 www.addthiscdn.com +0.0.0.0 www.adengage.com +0.0.0.0 www.adfunkyserver.com +0.0.0.0 www.adfusion.com +0.0.0.0 www.adimages.beeb.com +0.0.0.0 www.adipics.com +0.0.0.0 www.adireland.com +0.0.0.0 www.adjmps.com +0.0.0.0 www.adjug.com +0.0.0.0 www.adloader.com +0.0.0.0 www.adlogix.com +0.0.0.0 www.admex.com +0.0.0.0 www.adnet.biz +0.0.0.0 www.adnet.com +0.0.0.0 www.adnet.de +0.0.0.0 www.adobee.com +0.0.0.0 www.adocean.pl +0.0.0.0 www.adotube.com +0.0.0.0 www.adpepper.dk +0.0.0.0 www.adpowerzone.com +0.0.0.0 www.adquest3d.com +0.0.0.0 www.adreporting.com +0.0.0.0 www.ads2srv.com +0.0.0.0 www.adsentnetwork.com +0.0.0.0 www.adserver.co.il +0.0.0.0 www.adserver.com +0.0.0.0 www.adserver.com.my +0.0.0.0 www.adserver.com.pl +0.0.0.0 www.adserver-espnet.sportszone.net +0.0.0.0 www.adserver.janes.net +0.0.0.0 www.adserver.janes.org +0.0.0.0 www.adserver.jolt.co.uk +0.0.0.0 www.adserver.net +0.0.0.0 www.adserver.ugo.nl +0.0.0.0 www.adservtech.com +0.0.0.0 www.adsinimages.com +0.0.0.0 www.ads.joetec.net +0.0.0.0 www.adsoftware.com +0.0.0.0 www.ad-souk.com +0.0.0.0 www.adspics.com +0.0.0.0 www.ads.revenue.net +0.0.0.0 www.adstogo.com +0.0.0.0 www.adstreams.org +0.0.0.0 www.adtaily.pl +0.0.0.0 www.adtechus.com +0.0.0.0 www.ad.tgdaily.com +0.0.0.0 www.adtlgc.com +0.0.0.0 www.ad.tomshardware.com +0.0.0.0 www.adtrader.com +0.0.0.0 www.adtrix.com +0.0.0.0 www.ad.twitchguru.com +0.0.0.0 www.ad-up.com +0.0.0.0 www.advaliant.com +0.0.0.0 www.advertising-department.com +0.0.0.0 www.advertlets.com +0.0.0.0 www.advertpro.com +0.0.0.0 www.adverts.dcthomson.co.uk +0.0.0.0 www.advertyz.com +0.0.0.0 www.ad-words.ru +0.0.0.0 www.afcyhf.com +0.0.0.0 www.affiliateclick.com +0.0.0.0 www.affiliate-fr.com +0.0.0.0 www.affiliation-france.com +0.0.0.0 www.afform.co.uk +0.0.0.0 www.affpartners.com +0.0.0.0 www.afterdownload.com +0.0.0.0 www.agkn.com +0.0.0.0 www.alexxe.com +0.0.0.0 www.allosponsor.com +0.0.0.0 www.annuaire-autosurf.com +0.0.0.0 www.apparelncs.com +0.0.0.0 www.apparel-offer.com +0.0.0.0 www.applelounge.com +0.0.0.0 www.appnexus.com +0.0.0.0 www.art-music-rewardpath.com +0.0.0.0 www.art-offer.com +0.0.0.0 www.art-offer.net +0.0.0.0 www.art-photo-music-premiumblvd.com +0.0.0.0 www.art-photo-music-rewardempire.com +0.0.0.0 www.art-photo-music-savingblvd.com +0.0.0.0 www.auctionshare.net +0.0.0.0 www.aureate.com +0.0.0.0 www.autohipnose.com +0.0.0.0 www.automotive-offer.com +0.0.0.0 www.automotive-rewardpath.com +0.0.0.0 www.avcounter10.com +0.0.0.0 www.avsads.com +0.0.0.0 www.a.websponsors.com +0.0.0.0 www.awesomevipoffers.com +0.0.0.0 www.awin1.com +0.0.0.0 www.awltovhc.com #qksrv +0.0.0.0 www.bananacashback.com +0.0.0.0 www.banner4all.dk +0.0.0.0 www.bannerads.de +0.0.0.0 www.bannerbackup.com +0.0.0.0 www.bannerconnect.net +0.0.0.0 www.banners.paramountzone.com +0.0.0.0 www.bannersurvey.biz +0.0.0.0 www.banstex.com +0.0.0.0 www.bargainbeautybuys.com +0.0.0.0 www.bbelements.com +0.0.0.0 www.bestshopperrewards.com +0.0.0.0 www.bet365.com +0.0.0.0 www.bidtraffic.com +0.0.0.0 www.bidvertiser.com +0.0.0.0 www.bigbrandpromotions.com +0.0.0.0 www.bigbrandrewards.com +0.0.0.0 www.biggestgiftrewards.com +0.0.0.0 www.binarysystem4u.com +0.0.0.0 www.biz-offer.com +0.0.0.0 www.bizopprewards.com +0.0.0.0 www.blasphemysfhs.info +0.0.0.0 www.blatant8jh.info +0.0.0.0 www.bluediamondoffers.com +0.0.0.0 www.bnnr.nl +0.0.0.0 www.bonzi.com +0.0.0.0 www.bookclub-offer.com +0.0.0.0 www.books-media-edu-premiumblvd.com +0.0.0.0 www.books-media-edu-rewardempire.com +0.0.0.0 www.books-media-rewardpath.com +0.0.0.0 www.boonsolutions.com +0.0.0.0 www.bostonsubwayoffer.com +0.0.0.0 www.brandrewardcentral.com +0.0.0.0 www.brandsurveypanel.com +0.0.0.0 www.brokertraffic.com +0.0.0.0 www.budsinc.com +0.0.0.0 www.bugsbanner.it +0.0.0.0 www.bulkclicks.com +0.0.0.0 www.bulletads.com +0.0.0.0 www.burstnet.com +0.0.0.0 www.business-rewardpath.com +0.0.0.0 www.bus-offer.com +0.0.0.0 www.buttcandy.com +0.0.0.0 www.buwobarun.cn +0.0.0.0 www.buycheapadvertising.com +0.0.0.0 www.buyhitscheap.com +0.0.0.0 www.capath.com +0.0.0.0 www.careers-rewardpath.com +0.0.0.0 www.car-truck-boat-bonuspath.com +0.0.0.0 www.car-truck-boat-premiumblvd.com +0.0.0.0 www.cashback.co.uk +0.0.0.0 www.cashbackwow.co.uk +0.0.0.0 www.cashcount.com +0.0.0.0 www.casino770.com +0.0.0.0 www.catalinkcashback.com +0.0.0.0 www.cell-phone-giveaways.com +0.0.0.0 www.cellphoneincentives.com +0.0.0.0 www.chainsawoffer.com +0.0.0.0 www.chartbeat.com +0.0.0.0 www.choicedealz.com +0.0.0.0 www.choicesurveypanel.com +0.0.0.0 www.christianbusinessadvertising.com +0.0.0.0 www.ciqugasox.cn +0.0.0.0 www.claimfreerewards.com +0.0.0.0 www.clashmediausa.com +0.0.0.0 www.click10.com +0.0.0.0 www.click4click.com +0.0.0.0 www.clickbank.com +0.0.0.0 www.clickdensity.com +0.0.0.0 www.click-find-save.com +0.0.0.0 www.click-see-save.com +0.0.0.0 www.clicksor.com +0.0.0.0 www.clicksotrk.com +0.0.0.0 www.clicktale.com +0.0.0.0 www.clicktale.net +0.0.0.0 www.clickthruserver.com +0.0.0.0 www.clickthrutraffic.com +0.0.0.0 www.clicktilluwin.com +0.0.0.0 www.clicktorrent.info +0.0.0.0 www.clickxchange.com +0.0.0.0 www.closeoutproductsreview.com +0.0.0.0 www.cm1359.com +0.0.0.0 www.come-see-it-all.com +0.0.0.0 www.commerce-offer.com +0.0.0.0 www.commerce-rewardpath.com +0.0.0.0 www.computer-offer.com +0.0.0.0 www.computer-offer.net +0.0.0.0 www.computers-electronics-rewardpath.com +0.0.0.0 www.computersncs.com +0.0.0.0 www.consumergiftcenter.com +0.0.0.0 www.consumerincentivenetwork.com +0.0.0.0 www.consumer-org.com +0.0.0.0 www.contaxe.com +0.0.0.0 www.contextuads.com +0.0.0.0 www.contextweb.com +0.0.0.0 www.cookingtiprewards.com +0.0.0.0 www.coolconcepts.nl +0.0.0.0 www.cool-premiums.com +0.0.0.0 www.cool-premiums-now.com +0.0.0.0 www.coolpremiumsnow.com +0.0.0.0 www.coolsavings.com +0.0.0.0 www.coreglead.co.uk +0.0.0.0 www.cosmeticscentre.uk.com +0.0.0.0 www.cpabank.com +0.0.0.0 www.cpmadvisors.com +0.0.0.0 www.crazypopups.com +0.0.0.0 www.crazywinnings.com +0.0.0.0 www.crediblegfj.info +0.0.0.0 www.crispads.com +0.0.0.0 www.crowdgravity.com +0.0.0.0 www.crowdignite.com +0.0.0.0 www.ctbdev.net +0.0.0.0 www.cyber-incentives.com +0.0.0.0 www.d03x2011.com +0.0.0.0 www.da-ads.com +0.0.0.0 www.daily-saver.com +0.0.0.0 www.datatech.es +0.0.0.0 www.datingadvertising.com +0.0.0.0 www.dctracking.com +0.0.0.0 www.depravedwhores.com +0.0.0.0 www.designbloxlive.com +0.0.0.0 www.dgmaustralia.com +0.0.0.0 www.dietoftoday.ca.pn +0.0.0.0 www.digimedia.com +0.0.0.0 www.directnetadvertising.net +0.0.0.0 www.directpowerrewards.com +0.0.0.0 www.dirtyrhino.com +0.0.0.0 www.discount-savings-more.com +0.0.0.0 www.djugoogs.com +0.0.0.0 www.dl-plugin.com +0.0.0.0 www.drowle.com +0.0.0.0 www.dutchsales.org +0.0.0.0 www.earnmygift.com +0.0.0.0 www.earnpointsandgifts.com +0.0.0.0 www.easyadservice.com +0.0.0.0 www.e-bannerx.com +0.0.0.0 www.ebaybanner.com +0.0.0.0 www.education-rewardpath.com +0.0.0.0 www.edu-offer.com +0.0.0.0 www.electronics-bonuspath.com +0.0.0.0 www.electronics-offer.net +0.0.0.0 www.electronicspresent.com +0.0.0.0 www.electronics-rewardpath.com +0.0.0.0 www.emailadvantagegroup.com +0.0.0.0 www.emailproductreview.com +0.0.0.0 www.emarketmakers.com +0.0.0.0 www.entertainment-rewardpath.com +0.0.0.0 www.entertainment-specials.com +0.0.0.0 www.eshopads2.com +0.0.0.0 www.euros4click.de +0.0.0.0 www.exclusivegiftcards.com +0.0.0.0 www.eyeblaster-bs.com +0.0.0.0 www.eyewonder.com #: Interactive Digital Advertising, Rich Media Ads, Flash Ads, Online Advertising +0.0.0.0 www.falkag.de +0.0.0.0 www.family-offer.com +0.0.0.0 www.fast-adv.it +0.0.0.0 www.fatcatrewards.com +0.0.0.0 www.feedjit.com +0.0.0.0 www.feedstermedia.com +0.0.0.0 www.fif49.info +0.0.0.0 www.finance-offer.com +0.0.0.0 www.finder.cox.net +0.0.0.0 www.fineclicks.com +0.0.0.0 www.flagcounter.com +0.0.0.0 www.flowers-offer.com +0.0.0.0 www.flu23.com +0.0.0.0 www.focalex.com +0.0.0.0 www.folloyu.com +0.0.0.0 www.food-drink-bonuspath.com +0.0.0.0 www.food-drink-rewardpath.com +0.0.0.0 www.foodmixeroffer.com +0.0.0.0 www.food-offer.com +0.0.0.0 www.fpctraffic2.com +0.0.0.0 www.freeadguru.com +0.0.0.0 www.freebiegb.co.uk +0.0.0.0 www.freecameraonus.com +0.0.0.0 www.freecameraprovider.com +0.0.0.0 www.freecamerasource.com +0.0.0.0 www.freecamerauk.co.uk +0.0.0.0 www.freecamsecrets.com +0.0.0.0 www.freecoolgift.com +0.0.0.0 www.freedesignerhandbagreviews.com +0.0.0.0 www.freedinnersource.com +0.0.0.0 www.freedvddept.com +0.0.0.0 www.freeelectronicscenter.com +0.0.0.0 www.freeelectronicsdepot.com +0.0.0.0 www.freeelectronicsonus.com +0.0.0.0 www.freeelectronicssource.com +0.0.0.0 www.freeentertainmentsource.com +0.0.0.0 www.freefoodprovider.com +0.0.0.0 www.freefoodsource.com +0.0.0.0 www.freefuelcard.com +0.0.0.0 www.freefuelcoupon.com +0.0.0.0 www.freegasonus.com +0.0.0.0 www.freegasprovider.com +0.0.0.0 www.free-gift-cards-now.com +0.0.0.0 www.freegiftcardsource.com +0.0.0.0 www.freegiftreward.com +0.0.0.0 www.free-gifts-comp.com +0.0.0.0 www.freeipodnanouk.co.uk +0.0.0.0 www.freeipoduk.com +0.0.0.0 www.freeipoduk.co.uk +0.0.0.0 www.freelaptopgift.com +0.0.0.0 www.freelaptopnation.com +0.0.0.0 www.free-laptop-reward.com +0.0.0.0 www.freelaptopreward.com +0.0.0.0 www.freelaptopwebsites.com +0.0.0.0 www.freenation.com +0.0.0.0 www.freeoffers-toys.com +0.0.0.0 www.freepayasyougotopupuk.co.uk +0.0.0.0 www.freeplasmanation.com +0.0.0.0 www.freerestaurantprovider.com +0.0.0.0 www.freerestaurantsource.com +0.0.0.0 www.freeshoppingprovider.com +0.0.0.0 www.freeshoppingsource.com +0.0.0.0 www.friendlyduck.com +0.0.0.0 www.frontpagecash.com +0.0.0.0 www.ftjcfx.com #commission junction +0.0.0.0 www.fusionbanners.com +0.0.0.0 www.gameconsolerewards.com +0.0.0.0 www.games-toys-bonuspath.com +0.0.0.0 www.games-toys-free.com +0.0.0.0 www.games-toys-rewardpath.com +0.0.0.0 www.gatoradvertisinginformationnetwork.com +0.0.0.0 www.getacool100.com +0.0.0.0 www.getacool500.com +0.0.0.0 www.getacoollaptop.com +0.0.0.0 www.getacooltv.com +0.0.0.0 www.getagiftonline.com +0.0.0.0 www.getloan.com +0.0.0.0 www.getmyfreebabystuff.com +0.0.0.0 www.getmyfreegear.com +0.0.0.0 www.getmyfreegiftcard.com +0.0.0.0 www.getmyfreelaptop.com +0.0.0.0 www.getmyfreelaptophere.com +0.0.0.0 www.getmyfreeplasma.com +0.0.0.0 www.getmylaptopfree.com +0.0.0.0 www.getmyplasmatv.com +0.0.0.0 www.getspecialgifts.com +0.0.0.0 www.getyourfreecomputer.com +0.0.0.0 www.getyourfreetv.com +0.0.0.0 www.giftcardchallenge.com +0.0.0.0 www.giftcardsurveys.us.com +0.0.0.0 www.giftrewardzone.com +0.0.0.0 www.gifts-flowers-rewardpath.com +0.0.0.0 www.gimmethatreward.com +0.0.0.0 www.gmads.net +0.0.0.0 www.go-free-gifts.com +0.0.0.0 www.gofreegifts.com +0.0.0.0 www.goody-garage.com +0.0.0.0 www.gopopup.com +0.0.0.0 www.grabbit-rabbit.com +0.0.0.0 www.greasypalm.com +0.0.0.0 www.grz67.com +0.0.0.0 www.guesstheview.com +0.0.0.0 www.guptamedianetwork.com +0.0.0.0 www.happydiscountspecials.com +0.0.0.0 www.healthbeautyncs.com +0.0.0.0 www.health-beauty-rewardpath.com +0.0.0.0 www.health-beauty-savingblvd.com +0.0.0.0 www.healthclicks.co.uk +0.0.0.0 www.hebdotop.com +0.0.0.0 www.hightrafficads.com +0.0.0.0 www.holiday-gift-offers.com +0.0.0.0 www.holidayproductpromo.com +0.0.0.0 www.holidayshoppingrewards.com +0.0.0.0 www.home4bizstart.ru +0.0.0.0 www.homeelectronicproducts.com +0.0.0.0 www.home-garden-premiumblvd.com +0.0.0.0 www.home-garden-rewardempire.com +0.0.0.0 www.home-garden-rewardpath.com +0.0.0.0 www.hooqy.com +0.0.0.0 www.hot-daily-deal.com +0.0.0.0 www.hotgiftzone.com +0.0.0.0 www.hotkeys.com +0.0.0.0 www.hot-product-hangout.com +0.0.0.0 www.idealcasino.net +0.0.0.0 www.idirect.com +0.0.0.0 www.iicdn.com +0.0.0.0 www.ijacko.net +0.0.0.0 www.ilovecheating.com +0.0.0.0 www.impressionaffiliate.com +0.0.0.0 www.impressionaffiliate.mobi +0.0.0.0 www.impressionlead.com +0.0.0.0 www.impressionperformance.biz +0.0.0.0 www.incentivegateway.com +0.0.0.0 www.incentiverewardcenter.com +0.0.0.0 www.incentive-scene.com +0.0.0.0 www.inckamedia.com +0.0.0.0 www.indiads.com +0.0.0.0 www.infinite-ads.com # www.shareactor.com +0.0.0.0 www.ins-offer.com +0.0.0.0 www.insurance-rewardpath.com +0.0.0.0 www.intela.com +0.0.0.0 www.interstitialzone.com +0.0.0.0 www.intnet-offer.com +0.0.0.0 www.invitefashion.com +0.0.0.0 www.is1.clixgalore.com +0.0.0.0 www.itrackerpro.com +0.0.0.0 www.itsfree123.com +0.0.0.0 www.iwantmyfreecash.com +0.0.0.0 www.iwantmy-freelaptop.com +0.0.0.0 www.iwantmyfree-laptop.com +0.0.0.0 www.iwantmyfreelaptop.com +0.0.0.0 www.iwantmygiftcard.com +0.0.0.0 www.jersey-offer.com +0.0.0.0 www.jetseeker.com +0.0.0.0 www.jivox.com +0.0.0.0 www.jl29jd25sm24mc29.com +0.0.0.0 www.joinfree.ro +0.0.0.0 www.jxliu.com +0.0.0.0 www.keybinary.com +0.0.0.0 www.keywordblocks.com +0.0.0.0 www.kitaramarketplace.com +0.0.0.0 www.kitaramedia.com +0.0.0.0 www.kitaratrk.com +0.0.0.0 www.kixer.com +0.0.0.0 www.kliksaya.com +0.0.0.0 www.kmdl101.com +0.0.0.0 www.kontera.com +0.0.0.0 www.konversation.com +0.0.0.0 www.kreaffiliation.com +0.0.0.0 www.kuhdi.com +0.0.0.0 www.ladyclicks.ru +0.0.0.0 www.laptopreportcard.com +0.0.0.0 www.laptoprewards.com +0.0.0.0 www.laptoprewardsgroup.com +0.0.0.0 www.laptoprewardszone.com +0.0.0.0 www.larivieracasino.com +0.0.0.0 www.lasthr.info +0.0.0.0 www.lduhtrp.net #commission junction +0.0.0.0 www.le1er.net +0.0.0.0 www.leadgreed.com +0.0.0.0 www.learning-offer.com +0.0.0.0 www.legal-rewardpath.com +0.0.0.0 www.leisure-offer.com +0.0.0.0 www.linkhut.com +0.0.0.0 www.linkpulse.com +0.0.0.0 www.linkwithin.com +0.0.0.0 www.liveinternet.ru +0.0.0.0 www.lottoforever.com +0.0.0.0 www.lpcloudsvr302.com +0.0.0.0 www.lucky-day-uk.com +0.0.0.0 www.macombdisplayads.com +0.0.0.0 www.marketing-rewardpath.com +0.0.0.0 www.mastertracks.be +0.0.0.0 www.maxbounty.com +0.0.0.0 www.mb01.com +0.0.0.0 www.media2.travelzoo.com +0.0.0.0 www.media-motor.com +0.0.0.0 www.medical-offer.com +0.0.0.0 www.medical-rewardpath.com +0.0.0.0 www.merchantapp.com +0.0.0.0 www.merlin.co.il +0.0.0.0 www.mgid.com +0.0.0.0 www.mightymagoo.com +0.0.0.0 www.mktg-offer.com +0.0.0.0 www.mlntracker.com +0.0.0.0 www.mochibot.com +0.0.0.0 www.morefreecamsecrets.com +0.0.0.0 www.morevisits.info +0.0.0.0 www.mp3playersource.com +0.0.0.0 www.mpression.net +0.0.0.0 www.myadsl.co.za +0.0.0.0 www.myaffiliateprogram.com +0.0.0.0 www.myairbridge.com +0.0.0.0 www.mycashback.co.uk +0.0.0.0 www.mycelloffer.com +0.0.0.0 www.mychoicerewards.com +0.0.0.0 www.myexclusiverewards.com +0.0.0.0 www.myfreedinner.com +0.0.0.0 www.myfreegifts.co.uk +0.0.0.0 www.myfreemp3player.com +0.0.0.0 www.mygiftcardcenter.com +0.0.0.0 www.mygreatrewards.com +0.0.0.0 www.myoffertracking.com +0.0.0.0 www.my-reward-channel.com +0.0.0.0 www.my-rewardsvault.com +0.0.0.0 www.myseostats.com +0.0.0.0 www.my-stats.com +0.0.0.0 www.myuitm.com +0.0.0.0 www.myusersonline.com +0.0.0.0 www.na47.com +0.0.0.0 www.nationalissuepanel.com +0.0.0.0 www.nationalsurveypanel.com +0.0.0.0 www.nctracking.com +0.0.0.0 www.nearbyad.com +0.0.0.0 www.needadvertising.com +0.0.0.0 www.neptuneads.com +0.0.0.0 www.netpalnow.com +0.0.0.0 www.netpaloffers.net +0.0.0.0 www.news6health.com +0.0.0.0 www.newssourceoftoday.com +0.0.0.0 www.nospartenaires.com +0.0.0.0 www.nothing-but-value.com +0.0.0.0 www.nysubwayoffer.com +0.0.0.0 www.offerx.co.uk +0.0.0.0 www.oinadserve.com +0.0.0.0 www.onlinebestoffers.net +0.0.0.0 www.ontheweb.com +0.0.0.0 www.opendownload.de +0.0.0.0 www.openload.de +0.0.0.0 www.optiad.net +0.0.0.0 www.paperg.com +0.0.0.0 www.parsads.com +0.0.0.0 www.pathforpoints.com +0.0.0.0 www.paypopup.com +0.0.0.0 www.people-choice-sites.com +0.0.0.0 www.personalcare-offer.com +0.0.0.0 www.personalcashbailout.com +0.0.0.0 www.phoenixads.co.in +0.0.0.0 www.pick-savings.com +0.0.0.0 www.plasmatv4free.com +0.0.0.0 www.plasmatvreward.com +0.0.0.0 www.politicalopinionsurvey.com +0.0.0.0 www.poponclick.com +0.0.0.0 www.popupad.net +0.0.0.0 www.popupdomination.com +0.0.0.0 www.popuptraffic.com +0.0.0.0 www.postmasterbannernet.com +0.0.0.0 www.postmasterdirect.com +0.0.0.0 www.postnewsads.com +0.0.0.0 www.premiumholidayoffers.com +0.0.0.0 www.premiumproductsonline.com +0.0.0.0 www.premium-reward-club.com +0.0.0.0 www.prizes.co.uk +0.0.0.0 www.probabilidades.net +0.0.0.0 www.productopinionpanel.com +0.0.0.0 www.productresearchpanel.com +0.0.0.0 www.producttestpanel.com +0.0.0.0 www.psclicks.com +0.0.0.0 www.pubdirecte.com +0.0.0.0 www.qitrck.com +0.0.0.0 www.quickbrowsersearch.com +0.0.0.0 www.radiate.com +0.0.0.0 www.rankyou.com +0.0.0.0 www.ravel-rewardpath.com +0.0.0.0 www.recreation-leisure-rewardpath.com +0.0.0.0 www.regflow.com +0.0.0.0 www.registrarads.com +0.0.0.0 www.resolvingserver.com +0.0.0.0 www.rewardblvd.com +0.0.0.0 www.rewardhotspot.com +0.0.0.0 www.rewardsflow.com +0.0.0.0 www.ringtonepartner.com +0.0.0.0 www.romepartners.com +0.0.0.0 www.roulettebotplus.com +0.0.0.0 www.rovion.com +0.0.0.0 www.rscounter10.com +0.0.0.0 www.rtcode.com +0.0.0.0 www.rwpads.net +0.0.0.0 www.sa44.net +0.0.0.0 www.salesonline.ie +0.0.0.0 www.save-plan.com +0.0.0.0 www.savings-specials.com +0.0.0.0 www.savings-time.com +0.0.0.0 www.scoremygift.com +0.0.0.0 www.screen-mates.com +0.0.0.0 www.searchwe.com +0.0.0.0 www.seasonalsamplerspecials.com +0.0.0.0 www.securecontactinfo.com +0.0.0.0 www.securerunner.com +0.0.0.0 www.servedby.advertising.com +0.0.0.0 www.sexpartnerx.com +0.0.0.0 www.sexsponsors.com +0.0.0.0 www.shareasale.com +0.0.0.0 www.share-server.com +0.0.0.0 www.shc-rebates.com +0.0.0.0 www.shopperpromotions.com +0.0.0.0 www.shoppingjobshere.com +0.0.0.0 www.shopping-offer.com +0.0.0.0 www.shoppingsiterewards.com +0.0.0.0 www.shops-malls-rewardpath.com +0.0.0.0 www.shoptosaveenergy.com +0.0.0.0 www.simpli.fi +0.0.0.0 www.sizzle-savings.com +0.0.0.0 www.smartadserver.com +0.0.0.0 www.smart-scripts.com +0.0.0.0 www.smarttargetting.com +0.0.0.0 www.smokersopinionpoll.com +0.0.0.0 www.smspop.com +0.0.0.0 www.sochr.com +0.0.0.0 www.sociallypublish.com +0.0.0.0 www.soongu.info +0.0.0.0 www.specialgiftrewards.com +0.0.0.0 www.specialonlinegifts.com +0.0.0.0 www.specials-rewardpath.com +0.0.0.0 www.speedboink.com +0.0.0.0 www.speedyclick.com +0.0.0.0 www.spinbox.com +0.0.0.0 www.sponsorads.de +0.0.0.0 www.sponsoradulto.com +0.0.0.0 www.sports-bonuspath.com +0.0.0.0 www.sports-fitness-rewardpath.com +0.0.0.0 www.sports-offer.com +0.0.0.0 www.sports-offer.net +0.0.0.0 www.sports-premiumblvd.com +0.0.0.0 www.sq2trk2.com +0.0.0.0 www.star-advertising.com +0.0.0.0 www.subsitesadserver.co.uk +0.0.0.0 www.sudokuwhiz.com +0.0.0.0 www.superbrewards.com +0.0.0.0 www.supremeadsonline.com +0.0.0.0 www.surplus-suppliers.com +0.0.0.0 www.sweetsforfree.com +0.0.0.0 www.symbiosting.com +0.0.0.0 www.syncaccess.net +0.0.0.0 www.system-live-media.cz +0.0.0.0 www.tcimg.com +0.0.0.0 www.textbanners.net +0.0.0.0 www.text-link-ads.com +0.0.0.0 www.textsrv.com +0.0.0.0 www.tgpmanager.com +0.0.0.0 www.thatrendsystem.com +0.0.0.0 www.the-binary-options-guide.com +0.0.0.0 www.the-binary-theorem.com +0.0.0.0 www.the-path-gateway.com +0.0.0.0 www.the-smart-stop.com +0.0.0.0 www.thetraderinpajamas.com +0.0.0.0 www.theuseful.com +0.0.0.0 www.theuseful.net +0.0.0.0 www.thinktarget.com +0.0.0.0 www.thinlaptoprewards.com +0.0.0.0 www.thoughtfully-free.com +0.0.0.0 www.thruport.com +0.0.0.0 www.tons-to-see.com +0.0.0.0 www.top20free.com +0.0.0.0 www.topbrandrewards.com +0.0.0.0 www.topconsumergifts.com +0.0.0.0 www.topdemaroc.com +0.0.0.0 www.toy-offer.com +0.0.0.0 www.toy-offer.net +0.0.0.0 www.tqlkg.com #commission junction +0.0.0.0 www.trackadvertising.net +0.0.0.0 www.tracklead.net +0.0.0.0 www.trafficrevenue.net +0.0.0.0 www.traffictrader.net +0.0.0.0 www.traffictraders.com +0.0.0.0 www.trafsearchonline.com +0.0.0.0 www.traktum.com +0.0.0.0 www.traveladvertising.com +0.0.0.0 www.travel-leisure-bonuspath.com +0.0.0.0 www.travel-leisure-premiumblvd.com +0.0.0.0 www.traveller-offer.com +0.0.0.0 www.traveller-offer.net +0.0.0.0 www.travelncs.com +0.0.0.0 www.treeloot.com +0.0.0.0 www.trendnews.com +0.0.0.0 www.trendsonline.biz +0.0.0.0 www.trendsonline.me +0.0.0.0 www.trendsonline.mobi +0.0.0.0 www.trndsys.mobi +0.0.0.0 www.tutop.com +0.0.0.0 www.tuttosessogratis.org +0.0.0.0 www.ukbanners.com +0.0.0.0 www.uleadstrk.com +0.0.0.0 www.ultimatefashiongifts.com +0.0.0.0 www.uproar.com +0.0.0.0 www.upsellit.com +0.0.0.0 www.usatravel-specials.com +0.0.0.0 www.usatravel-specials.net +0.0.0.0 www.us-choicevalue.com +0.0.0.0 www.usemax.de +0.0.0.0 www.us-topsites.com +0.0.0.0 www.utarget.co.uk +0.0.0.0 www.valueclick.com +0.0.0.0 www.via22.net +0.0.0.0 www.vibrantmedia.com +0.0.0.0 www.video-game-rewards-central.com +0.0.0.0 www.videogamerewardscentral.com +0.0.0.0 www.view4cash.de +0.0.0.0 www.virtumundo.com +0.0.0.0 www.vmcsatellite.com +0.0.0.0 www.wdm29.com +0.0.0.0 www.webcashvideos.com +0.0.0.0 www.webcompteur.com +0.0.0.0 www.webservices-rewardpath.com +0.0.0.0 www.websponsors.com +0.0.0.0 www.wegetpaid.net +0.0.0.0 www.whatuwhatuwhatuwant.com +0.0.0.0 www.widgetbucks.com +0.0.0.0 www.wigetmedia.com +0.0.0.0 www.williamhill.es +0.0.0.0 www.windaily.com +0.0.0.0 www.winnerschoiceservices.com +0.0.0.0 www.wordplaywhiz.com +0.0.0.0 www.work-offer.com +0.0.0.0 www.worry-free-savings.com +0.0.0.0 www.wppluginspro.com +0.0.0.0 www.wtp101.com +0.0.0.0 www.xbn.ru # exclusive banner network (Russian) +0.0.0.0 www.yceml.net +0.0.0.0 www.yibaruxet.cn +0.0.0.0 www.yieldmanager.net +0.0.0.0 www.youfck.com +0.0.0.0 www.yourdvdplayer.com +0.0.0.0 www.yourfreegascard.com +0.0.0.0 www.yourgascards.com +0.0.0.0 www.yourgiftrewards.com +0.0.0.0 www.your-gift-zone.com +0.0.0.0 www.yourgiftzone.com +0.0.0.0 www.yourhandytips.com +0.0.0.0 www.yourhotgiftzone.com +0.0.0.0 www.youripad4free.com +0.0.0.0 www.yourrewardzone.com +0.0.0.0 www.yoursmartrewards.com +0.0.0.0 www.zemgo.com +0.0.0.0 www.zevents.com +0.0.0.0 x86adserve006.adtech.de +0.0.0.0 xads.zedo.com +0.0.0.0 x.azjmp.com +0.0.0.0 x.iasrv.com +0.0.0.0 x.interia.pl +0.0.0.0 xlonhcld.xlontech.net +0.0.0.0 xml.adtech.de +0.0.0.0 xml.adtech.fr +0.0.0.0 xml.adtech.us +0.0.0.0 xml.click9.com +0.0.0.0 x.mochiads.com +0.0.0.0 xpantivirus.com +0.0.0.0 xpcs.ads.yahoo.com +0.0.0.0 xstatic.nk-net.pl +0.0.0.0 yadro.ru +0.0.0.0 y.cdn.adblade.com +0.0.0.0 yieldmanagement.adbooth.net +0.0.0.0 yieldmanager.net +0.0.0.0 ym.adnxs.com +0.0.0.0 yodleeinc.tt.omtrdc.net +0.0.0.0 youfck.com +0.0.0.0 yourdvdplayer.com +0.0.0.0 yourfreegascard.com +0.0.0.0 your-free-iphone.com +0.0.0.0 yourgascards.com +0.0.0.0 yourgiftrewards.com +0.0.0.0 your-gift-zone.com +0.0.0.0 yourgiftzone.com +0.0.0.0 yourhandytips.com +0.0.0.0 yourhotgiftzone.com +0.0.0.0 youripad4free.com +0.0.0.0 yourrewardzone.com +0.0.0.0 yoursmartrewards.com +0.0.0.0 ypn-js.overture.com +0.0.0.0 ysiu.freenation.com +0.0.0.0 ytaahg.vo.llnwd.net +0.0.0.0 yumenetworks.com +0.0.0.0 yx-in-f108.1e100.net +0.0.0.0 z1.adserver.com +0.0.0.0 zads.zedo.com +0.0.0.0 z.blogads.com +0.0.0.0 z.ceotrk.com +0.0.0.0 zdads.e-media.com +0.0.0.0 zeevex-online.com +0.0.0.0 zemgo.com +0.0.0.0 zevents.com +0.0.0.0 zuzzer5.com +# + +# + +# yahoo banner ads +0.0.0.0 eur.a1.yimg.com +0.0.0.0 in.yimg.com +0.0.0.0 sg.yimg.com +0.0.0.0 uk.i1.yimg.com +0.0.0.0 us.a1.yimg.com +0.0.0.0 us.b1.yimg.com +0.0.0.0 us.c1.yimg.com +0.0.0.0 us.d1.yimg.com +0.0.0.0 us.e1.yimg.com +0.0.0.0 us.f1.yimg.com +0.0.0.0 us.g1.yimg.com +0.0.0.0 us.h1.yimg.com +#0.0.0.0 us.i1.yimg.com #Uncomment this to block yahoo images +0.0.0.0 us.j1.yimg.com +0.0.0.0 us.k1.yimg.com +0.0.0.0 us.l1.yimg.com +0.0.0.0 us.m1.yimg.com +0.0.0.0 us.n1.yimg.com +0.0.0.0 us.o1.yimg.com +0.0.0.0 us.p1.yimg.com +0.0.0.0 us.q1.yimg.com +0.0.0.0 us.r1.yimg.com +0.0.0.0 us.s1.yimg.com +0.0.0.0 us.t1.yimg.com +0.0.0.0 us.u1.yimg.com +0.0.0.0 us.v1.yimg.com +0.0.0.0 us.w1.yimg.com +0.0.0.0 us.x1.yimg.com +0.0.0.0 us.y1.yimg.com +0.0.0.0 us.z1.yimg.com +# + +# + +# hitbox.com web bugs +0.0.0.0 1cgi.hitbox.com +0.0.0.0 2cgi.hitbox.com +0.0.0.0 adminec1.hitbox.com +0.0.0.0 ads.hitbox.com +0.0.0.0 ag1.hitbox.com +0.0.0.0 ahbn1.hitbox.com +0.0.0.0 ahbn2.hitbox.com +0.0.0.0 ahbn3.hitbox.com +0.0.0.0 ahbn4.hitbox.com +0.0.0.0 aibg.hitbox.com +0.0.0.0 aibl.hitbox.com +0.0.0.0 aics.hitbox.com +0.0.0.0 ai.hitbox.com +0.0.0.0 aiui.hitbox.com +0.0.0.0 bigip1.hitbox.com +0.0.0.0 bigip2.hitbox.com +0.0.0.0 blowfish.hitbox.com +0.0.0.0 cdb.hitbox.com +0.0.0.0 cgi.hitbox.com +0.0.0.0 counter2.hitbox.com +0.0.0.0 counter.hitbox.com +0.0.0.0 dev101.hitbox.com +0.0.0.0 dev102.hitbox.com +0.0.0.0 dev103.hitbox.com +0.0.0.0 dev.hitbox.com +0.0.0.0 download.hitbox.com +0.0.0.0 ec1.hitbox.com +0.0.0.0 ehg-247internet.hitbox.com +0.0.0.0 ehg-accuweather.hitbox.com +0.0.0.0 ehg-acdsystems.hitbox.com +0.0.0.0 ehg-adeptscience.hitbox.com +0.0.0.0 ehg-affinitynet.hitbox.com +0.0.0.0 ehg-aha.hitbox.com +0.0.0.0 ehg-amerix.hitbox.com +0.0.0.0 ehg-apcc.hitbox.com +0.0.0.0 ehg-associatenewmedia.hitbox.com +0.0.0.0 ehg-ati.hitbox.com +0.0.0.0 ehg-attenza.hitbox.com +0.0.0.0 ehg-autodesk.hitbox.com +0.0.0.0 ehg-baa.hitbox.com +0.0.0.0 ehg-backweb.hitbox.com +0.0.0.0 ehg-bestbuy.hitbox.com +0.0.0.0 ehg-bizjournals.hitbox.com +0.0.0.0 ehg-bmwna.hitbox.com +0.0.0.0 ehg-boschsiemens.hitbox.com +0.0.0.0 ehg-bskyb.hitbox.com +0.0.0.0 ehg-cafepress.hitbox.com +0.0.0.0 ehg-careerbuilder.hitbox.com +0.0.0.0 ehg-cbc.hitbox.com +0.0.0.0 ehg-cbs.hitbox.com +0.0.0.0 ehg-cbsradio.hitbox.com +0.0.0.0 ehg-cedarpoint.hitbox.com +0.0.0.0 ehg-clearchannel.hitbox.com +0.0.0.0 ehg-closetmaid.hitbox.com +0.0.0.0 ehg-commjun.hitbox.com +0.0.0.0 ehg.commjun.hitbox.com +0.0.0.0 ehg-communityconnect.hitbox.com +0.0.0.0 ehg-communityconnet.hitbox.com +0.0.0.0 ehg-comscore.hitbox.com +0.0.0.0 ehg-corusentertainment.hitbox.com +0.0.0.0 ehg-coverityinc.hitbox.com +0.0.0.0 ehg-crain.hitbox.com +0.0.0.0 ehg-ctv.hitbox.com +0.0.0.0 ehg-cygnusbm.hitbox.com +0.0.0.0 ehg-datamonitor.hitbox.com +0.0.0.0 ehg-digg.hitbox.com +0.0.0.0 ehg-dig.hitbox.com +0.0.0.0 ehg-eckounlimited.hitbox.com +0.0.0.0 ehg-esa.hitbox.com +0.0.0.0 ehg-espn.hitbox.com +0.0.0.0 ehg-fifa.hitbox.com +0.0.0.0 ehg-findlaw.hitbox.com +0.0.0.0 ehg-foundation.hitbox.com +0.0.0.0 ehg-foxsports.hitbox.com +0.0.0.0 ehg-futurepub.hitbox.com +0.0.0.0 ehg-gamedaily.hitbox.com +0.0.0.0 ehg-gamespot.hitbox.com +0.0.0.0 ehg-gatehousemedia.hitbox.com +0.0.0.0 ehg-gatehoussmedia.hitbox.com +0.0.0.0 ehg-glam.hitbox.com +0.0.0.0 ehg-groceryworks.hitbox.com +0.0.0.0 ehg-groupernetworks.hitbox.com +0.0.0.0 ehg-guardian.hitbox.com +0.0.0.0 ehg-hasbro.hitbox.com +0.0.0.0 ehg-hellodirect.hitbox.com +0.0.0.0 ehg-himedia.hitbox.com +0.0.0.0 ehg.hitbox.com +0.0.0.0 ehg-hitent.hitbox.com +0.0.0.0 ehg-hollywood.hitbox.com +0.0.0.0 ehg-idgentertainment.hitbox.com +0.0.0.0 ehg-idg.hitbox.com +0.0.0.0 ehg-ifilm.hitbox.com +0.0.0.0 ehg-ignitemedia.hitbox.com +0.0.0.0 ehg-intel.hitbox.com +0.0.0.0 ehg-ittoolbox.hitbox.com +0.0.0.0 ehg-itworldcanada.hitbox.com +0.0.0.0 ehg-kingstontechnology.hitbox.com +0.0.0.0 ehg-knightridder.hitbox.com +0.0.0.0 ehg-learningco.hitbox.com +0.0.0.0 ehg-legonewyorkinc.hitbox.com +0.0.0.0 ehg-liveperson.hitbox.com +0.0.0.0 ehg-macpublishingllc.hitbox.com +0.0.0.0 ehg-macromedia.hitbox.com +0.0.0.0 ehg-magicalia.hitbox.com +0.0.0.0 ehg-maplesoft.hitbox.com +0.0.0.0 ehg-mgnlimited.hitbox.com +0.0.0.0 ehg-mindshare.hitbox.com +0.0.0.0 ehg.mindshare.hitbox.com +0.0.0.0 ehg-mtv.hitbox.com +0.0.0.0 ehg-mybc.hitbox.com +0.0.0.0 ehg-newarkinone.hitbox.com.hitbox.com +0.0.0.0 ehg-newegg.hitbox.com +0.0.0.0 ehg-newscientist.hitbox.com +0.0.0.0 ehg-newsinternational.hitbox.com +0.0.0.0 ehg-nokiafin.hitbox.com +0.0.0.0 ehg-novell.hitbox.com +0.0.0.0 ehg-nvidia.hitbox.com +0.0.0.0 ehg-oreilley.hitbox.com +0.0.0.0 ehg-oreilly.hitbox.com +0.0.0.0 ehg-pacifictheatres.hitbox.com +0.0.0.0 ehg-pennwell.hitbox.com +0.0.0.0 ehg-peoplesoft.hitbox.com +0.0.0.0 ehg-philipsvheusen.hitbox.com +0.0.0.0 ehg-pizzahut.hitbox.com +0.0.0.0 ehg-playboy.hitbox.com +0.0.0.0 ehg-presentigsolutions.hitbox.com +0.0.0.0 ehg-qualcomm.hitbox.com +0.0.0.0 ehg-quantumcorp.hitbox.com +0.0.0.0 ehg-randomhouse.hitbox.com +0.0.0.0 ehg-redherring.hitbox.com +0.0.0.0 ehg-register.hitbox.com +0.0.0.0 ehg-researchinmotion.hitbox.com +0.0.0.0 ehg-rfa.hitbox.com +0.0.0.0 ehg-rodale.hitbox.com +0.0.0.0 ehg-salesforce.hitbox.com +0.0.0.0 ehg-salonmedia.hitbox.com +0.0.0.0 ehg-samsungusa.hitbox.com +0.0.0.0 ehg-seca.hitbox.com +0.0.0.0 ehg-shoppersdrugmart.hitbox.com +0.0.0.0 ehg-sonybssc.hitbox.com +0.0.0.0 ehg-sonycomputer.hitbox.com +0.0.0.0 ehg-sonyelec.hitbox.com +0.0.0.0 ehg-sonymusic.hitbox.com +0.0.0.0 ehg-sonyny.hitbox.com +0.0.0.0 ehg-space.hitbox.com +0.0.0.0 ehg-sportsline.hitbox.com +0.0.0.0 ehg-streamload.hitbox.com +0.0.0.0 ehg-superpages.hitbox.com +0.0.0.0 ehg-techtarget.hitbox.com +0.0.0.0 ehg-tfl.hitbox.com +0.0.0.0 ehg-thefirstchurchchrist.hitbox.com +0.0.0.0 ehg-tigerdirect2.hitbox.com +0.0.0.0 ehg-tigerdirect.hitbox.com +0.0.0.0 ehg-topps.hitbox.com +0.0.0.0 ehg-tribute.hitbox.com +0.0.0.0 ehg-tumbleweed.hitbox.com +0.0.0.0 ehg-ubisoft.hitbox.com +0.0.0.0 ehg-uniontrib.hitbox.com +0.0.0.0 ehg-usnewsworldreport.hitbox.com +0.0.0.0 ehg-verizoncommunications.hitbox.com +0.0.0.0 ehg-viacom.hitbox.com +0.0.0.0 ehg-vmware.hitbox.com +0.0.0.0 ehg-vonage.hitbox.com +0.0.0.0 ehg-wachovia.hitbox.com +0.0.0.0 ehg-wacomtechnology.hitbox.com +0.0.0.0 ehg-warner-brothers.hitbox.com +0.0.0.0 ehg-wizardsofthecoast.hitbox.com.hitbox.com +0.0.0.0 ehg-womanswallstreet.hitbox.com +0.0.0.0 ehg-wss.hitbox.com +0.0.0.0 ehg-xxolympicwintergames.hitbox.com +0.0.0.0 ehg-yellowpages.hitbox.com +0.0.0.0 ehg-youtube.hitbox.com +0.0.0.0 ejs.hitbox.com +0.0.0.0 enterprise-admin.hitbox.com +0.0.0.0 enterprise.hitbox.com +0.0.0.0 esg.hitbox.com +0.0.0.0 evwr.hitbox.com +0.0.0.0 get.hitbox.com +0.0.0.0 hg10.hitbox.com +0.0.0.0 hg11.hitbox.com +0.0.0.0 hg12.hitbox.com +0.0.0.0 hg13.hitbox.com +0.0.0.0 hg14.hitbox.com +0.0.0.0 hg15.hitbox.com +0.0.0.0 hg16.hitbox.com +0.0.0.0 hg17.hitbox.com +0.0.0.0 hg1.hitbox.com +0.0.0.0 hg2.hitbox.com +0.0.0.0 hg3.hitbox.com +0.0.0.0 hg4.hitbox.com +0.0.0.0 hg5.hitbox.com +0.0.0.0 hg6a.hitbox.com +0.0.0.0 hg6.hitbox.com +0.0.0.0 hg7.hitbox.com +0.0.0.0 hg8.hitbox.com +0.0.0.0 hg9.hitbox.com +0.0.0.0 hitboxbenchmarker.com +0.0.0.0 hitboxcentral.com +0.0.0.0 hitbox.com +0.0.0.0 hitboxenterprise.com +0.0.0.0 hitboxwireless.com +0.0.0.0 host6.hitbox.com +0.0.0.0 ias2.hitbox.com +0.0.0.0 ias.hitbox.com +0.0.0.0 ibg.hitbox.com +0.0.0.0 ics.hitbox.com +0.0.0.0 idb.hitbox.com +0.0.0.0 js1.hitbox.com +0.0.0.0 lb.hitbox.com +0.0.0.0 lesbian-erotica.hitbox.com +0.0.0.0 lookup2.hitbox.com +0.0.0.0 lookup.hitbox.com +0.0.0.0 mrtg.hitbox.com +0.0.0.0 myhitbox.com +0.0.0.0 na.hitbox.com +0.0.0.0 narwhal.hitbox.com +0.0.0.0 nei.hitbox.com +0.0.0.0 nocboard.hitbox.com +0.0.0.0 noc.hitbox.com +0.0.0.0 noc-request.hitbox.com +0.0.0.0 ns1.hitbox.com +0.0.0.0 oas.hitbox.com +0.0.0.0 phg.hitbox.com +0.0.0.0 pure.hitbox.com +0.0.0.0 rainbowclub.hitbox.com +0.0.0.0 rd1.hitbox.com +0.0.0.0 reseller.hitbox.com +0.0.0.0 resources.hitbox.com +0.0.0.0 sitesearch.hitbox.com +0.0.0.0 specialtyclub.hitbox.com +0.0.0.0 ss.hitbox.com +0.0.0.0 stage101.hitbox.com +0.0.0.0 stage102.hitbox.com +0.0.0.0 stage103.hitbox.com +0.0.0.0 stage104.hitbox.com +0.0.0.0 stage105.hitbox.com +0.0.0.0 stage.hitbox.com +0.0.0.0 stats2.hitbox.com +0.0.0.0 stats3.hitbox.com +0.0.0.0 stats.hitbox.com +0.0.0.0 switch10.hitbox.com +0.0.0.0 switch11.hitbox.com +0.0.0.0 switch1.hitbox.com +0.0.0.0 switch5.hitbox.com +0.0.0.0 switch6.hitbox.com +0.0.0.0 switch8.hitbox.com +0.0.0.0 switch9.hitbox.com +0.0.0.0 switch.hitbox.com +0.0.0.0 tetra.hitbox.com +0.0.0.0 tools2.hitbox.com +0.0.0.0 toolsa.hitbox.com +0.0.0.0 tools.hitbox.com +0.0.0.0 ts1.hitbox.com +0.0.0.0 ts2.hitbox.com +0.0.0.0 vwr1.hitbox.com +0.0.0.0 vwr2.hitbox.com +0.0.0.0 vwr3.hitbox.com +0.0.0.0 w100.hitbox.com +0.0.0.0 w101.hitbox.com +0.0.0.0 w102.hitbox.com +0.0.0.0 w103.hitbox.com +0.0.0.0 w104.hitbox.com +0.0.0.0 w105.hitbox.com +0.0.0.0 w106.hitbox.com +0.0.0.0 w107.hitbox.com +0.0.0.0 w108.hitbox.com +0.0.0.0 w109.hitbox.com +0.0.0.0 w10.hitbox.com +0.0.0.0 w110.hitbox.com +0.0.0.0 w111.hitbox.com +0.0.0.0 w112.hitbox.com +0.0.0.0 w113.hitbox.com +0.0.0.0 w114.hitbox.com +0.0.0.0 w115.hitbox.com +0.0.0.0 w116.hitbox.com +0.0.0.0 w117.hitbox.com +0.0.0.0 w118.hitbox.com +0.0.0.0 w119.hitbox.com +0.0.0.0 w11.hitbox.com +0.0.0.0 w120.hitbox.com +0.0.0.0 w121.hitbox.com +0.0.0.0 w122.hitbox.com +0.0.0.0 w123.hitbox.com +0.0.0.0 w124.hitbox.com +0.0.0.0 w126.hitbox.com +0.0.0.0 w128.hitbox.com +0.0.0.0 w129.hitbox.com +0.0.0.0 w12.hitbox.com +0.0.0.0 w130.hitbox.com +0.0.0.0 w131.hitbox.com +0.0.0.0 w132.hitbox.com +0.0.0.0 w133.hitbox.com +0.0.0.0 w135.hitbox.com +0.0.0.0 w136.hitbox.com +0.0.0.0 w137.hitbox.com +0.0.0.0 w138.hitbox.com +0.0.0.0 w139.hitbox.com +0.0.0.0 w13.hitbox.com +0.0.0.0 w140.hitbox.com +0.0.0.0 w141.hitbox.com +0.0.0.0 w144.hitbox.com +0.0.0.0 w147.hitbox.com +0.0.0.0 w14.hitbox.com +0.0.0.0 w153.hitbox.com +0.0.0.0 w154.hitbox.com +0.0.0.0 w155.hitbox.com +0.0.0.0 w157.hitbox.com +0.0.0.0 w159.hitbox.com +0.0.0.0 w15.hitbox.com +0.0.0.0 w161.hitbox.com +0.0.0.0 w162.hitbox.com +0.0.0.0 w167.hitbox.com +0.0.0.0 w168.hitbox.com +0.0.0.0 w16.hitbox.com +0.0.0.0 w170.hitbox.com +0.0.0.0 w175.hitbox.com +0.0.0.0 w177.hitbox.com +0.0.0.0 w179.hitbox.com +0.0.0.0 w17.hitbox.com +0.0.0.0 w18.hitbox.com +0.0.0.0 w19.hitbox.com +0.0.0.0 w1.hitbox.com +0.0.0.0 w20.hitbox.com +0.0.0.0 w21.hitbox.com +0.0.0.0 w22.hitbox.com +0.0.0.0 w23.hitbox.com +0.0.0.0 w24.hitbox.com +0.0.0.0 w25.hitbox.com +0.0.0.0 w26.hitbox.com +0.0.0.0 w27.hitbox.com +0.0.0.0 w28.hitbox.com +0.0.0.0 w29.hitbox.com +0.0.0.0 w2.hitbox.com +0.0.0.0 w30.hitbox.com +0.0.0.0 w31.hitbox.com +0.0.0.0 w32.hitbox.com +0.0.0.0 w33.hitbox.com +0.0.0.0 w34.hitbox.com +0.0.0.0 w35.hitbox.com +0.0.0.0 w36.hitbox.com +0.0.0.0 w3.hitbox.com +0.0.0.0 w4.hitbox.com +0.0.0.0 w5.hitbox.com +0.0.0.0 w6.hitbox.com +0.0.0.0 w7.hitbox.com +0.0.0.0 w8.hitbox.com +0.0.0.0 w9.hitbox.com +0.0.0.0 webload101.hitbox.com +0.0.0.0 wss-gw-1.hitbox.com +0.0.0.0 wss-gw-3.hitbox.com +0.0.0.0 wvwr1.hitbox.com +0.0.0.0 ww1.hitbox.com +0.0.0.0 ww2.hitbox.com +0.0.0.0 ww3.hitbox.com +0.0.0.0 wwa.hitbox.com +0.0.0.0 wwb.hitbox.com +0.0.0.0 wwc.hitbox.com +0.0.0.0 wwd.hitbox.com +0.0.0.0 www.ehg-rr.hitbox.com +0.0.0.0 www.hitbox.com +0.0.0.0 www.hitboxwireless.com +0.0.0.0 y2k.hitbox.com +0.0.0.0 yang.hitbox.com +0.0.0.0 ying.hitbox.com +# + +# + +# www.extreme-dm.com tracking +0.0.0.0 extreme-dm.com +0.0.0.0 reports.extreme-dm.com +0.0.0.0 t0.extreme-dm.com +0.0.0.0 t1.extreme-dm.com +0.0.0.0 t.extreme-dm.com +0.0.0.0 u0.extreme-dm.com +0.0.0.0 u1.extreme-dm.com +0.0.0.0 u.extreme-dm.com +0.0.0.0 v0.extreme-dm.com +0.0.0.0 v1.extreme-dm.com +0.0.0.0 v.extreme-dm.com +0.0.0.0 w0.extreme-dm.com +0.0.0.0 w1.extreme-dm.com +0.0.0.0 w2.extreme-dm.com +0.0.0.0 w3.extreme-dm.com +0.0.0.0 w4.extreme-dm.com +0.0.0.0 w5.extreme-dm.com +0.0.0.0 w6.extreme-dm.com +0.0.0.0 w7.extreme-dm.com +0.0.0.0 w8.extreme-dm.com +0.0.0.0 w9.extreme-dm.com +0.0.0.0 w.extreme-dm.com +0.0.0.0 www.extreme-dm.com +0.0.0.0 x3.extreme-dm.com +0.0.0.0 y0.extreme-dm.com +0.0.0.0 y1.extreme-dm.com +0.0.0.0 y.extreme-dm.com +0.0.0.0 z0.extreme-dm.com +0.0.0.0 z1.extreme-dm.com +0.0.0.0 z.extreme-dm.com +# + +# + +# realmedia.com's Open Ad Stream +0.0.0.0 ap.oasfile.aftenposten.no +0.0.0.0 imagenen1.247realmedia.com +0.0.0.0 oacentral.cepro.com +0.0.0.0 oas.adx.nu +0.0.0.0 oas.aurasports.com +0.0.0.0 oas.benchmark.fr +0.0.0.0 oasc03012.247realmedia.com +0.0.0.0 oasc03049.247realmedia.com +0.0.0.0 oasc06006.247realmedia.com +0.0.0.0 oasc08008.247realmedia.com +0.0.0.0 oasc09.247realmedia.com +0.0.0.0 oascentral.123greetings.com +0.0.0.0 oascentral.abclocal.go.com +0.0.0.0 oascentral.adage.com +0.0.0.0 oascentral.adageglobal.com +0.0.0.0 oascentral.aircanada.com +0.0.0.0 oascentral.alanicnewsnet.ca +0.0.0.0 oascentral.alanticnewsnet.ca +0.0.0.0 oascentral.americanheritage.com +0.0.0.0 oascentral.artistdirect.com +0.0.0.0 oascentral.artistirect.com +0.0.0.0 oascentral.askmen.com +0.0.0.0 oascentral.aviationnow.com +0.0.0.0 oascentral.blackenterprises.com +0.0.0.0 oascentral.blogher.org +0.0.0.0 oascentral.bostonherald.com +0.0.0.0 oascentral.bostonphoenix.com +0.0.0.0 oascentral.businessinsider.com +0.0.0.0 oascentral.businessweek.com +0.0.0.0 oascentral.businessweeks.com +0.0.0.0 oascentral.buy.com +0.0.0.0 oascentral.canadaeast.com +0.0.0.0 oascentral.canadianliving.com +0.0.0.0 oascentral.charleston.net +0.0.0.0 oascentral.chicagobusiness.com +0.0.0.0 oascentral.chron.com +0.0.0.0 oascentral.citypages.com +0.0.0.0 oascentral.clearchannel.com +0.0.0.0 oascentral.comcast.net +0.0.0.0 oascentral.comics.com +0.0.0.0 oascentral.construction.com +0.0.0.0 oascentral.consumerreports.org +0.0.0.0 oascentral.covers.com +0.0.0.0 oascentral.crainsdetroit.com +0.0.0.0 oascentral.crimelibrary.com +0.0.0.0 oascentral.cybereps.com +0.0.0.0 oascentral.dailybreeze.com +0.0.0.0 oascentral.dailyherald.com +0.0.0.0 oascentral.dilbert.com +0.0.0.0 oascentral.discovery.com +0.0.0.0 oascentral.drphil.com +0.0.0.0 oascentral.eastbayexpress.com +0.0.0.0 oas-central.east.realmedia.com +0.0.0.0 oascentral.encyclopedia.com +0.0.0.0 oascentral.fashionmagazine.com +0.0.0.0 oascentral.fayettevillenc.com +0.0.0.0 oascentral.feedroom.com +0.0.0.0 oascentral.forsythnews.com +0.0.0.0 oascentral.fortunecity.com +0.0.0.0 oascentral.foxnews.com +0.0.0.0 oascentral.freedom.com +0.0.0.0 oascentral.g4techtv.com +0.0.0.0 oascentral.ggl.com +0.0.0.0 oascentral.gigex.com +0.0.0.0 oascentral.globalpost.com +0.0.0.0 oascentral.hamptonroads.com +0.0.0.0 oascentral.hamptoroads.com +0.0.0.0 oascentral.hamtoroads.com +0.0.0.0 oascentral.herenb.com +0.0.0.0 oascentral.hollywood.com +0.0.0.0 oascentral.houstonpress.com +0.0.0.0 oascentral.inq7.net +0.0.0.0 oascentral.investors.com +0.0.0.0 oascentral.investorwords.com +0.0.0.0 oascentral.itbusiness.ca +0.0.0.0 oascentral.killsometime.com +0.0.0.0 oascentral.laptopmag.com +0.0.0.0 oascentral.law.com +0.0.0.0 oascentral.laweekly.com +0.0.0.0 oascentral.looksmart.com +0.0.0.0 oascentral.lycos.com +0.0.0.0 oascentral.mailtribune.com +0.0.0.0 oascentral.mayoclinic.com +0.0.0.0 oascentral.medbroadcast.com +0.0.0.0 oascentral.metro.us +0.0.0.0 oascentral.minnpost.com +0.0.0.0 oascentral.mochila.com +0.0.0.0 oascentral.motherjones.com +0.0.0.0 oascentral.nerve.com +0.0.0.0 oascentral.newsmax.com +0.0.0.0 oascentral.nowtoronto.com +0.0.0.0 oascentralnx.comcast.net +0.0.0.0 oascentral.onwisconsin.com +0.0.0.0 oascentral.phoenixnewtimes.com +0.0.0.0 oascentral.phoenixvillenews.com +0.0.0.0 oascentral.pitch.com +0.0.0.0 oascentral.poconorecord.com +0.0.0.0 oascentral.politico.com +0.0.0.0 oascentral.post-gazette.com +0.0.0.0 oascentral.pottsmerc.com +0.0.0.0 oascentral.princetonreview.com +0.0.0.0 oascentral.publicradio.org +0.0.0.0 oascentral.radaronline.com +0.0.0.0 oascentral.rcrnews.com +0.0.0.0 oas-central.realmedia.com +0.0.0.0 oascentral.redherring.com +0.0.0.0 oascentral.redorbit.com +0.0.0.0 oascentral.redstate.com +0.0.0.0 oascentral.reference.com +0.0.0.0 oascentral.regalinterative.com +0.0.0.0 oascentral.register.com +0.0.0.0 oascentral.registerguard.com +0.0.0.0 oascentral.registguard.com +0.0.0.0 oascentral.riverfronttimes.com +0.0.0.0 oascentral.salon.com +0.0.0.0 oascentral.santacruzsentinel.com +0.0.0.0 oascentral.sciam.com +0.0.0.0 oascentral.scientificamerican.com +0.0.0.0 oascentral.seacoastonline.com +0.0.0.0 oascentral.seattleweekly.com +0.0.0.0 oascentral.sfgate.com +0.0.0.0 oascentral.sfweekly.com +0.0.0.0 oascentral.sina.com +0.0.0.0 oascentral.sina.com.hk +0.0.0.0 oascentral.sparknotes.com +0.0.0.0 oascentral.sptimes.com +0.0.0.0 oascentral.starbulletin.com +0.0.0.0 oascentral.suntimes.com +0.0.0.0 oascentral.surfline.com +0.0.0.0 oascentral.thechronicleherald.ca +0.0.0.0 oascentral.thehockeynews.com +0.0.0.0 oascentral.thenation.com +0.0.0.0 oascentral.theonionavclub.com +0.0.0.0 oascentral.theonion.com +0.0.0.0 oascentral.thephoenix.com +0.0.0.0 oascentral.thesmokinggun.com +0.0.0.0 oascentral.thespark.com +0.0.0.0 oascentral.tmcnet.com +0.0.0.0 oascentral.tnr.com +0.0.0.0 oascentral.tourismvancouver.com +0.0.0.0 oascentral.townhall.com +0.0.0.0 oascentral.tribe.net +0.0.0.0 oascentral.trutv.com +0.0.0.0 oascentral.upi.com +0.0.0.0 oascentral.urbanspoon.com +0.0.0.0 oascentral.villagevoice.com +0.0.0.0 oascentral.virtualtourist.com +0.0.0.0 oascentral.warcry.com +0.0.0.0 oascentral.washtimes.com +0.0.0.0 oascentral.wciv.com +0.0.0.0 oascentral.westword.com +0.0.0.0 oascentral.where.ca +0.0.0.0 oascentral.wjla.com +0.0.0.0 oascentral.wkrn.com +0.0.0.0 oascentral.wwe.com +0.0.0.0 oascentral.yellowpages.com +0.0.0.0 oascentral.ywlloewpages.ca +0.0.0.0 oascentral.zwire.com +0.0.0.0 oascentreal.adcritic.com +0.0.0.0 oascetral.laweekly.com +0.0.0.0 oas.dispatch.com +0.0.0.0 oas.foxnews.com +0.0.0.0 oas.greensboro.com +0.0.0.0 oas.guardian.co.uk +0.0.0.0 oas.ibnlive.com +0.0.0.0 oas.lee.net +0.0.0.0 oas.nrjlink.fr +0.0.0.0 oas.nzz.ch +0.0.0.0 oas.portland.com +0.0.0.0 oas.publicitas.ch +0.0.0.0 oasroanoke.com +0.0.0.0 oas.salon.com +0.0.0.0 oas.sciencemag.org +0.0.0.0 oas.signonsandiego.com +0.0.0.0 oas.startribune.com +0.0.0.0 oas.toronto.com +0.0.0.0 oas.uniontrib.com +0.0.0.0 oas.villagevoice.com +0.0.0.0 oas.vtsgonline.com +# + +# + +# fastclick banner ads +0.0.0.0 media1.fastclick.net +0.0.0.0 media2.fastclick.net +0.0.0.0 media3.fastclick.net +0.0.0.0 media4.fastclick.net +0.0.0.0 media5.fastclick.net +0.0.0.0 media6.fastclick.net +0.0.0.0 media7.fastclick.net +0.0.0.0 media8.fastclick.net +0.0.0.0 media9.fastclick.net +0.0.0.0 media10.fastclick.net +0.0.0.0 media11.fastclick.net +0.0.0.0 media12.fastclick.net +0.0.0.0 media13.fastclick.net +0.0.0.0 media14.fastclick.net +0.0.0.0 media15.fastclick.net +0.0.0.0 media16.fastclick.net +0.0.0.0 media17.fastclick.net +0.0.0.0 media18.fastclick.net +0.0.0.0 media19.fastclick.net +0.0.0.0 media20.fastclick.net +0.0.0.0 media21.fastclick.net +0.0.0.0 media22.fastclick.net +0.0.0.0 media23.fastclick.net +0.0.0.0 media24.fastclick.net +0.0.0.0 media25.fastclick.net +0.0.0.0 media26.fastclick.net +0.0.0.0 media27.fastclick.net +0.0.0.0 media28.fastclick.net +0.0.0.0 media29.fastclick.net +0.0.0.0 media30.fastclick.net +0.0.0.0 media31.fastclick.net +0.0.0.0 media32.fastclick.net +0.0.0.0 media33.fastclick.net +0.0.0.0 media34.fastclick.net +0.0.0.0 media35.fastclick.net +0.0.0.0 media36.fastclick.net +0.0.0.0 media37.fastclick.net +0.0.0.0 media38.fastclick.net +0.0.0.0 media39.fastclick.net +0.0.0.0 media40.fastclick.net +0.0.0.0 media41.fastclick.net +0.0.0.0 media42.fastclick.net +0.0.0.0 media43.fastclick.net +0.0.0.0 media44.fastclick.net +0.0.0.0 media45.fastclick.net +0.0.0.0 media46.fastclick.net +0.0.0.0 media47.fastclick.net +0.0.0.0 media48.fastclick.net +0.0.0.0 media49.fastclick.net +0.0.0.0 media50.fastclick.net +0.0.0.0 media51.fastclick.net +0.0.0.0 media52.fastclick.net +0.0.0.0 media53.fastclick.net +0.0.0.0 media54.fastclick.net +0.0.0.0 media55.fastclick.net +0.0.0.0 media56.fastclick.net +0.0.0.0 media57.fastclick.net +0.0.0.0 media58.fastclick.net +0.0.0.0 media59.fastclick.net +0.0.0.0 media60.fastclick.net +0.0.0.0 media61.fastclick.net +0.0.0.0 media62.fastclick.net +0.0.0.0 media63.fastclick.net +0.0.0.0 media64.fastclick.net +0.0.0.0 media65.fastclick.net +0.0.0.0 media66.fastclick.net +0.0.0.0 media67.fastclick.net +0.0.0.0 media68.fastclick.net +0.0.0.0 media69.fastclick.net +0.0.0.0 media70.fastclick.net +0.0.0.0 media71.fastclick.net +0.0.0.0 media72.fastclick.net +0.0.0.0 media73.fastclick.net +0.0.0.0 media74.fastclick.net +0.0.0.0 media75.fastclick.net +0.0.0.0 media76.fastclick.net +0.0.0.0 media77.fastclick.net +0.0.0.0 media78.fastclick.net +0.0.0.0 media79.fastclick.net +0.0.0.0 media80.fastclick.net +0.0.0.0 media81.fastclick.net +0.0.0.0 media82.fastclick.net +0.0.0.0 media83.fastclick.net +0.0.0.0 media84.fastclick.net +0.0.0.0 media85.fastclick.net +0.0.0.0 media86.fastclick.net +0.0.0.0 media87.fastclick.net +0.0.0.0 media88.fastclick.net +0.0.0.0 media89.fastclick.net +0.0.0.0 media90.fastclick.net +0.0.0.0 media91.fastclick.net +0.0.0.0 media92.fastclick.net +0.0.0.0 media93.fastclick.net +0.0.0.0 media94.fastclick.net +0.0.0.0 media95.fastclick.net +0.0.0.0 media96.fastclick.net +0.0.0.0 media97.fastclick.net +0.0.0.0 media98.fastclick.net +0.0.0.0 media99.fastclick.net +0.0.0.0 fastclick.net +# + +# + +# belo interactive ads +0.0.0.0 te.about.com +0.0.0.0 te.adlandpro.com +0.0.0.0 te.advance.net +0.0.0.0 te.ap.org +0.0.0.0 te.astrology.com +0.0.0.0 te.audiencematch.net +0.0.0.0 te.belointeractive.com +0.0.0.0 te.boston.com +0.0.0.0 te.businessweek.com +0.0.0.0 te.chicagotribune.com +0.0.0.0 te.chron.com +0.0.0.0 te.cleveland.net +0.0.0.0 te.ctnow.com +0.0.0.0 te.dailycamera.com +0.0.0.0 te.dailypress.com +0.0.0.0 te.dentonrc.com +0.0.0.0 te.greenwichtime.com +0.0.0.0 te.idg.com +0.0.0.0 te.infoworld.com +0.0.0.0 te.ivillage.com +0.0.0.0 te.journalnow.com +0.0.0.0 te.latimes.com +0.0.0.0 te.mcall.com +0.0.0.0 te.mgnetwork.com +0.0.0.0 te.mysanantonio.com +0.0.0.0 te.newsday.com +0.0.0.0 te.nytdigital.com +0.0.0.0 te.orlandosentinel.com +0.0.0.0 te.scripps.com +0.0.0.0 te.scrippsnetworksprivacy.com +0.0.0.0 te.scrippsnewspapersprivacy.com +0.0.0.0 te.sfgate.com +0.0.0.0 te.signonsandiego.com +0.0.0.0 te.stamfordadvocate.com +0.0.0.0 te.sun-sentinel.com +0.0.0.0 te.sunspot.net +0.0.0.0 te.suntimes.com +0.0.0.0 te.tbo.com +0.0.0.0 te.thestar.ca +0.0.0.0 te.thestar.com +0.0.0.0 te.trb.com +0.0.0.0 te.versiontracker.com +0.0.0.0 te.wsls.com +# + +# + +# popup traps -- sites that bounce you around or won't let you leave +0.0.0.0 24hwebsex.com +0.0.0.0 adultfriendfinder.com +0.0.0.0 all-tgp.org +0.0.0.0 fioe.info +0.0.0.0 incestland.com +0.0.0.0 lesview.com +0.0.0.0 searchforit.com +0.0.0.0 www.asiansforu.com +0.0.0.0 www.bangbuddy.com +0.0.0.0 www.datanotary.com +0.0.0.0 www.entercasino.com +0.0.0.0 www.incestdot.com +0.0.0.0 www.incestgold.com +0.0.0.0 www.justhookup.com +0.0.0.0 www.mangayhentai.com +0.0.0.0 www.myluvcrush.ca +0.0.0.0 www.ourfuckbook.com +0.0.0.0 www.realincestvideos.com +0.0.0.0 www.searchforit.com +0.0.0.0 www.searchv.com +0.0.0.0 www.secretosx.com +0.0.0.0 www.seductiveamateurs.com +0.0.0.0 www.smsmovies.net +0.0.0.0 www.wowjs.1www.cn +0.0.0.0 www.xxxnations.com +0.0.0.0 www.xxxnightly.com +0.0.0.0 www.xxxtoolbar.com +0.0.0.0 www.yourfuckbook.com +# + +# + +# malicious e-card -- these sites send out mass quantities of spam + # and some distribute adware and spyware +0.0.0.0 123greetings.com # contains one link to distributor of adware or spyware +0.0.0.0 2000greetings.com +0.0.0.0 celebwelove.com +0.0.0.0 ecard4all.com +0.0.0.0 eforu.com +0.0.0.0 freewebcards.com +0.0.0.0 fukkad.com +0.0.0.0 fun-e-cards.com +0.0.0.0 funnyreign.com # heavy spam (Site Advisor received 1075 e-mails/week) +0.0.0.0 funsilly.com +0.0.0.0 myfuncards.com +0.0.0.0 www.cool-downloads.com +0.0.0.0 www.cool-downloads.net +0.0.0.0 www.friend-card.com +0.0.0.0 www.friend-cards.com +0.0.0.0 www.friend-cards.net +0.0.0.0 www.friend-greeting.com +0.0.0.0 www.friend-greetings.com +0.0.0.0 www.friendgreetings.com +0.0.0.0 www.friend-greetings.net +0.0.0.0 www.friendgreetings.net +0.0.0.0 www.laugh-mail.com +0.0.0.0 www.laugh-mail.net +# + +# + +# European network of tracking sites +0.0.0.0 0ivwbox.de +0.0.0.0 1ivwbox.de +0.0.0.0 1und1.ivwbox.de +0.0.0.0 2ivwbox.de +0.0.0.0 3ivwbox.de +0.0.0.0 4ivwbox.de +0.0.0.0 5ivwbox.de +0.0.0.0 6ivwbox.de +0.0.0.0 7ivwbox.de +0.0.0.0 8ivwbox.de +0.0.0.0 8vwbox.de +0.0.0.0 9ivwbox.de +0.0.0.0 9vwbox.de +0.0.0.0 aivwbox.de +0.0.0.0 avwbox.de +0.0.0.0 bild.ivwbox.de +0.0.0.0 bivwbox.de +0.0.0.0 civwbox.de +0.0.0.0 divwbox.de +0.0.0.0 eevwbox.de +0.0.0.0 eivwbox.de +0.0.0.0 evwbox.de +0.0.0.0 faz.ivwbox.de +0.0.0.0 fivwbox.de +0.0.0.0 givwbox.de +0.0.0.0 hivwbox.de +0.0.0.0 i8vwbox.de +0.0.0.0 i9vwbox.de +0.0.0.0 iavwbox.de +0.0.0.0 ibvwbox.de +0.0.0.0 ibwbox.de +0.0.0.0 icvwbox.de +0.0.0.0 icwbox.de +0.0.0.0 ievwbox.de +0.0.0.0 ifvwbox.de +0.0.0.0 ifwbox.de +0.0.0.0 igvwbox.de +0.0.0.0 igwbox.de +0.0.0.0 iivwbox.de +0.0.0.0 ijvwbox.de +0.0.0.0 ikvwbox.de +0.0.0.0 iovwbox.de +0.0.0.0 iuvwbox.de +0.0.0.0 iv2box.de +0.0.0.0 iv2wbox.de +0.0.0.0 iv3box.de +0.0.0.0 iv3wbox.de +0.0.0.0 ivabox.de +0.0.0.0 ivawbox.de +0.0.0.0 ivbox.de +0.0.0.0 ivbwbox.de +0.0.0.0 ivbwox.de +0.0.0.0 ivcwbox.de +0.0.0.0 ivebox.de +0.0.0.0 ivewbox.de +0.0.0.0 ivfwbox.de +0.0.0.0 ivgwbox.de +0.0.0.0 ivqbox.de +0.0.0.0 ivqwbox.de +0.0.0.0 ivsbox.de +0.0.0.0 ivswbox.de +0.0.0.0 ivvbox.de +0.0.0.0 ivvwbox.de +0.0.0.0 ivw2box.de +0.0.0.0 ivw3box.de +0.0.0.0 ivwabox.de +0.0.0.0 ivwb0ox.de +0.0.0.0 ivwb0x.de +0.0.0.0 ivwb9ox.de +0.0.0.0 ivwb9x.de +0.0.0.0 ivwbaox.de +0.0.0.0 ivwbax.de +0.0.0.0 ivwbbox.de +0.0.0.0 ivwbeox.de +0.0.0.0 ivwbex.de +0.0.0.0 ivwbgox.de +0.0.0.0 ivwbhox.de +0.0.0.0 ivwbiox.de +0.0.0.0 ivwbix.de +0.0.0.0 ivwbkox.de +0.0.0.0 ivwbkx.de +0.0.0.0 ivwblox.de +0.0.0.0 ivwblx.de +0.0.0.0 ivwbnox.de +0.0.0.0 ivwbo0x.de +0.0.0.0 ivwbo9x.de +0.0.0.0 ivwboax.de +0.0.0.0 ivwboc.de +0.0.0.0 ivwbock.de +0.0.0.0 ivwbocx.de +0.0.0.0 ivwbod.de +0.0.0.0 ivwbo.de +0.0.0.0 ivwbodx.de +0.0.0.0 ivwboex.de +0.0.0.0 ivwboix.de +0.0.0.0 ivwboks.de +0.0.0.0 ivwbokx.de +0.0.0.0 ivwbolx.de +0.0.0.0 ivwboox.de +0.0.0.0 ivwbopx.de +0.0.0.0 ivwbos.de +0.0.0.0 ivwbosx.de +0.0.0.0 ivwboux.de +0.0.0.0 ivwbox0.de +0.0.0.0 ivwbox1.de +0.0.0.0 ivwbox2.de +0.0.0.0 ivwbox3.de +0.0.0.0 ivwbox4.de +0.0.0.0 ivwbox5.de +0.0.0.0 ivwbox6.de +0.0.0.0 ivwbox7.de +0.0.0.0 ivwbox8.de +0.0.0.0 ivwbox9.de +0.0.0.0 ivwboxa.de +0.0.0.0 ivwboxb.de +0.0.0.0 ivwboxc.de +0.0.0.0 ivwboxd.de +0.0.0.0 ivwbox.de +0.0.0.0 ivwboxe.de +0.0.0.0 ivwboxes.de +0.0.0.0 ivwboxf.de +0.0.0.0 ivwboxg.de +0.0.0.0 ivwboxh.de +0.0.0.0 ivwboxi.de +0.0.0.0 ivwboxj.de +0.0.0.0 ivwboxk.de +0.0.0.0 ivwboxl.de +0.0.0.0 ivwboxm.de +0.0.0.0 ivwboxn.de +0.0.0.0 ivwboxo.de +0.0.0.0 ivwboxp.de +0.0.0.0 ivwboxq.de +0.0.0.0 ivwboxr.de +0.0.0.0 ivwboxs.de +0.0.0.0 ivwboxt.de +0.0.0.0 ivwboxu.de +0.0.0.0 ivwboxv.de +0.0.0.0 ivwboxw.de +0.0.0.0 ivwboxx.de +0.0.0.0 ivwboxy.de +0.0.0.0 ivwboxz.de +0.0.0.0 ivwboyx.de +0.0.0.0 ivwboz.de +0.0.0.0 ivwbozx.de +0.0.0.0 ivwbpox.de +0.0.0.0 ivwbpx.de +0.0.0.0 ivwbuox.de +0.0.0.0 ivwbux.de +0.0.0.0 ivwbvox.de +0.0.0.0 ivwbx.de +0.0.0.0 ivwbxo.de +0.0.0.0 ivwbyox.de +0.0.0.0 ivwbyx.de +0.0.0.0 ivwebox.de +0.0.0.0 ivwgbox.de +0.0.0.0 ivwgox.de +0.0.0.0 ivwhbox.de +0.0.0.0 ivwhox.de +0.0.0.0 ivwnbox.de +0.0.0.0 ivwnox.de +0.0.0.0 ivwobx.de +0.0.0.0 ivwox.de +0.0.0.0 ivwpbox.de +0.0.0.0 ivwpox.de +0.0.0.0 ivwqbox.de +0.0.0.0 ivwsbox.de +0.0.0.0 ivwvbox.de +0.0.0.0 ivwvox.de +0.0.0.0 ivwwbox.de +0.0.0.0 iwbox.de +0.0.0.0 iwvbox.de +0.0.0.0 iwvwbox.de +0.0.0.0 iwwbox.de +0.0.0.0 iyvwbox.de +0.0.0.0 jivwbox.de +0.0.0.0 jvwbox.de +0.0.0.0 kicker.ivwbox.de +0.0.0.0 kivwbox.de +0.0.0.0 kvwbox.de +0.0.0.0 livwbox.de +0.0.0.0 mivwbox.de +0.0.0.0 netzmarkt.ivwbox.de +0.0.0.0 nivwbox.de +0.0.0.0 ntv.ivwbox.de +0.0.0.0 oivwbox.de +0.0.0.0 onvis.ivwbox.de +0.0.0.0 ovwbox.de +0.0.0.0 pivwbox.de +0.0.0.0 qivwbox.de +0.0.0.0 rivwbox.de +0.0.0.0 sivwbox.de +0.0.0.0 spiegel.ivwbox.de +0.0.0.0 tivwbox.de +0.0.0.0 uivwbox.de +0.0.0.0 uvwbox.de +0.0.0.0 vivwbox.de +0.0.0.0 viwbox.de +0.0.0.0 vwbox.de +0.0.0.0 wivwbox.de +0.0.0.0 wwivwbox.de +0.0.0.0 www.0ivwbox.de +0.0.0.0 www.1ivwbox.de +0.0.0.0 www.2ivwbox.de +0.0.0.0 www.3ivwbox.de +0.0.0.0 www.4ivwbox.de +0.0.0.0 www.5ivwbox.de +0.0.0.0 www.6ivwbox.de +0.0.0.0 www.7ivwbox.de +0.0.0.0 www.8ivwbox.de +0.0.0.0 www.8vwbox.de +0.0.0.0 www.9ivwbox.de +0.0.0.0 www.9vwbox.de +0.0.0.0 www.aivwbox.de +0.0.0.0 www.avwbox.de +0.0.0.0 www.bivwbox.de +0.0.0.0 www.civwbox.de +0.0.0.0 www.divwbox.de +0.0.0.0 www.eevwbox.de +0.0.0.0 www.eivwbox.de +0.0.0.0 www.evwbox.de +0.0.0.0 www.fivwbox.de +0.0.0.0 www.givwbox.de +0.0.0.0 www.hivwbox.de +0.0.0.0 www.i8vwbox.de +0.0.0.0 www.i9vwbox.de +0.0.0.0 www.iavwbox.de +0.0.0.0 www.ibvwbox.de +0.0.0.0 www.ibwbox.de +0.0.0.0 www.icvwbox.de +0.0.0.0 www.icwbox.de +0.0.0.0 www.ievwbox.de +0.0.0.0 www.ifvwbox.de +0.0.0.0 www.ifwbox.de +0.0.0.0 www.igvwbox.de +0.0.0.0 www.igwbox.de +0.0.0.0 www.iivwbox.de +0.0.0.0 www.ijvwbox.de +0.0.0.0 www.ikvwbox.de +0.0.0.0 www.iovwbox.de +0.0.0.0 www.iuvwbox.de +0.0.0.0 www.iv2box.de +0.0.0.0 www.iv2wbox.de +0.0.0.0 www.iv3box.de +0.0.0.0 www.iv3wbox.de +0.0.0.0 www.ivabox.de +0.0.0.0 www.ivawbox.de +0.0.0.0 www.ivbox.de +0.0.0.0 www.ivbwbox.de +0.0.0.0 www.ivbwox.de +0.0.0.0 www.ivcwbox.de +0.0.0.0 www.ivebox.de +0.0.0.0 www.ivewbox.de +0.0.0.0 www.ivfwbox.de +0.0.0.0 www.ivgwbox.de +0.0.0.0 www.ivqbox.de +0.0.0.0 www.ivqwbox.de +0.0.0.0 www.ivsbox.de +0.0.0.0 www.ivswbox.de +0.0.0.0 www.ivvbox.de +0.0.0.0 www.ivvwbox.de +0.0.0.0 www.ivw2box.de +0.0.0.0 www.ivw3box.de +0.0.0.0 www.ivwabox.de +0.0.0.0 www.ivwb0ox.de +0.0.0.0 www.ivwb0x.de +0.0.0.0 www.ivwb9ox.de +0.0.0.0 www.ivwb9x.de +0.0.0.0 www.ivwbaox.de +0.0.0.0 www.ivwbax.de +0.0.0.0 www.ivwbbox.de +0.0.0.0 www.ivwbeox.de +0.0.0.0 www.ivwbex.de +0.0.0.0 www.ivwbgox.de +0.0.0.0 www.ivwbhox.de +0.0.0.0 www.ivwbiox.de +0.0.0.0 www.ivwbix.de +0.0.0.0 www.ivwbkox.de +0.0.0.0 www.ivwbkx.de +0.0.0.0 www.ivwblox.de +0.0.0.0 www.ivwblx.de +0.0.0.0 www.ivwbnox.de +0.0.0.0 www.ivwbo0x.de +0.0.0.0 www.ivwbo9x.de +0.0.0.0 www.ivwboax.de +0.0.0.0 www.ivwboc.de +0.0.0.0 www.ivwbock.de +0.0.0.0 www.ivwbocx.de +0.0.0.0 www.ivwbod.de +0.0.0.0 www.ivwbo.de +0.0.0.0 www.ivwbodx.de +0.0.0.0 www.ivwboex.de +0.0.0.0 www.ivwboix.de +0.0.0.0 www.ivwboks.de +0.0.0.0 www.ivwbokx.de +0.0.0.0 www.ivwbolx.de +0.0.0.0 www.ivwboox.de +0.0.0.0 www.ivwbopx.de +0.0.0.0 www.ivwbos.de +0.0.0.0 www.ivwbosx.de +0.0.0.0 www.ivwboux.de +0.0.0.0 www.ivwbox0.de +0.0.0.0 www.ivwbox1.de +0.0.0.0 www.ivwbox2.de +0.0.0.0 www.ivwbox3.de +0.0.0.0 www.ivwbox4.de +0.0.0.0 www.ivwbox5.de +0.0.0.0 www.ivwbox6.de +0.0.0.0 www.ivwbox7.de +0.0.0.0 www.ivwbox8.de +0.0.0.0 www.ivwbox9.de +0.0.0.0 www.ivwboxa.de +0.0.0.0 www.ivwboxb.de +0.0.0.0 www.ivwboxc.de +0.0.0.0 www.ivwboxd.de +0.0.0.0 www.ivwbox.de +0.0.0.0 wwwivwbox.de +0.0.0.0 www.ivwboxe.de +0.0.0.0 www.ivwboxes.de +0.0.0.0 www.ivwboxf.de +0.0.0.0 www.ivwboxg.de +0.0.0.0 www.ivwboxh.de +0.0.0.0 www.ivwboxi.de +0.0.0.0 www.ivwboxj.de +0.0.0.0 www.ivwboxk.de +0.0.0.0 www.ivwboxl.de +0.0.0.0 www.ivwboxm.de +0.0.0.0 www.ivwboxn.de +0.0.0.0 www.ivwboxo.de +0.0.0.0 www.ivwboxp.de +0.0.0.0 www.ivwboxq.de +0.0.0.0 www.ivwboxr.de +0.0.0.0 www.ivwboxs.de +0.0.0.0 www.ivwboxt.de +0.0.0.0 www.ivwboxu.de +0.0.0.0 www.ivwboxv.de +0.0.0.0 www.ivwboxw.de +0.0.0.0 www.ivwboxx.de +0.0.0.0 www.ivwboxy.de +0.0.0.0 www.ivwboxz.de +0.0.0.0 www.ivwboyx.de +0.0.0.0 www.ivwboz.de +0.0.0.0 www.ivwbozx.de +0.0.0.0 www.ivwbpox.de +0.0.0.0 www.ivwbpx.de +0.0.0.0 www.ivwbuox.de +0.0.0.0 www.ivwbux.de +0.0.0.0 www.ivwbvox.de +0.0.0.0 www.ivwbx.de +0.0.0.0 www.ivwbxo.de +0.0.0.0 www.ivwbyox.de +0.0.0.0 www.ivwbyx.de +0.0.0.0 www.ivwebox.de +0.0.0.0 www.ivwgbox.de +0.0.0.0 www.ivwgox.de +0.0.0.0 www.ivwhbox.de +0.0.0.0 www.ivwhox.de +0.0.0.0 www.ivwnbox.de +0.0.0.0 www.ivwnox.de +0.0.0.0 www.ivwobx.de +0.0.0.0 www.ivwox.de +0.0.0.0 www.ivwpbox.de +0.0.0.0 www.ivwpox.de +0.0.0.0 www.ivwqbox.de +0.0.0.0 www.ivwsbox.de +0.0.0.0 www.ivwvbox.de +0.0.0.0 www.ivwvox.de +0.0.0.0 www.ivwwbox.de +0.0.0.0 www.iwbox.de +0.0.0.0 www.iwvbox.de +0.0.0.0 www.iwvwbox.de +0.0.0.0 www.iwwbox.de +0.0.0.0 www.iyvwbox.de +0.0.0.0 www.jivwbox.de +0.0.0.0 www.jvwbox.de +0.0.0.0 www.kivwbox.de +0.0.0.0 www.kvwbox.de +0.0.0.0 www.livwbox.de +0.0.0.0 www.mivwbox.de +0.0.0.0 www.nivwbox.de +0.0.0.0 www.oivwbox.de +0.0.0.0 www.ovwbox.de +0.0.0.0 www.pivwbox.de +0.0.0.0 www.qivwbox.de +0.0.0.0 www.rivwbox.de +0.0.0.0 www.sivwbox.de +0.0.0.0 www.tivwbox.de +0.0.0.0 www.uivwbox.de +0.0.0.0 www.uvwbox.de +0.0.0.0 www.vivwbox.de +0.0.0.0 www.viwbox.de +0.0.0.0 www.vwbox.de +0.0.0.0 www.wivwbox.de +0.0.0.0 www.wwivwbox.de +0.0.0.0 www.wwwivwbox.de +0.0.0.0 www.xivwbox.de +0.0.0.0 www.yevwbox.de +0.0.0.0 www.yivwbox.de +0.0.0.0 www.yvwbox.de +0.0.0.0 www.zivwbox.de +0.0.0.0 xivwbox.de +0.0.0.0 yevwbox.de +0.0.0.0 yivwbox.de +0.0.0.0 yvwbox.de +0.0.0.0 zivwbox.de +# + +# + +# message board and wiki spam -- these sites are linked in + # message board spam and are unlikely to be real sites +0.0.0.0 10pg.scl5fyd.info +0.0.0.0 21jewelry.com +0.0.0.0 24x7.soliday.org +0.0.0.0 2site.com +0.0.0.0 33b.b33r.net +0.0.0.0 48.2mydns.net +0.0.0.0 4allfree.com +0.0.0.0 55.2myip.com +0.0.0.0 6165.rapidforum.com +0.0.0.0 6pg.ryf3hgf.info +0.0.0.0 7x7.ruwe.net +0.0.0.0 7x.cc +0.0.0.0 911.x24hr.com +0.0.0.0 ab.5.p2l.info +0.0.0.0 aboutharrypotter.fasthost.tv +0.0.0.0 aciphex.about-tabs.com +0.0.0.0 actonel.about-tabs.com +0.0.0.0 actos.about-tabs.com +0.0.0.0 acyclovir.1.p2l.info +0.0.0.0 adderall.ourtablets.com +0.0.0.0 adderallxr.freespaces.com +0.0.0.0 adipex.1.p2l.info +0.0.0.0 adipex.24sws.ws +0.0.0.0 adipex.3.p2l.info +0.0.0.0 adipex.4.p2l.info +0.0.0.0 adipex.hut1.ru +0.0.0.0 adipex.ourtablets.com +0.0.0.0 adipexp.3xforum.ro +0.0.0.0 adipex.shengen.ru +0.0.0.0 adipex.t-amo.net +0.0.0.0 adsearch.www1.biz +0.0.0.0 adult.shengen.ru +0.0.0.0 aguileranude.1stOK.com +0.0.0.0 ahh-teens.com +0.0.0.0 aid-golf-golfdust-training.tabrays.com +0.0.0.0 airline-ticket.gloses.net +0.0.0.0 air-plane-ticket.beesearch.info +0.0.0.0 ak.5.p2l.info +0.0.0.0 al.5.p2l.info +0.0.0.0 alcohol-treatment.gloses.net +0.0.0.0 allegra.1.p2l.info +0.0.0.0 allergy.1.p2l.info +0.0.0.0 all-sex.shengen.ru +0.0.0.0 alprazolamonline.findmenow.info +0.0.0.0 alprazolam.ourtablets.com +0.0.0.0 alyssamilano.1stOK.com +0.0.0.0 alyssamilano.ca.tt +0.0.0.0 alyssamilano.home.sapo.pt +0.0.0.0 amateur-mature-sex.adaltabaza.net +0.0.0.0 ambien.1.p2l.info +0.0.0.0 ambien.3.p2l.info +0.0.0.0 ambien.4.p2l.info +0.0.0.0 ambien.ourtablets.com +0.0.0.0 amoxicillin.ourtablets.com +0.0.0.0 angelinajolie.1stOK.com +0.0.0.0 angelinajolie.ca.tt +0.0.0.0 anklets.shengen.ru +0.0.0.0 annanicolesannanicolesmith.ca.tt +0.0.0.0 annanicolesmith.1stOK.com +0.0.0.0 antidepressants.1.p2l.info +0.0.0.0 anxiety.1.p2l.info +0.0.0.0 aol.spb.su +0.0.0.0 ar.5.p2l.info +0.0.0.0 arcade.ya.com +0.0.0.0 armanix.white.prohosting.com +0.0.0.0 arthritis.atspace.com +0.0.0.0 as.5.p2l.info +0.0.0.0 aspirin.about-tabs.com +0.0.0.0 ativan.ourtablets.com +0.0.0.0 austria-car-rental.findworm.net +0.0.0.0 auto.allewagen.de +0.0.0.0 az.5.p2l.info +0.0.0.0 azz.badazz.org +0.0.0.0 balabass.peerserver.com +0.0.0.0 balab.portx.net +0.0.0.0 bbs.ws +0.0.0.0 bc.5.p2l.info +0.0.0.0 beauty.finaltips.com +0.0.0.0 berkleynude.ca.tt +0.0.0.0 bestlolaray.com +0.0.0.0 bet-online.petrovka.info +0.0.0.0 betting-online.petrovka.info +0.0.0.0 bextra.ourtablets.com +0.0.0.0 bextra-store.shengen.ru +0.0.0.0 bingo-online.petrovka.info +0.0.0.0 birth-control.1.p2l.info +0.0.0.0 bontril.1.p2l.info +0.0.0.0 bontril.ourtablets.com +0.0.0.0 britneyspears.1stOK.com +0.0.0.0 britneyspears.ca.tt +0.0.0.0 br.rawcomm.net +0.0.0.0 bupropion-hcl.1.p2l.info +0.0.0.0 buspar.1.p2l.info +0.0.0.0 buspirone.1.p2l.info +0.0.0.0 butalbital-apap.1.p2l.info +0.0.0.0 buy-adipex.aca.ru +0.0.0.0 buy-adipex-cheap-adipex-online.com +0.0.0.0 buy-adipex.hut1.ru +0.0.0.0 buy-adipex.i-jogo.net +0.0.0.0 buy-adipex-online.md-online24.de +0.0.0.0 buy-adipex.petrovka.info +0.0.0.0 buy-carisoprodol.polybuild.ru +0.0.0.0 buy-cheap-phentermine.blogspot.com +0.0.0.0 buy-cheap-xanax.all.at +0.0.0.0 buy-cialis-cheap-cialis-online.info +0.0.0.0 buy-cialis.freewebtools.com +0.0.0.0 buycialisonline.7h.com +0.0.0.0 buycialisonline.bigsitecity.com +0.0.0.0 buy-cialis-online.iscool.nl +0.0.0.0 buy-cialis-online.meperdoe.net +0.0.0.0 buy-cialis.splinder.com +0.0.0.0 buy-diazepam.connect.to +0.0.0.0 buyfioricet.findmenow.info +0.0.0.0 buy-fioricet.hut1.ru +0.0.0.0 buyfioricetonline.7h.com +0.0.0.0 buyfioricetonline.bigsitecity.com +0.0.0.0 buyfioricetonline.freeservers.com +0.0.0.0 buy-flower.petrovka.info +0.0.0.0 buy-hydrocodone.aca.ru +0.0.0.0 buyhydrocodone.all.at +0.0.0.0 buy-hydrocodone-cheap-hydrocodone-online.com +0.0.0.0 buy-hydrocodone.este.ru +0.0.0.0 buyhydrocodoneonline.findmenow.info +0.0.0.0 buy-hydrocodone-online.tche.com +0.0.0.0 buy-hydrocodone.petrovka.info +0.0.0.0 buy-hydrocodone.polybuild.ru +0.0.0.0 buy-hydrocodone.quesaudade.net +0.0.0.0 buy-hydrocodone.scromble.com +0.0.0.0 buylevitra.3xforum.ro +0.0.0.0 buy-levitra-cheap-levitra-online.info +0.0.0.0 buylevitraonline.7h.com +0.0.0.0 buylevitraonline.bigsitecity.com +0.0.0.0 buy-lortab-cheap-lortab-online.com +0.0.0.0 buy-lortab.hut1.ru +0.0.0.0 buylortabonline.7h.com +0.0.0.0 buylortabonline.bigsitecity.com +0.0.0.0 buy-lortab-online.iscool.nl +0.0.0.0 buypaxilonline.7h.com +0.0.0.0 buypaxilonline.bigsitecity.com +0.0.0.0 buy-phentermine-cheap-phentermine-online.com +0.0.0.0 buy-phentermine.hautlynx.com +0.0.0.0 buy-phentermine-online.135.it +0.0.0.0 buyphentermineonline.7h.com +0.0.0.0 buyphentermineonline.bigsitecity.com +0.0.0.0 buy-phentermine-online.i-jogo.net +0.0.0.0 buy-phentermine-online.i-ltda.net +0.0.0.0 buy-phentermine.polybuild.ru +0.0.0.0 buy-phentermine.thepizza.net +0.0.0.0 buy-tamiflu.asian-flu-vaccine.com +0.0.0.0 buy-ultram-online.iscool.nl +0.0.0.0 buy-valium-cheap-valium-online.com +0.0.0.0 buy-valium.este.ru +0.0.0.0 buy-valium.hut1.ru +0.0.0.0 buy-valium.polybuild.ru +0.0.0.0 buyvalium.polybuild.ru +0.0.0.0 buy-viagra.aca.ru +0.0.0.0 buy-viagra.go.to +0.0.0.0 buy-viagra.polybuild.ru +0.0.0.0 buyviagra.polybuild.ru +0.0.0.0 buy-vicodin-cheap-vicodin-online.com +0.0.0.0 buy-vicodin.dd.vu +0.0.0.0 buy-vicodin.hut1.ru +0.0.0.0 buy-vicodin.iscool.nl +0.0.0.0 buy-vicodin-online.i-blog.net +0.0.0.0 buy-vicodin-online.seumala.net +0.0.0.0 buy-vicodin-online.supersite.fr +0.0.0.0 buyvicodinonline.veryweird.com +0.0.0.0 buy-xanax.aztecaonline.net +0.0.0.0 buy-xanax-cheap-xanax-online.com +0.0.0.0 buy-xanax.hut1.ru +0.0.0.0 buy-xanax-online.amovoce.net +0.0.0.0 buy-zyban.all.at +0.0.0.0 bx6.blrf.net +0.0.0.0 ca.5.p2l.info +0.0.0.0 camerondiaznude.1stOK.com +0.0.0.0 camerondiaznude.ca.tt +0.0.0.0 car-donation.shengen.ru +0.0.0.0 car-insurance.inshurance-from.com +0.0.0.0 carisoprodol.1.p2l.info +0.0.0.0 carisoprodol.hut1.ru +0.0.0.0 carisoprodol.ourtablets.com +0.0.0.0 carisoprodol.polybuild.ru +0.0.0.0 carisoprodol.shengen.ru +0.0.0.0 car-loan.shengen.ru +0.0.0.0 carmenelectra.1stOK.com +0.0.0.0 cash-advance.now-cash.com +0.0.0.0 casino-gambling-online.searchservice.info +0.0.0.0 casino-online.100gal.net +0.0.0.0 cat.onlinepeople.net +0.0.0.0 cc5f.dnyp.com +0.0.0.0 celebrex.1.p2l.info +0.0.0.0 celexa.1.p2l.info +0.0.0.0 celexa.3.p2l.info +0.0.0.0 celexa.4.p2l.info +0.0.0.0 cephalexin.ourtablets.com +0.0.0.0 charlizetheron.1stOK.com +0.0.0.0 cheap-adipex.hut1.ru +0.0.0.0 cheap-carisoprodol.polybuild.ru +0.0.0.0 cheap-hydrocodone.go.to +0.0.0.0 cheap-hydrocodone.polybuild.ru +0.0.0.0 cheap-phentermine.polybuild.ru +0.0.0.0 cheap-valium.polybuild.ru +0.0.0.0 cheap-viagra.polybuild.ru +0.0.0.0 cheap-web-hosting-here.blogspot.com +0.0.0.0 cheap-xanax-here.blogspot.com +0.0.0.0 cheapxanax.hut1.ru +0.0.0.0 cialis.1.p2l.info +0.0.0.0 cialis.3.p2l.info +0.0.0.0 cialis.4.p2l.info +0.0.0.0 cialis-finder.com +0.0.0.0 cialis-levitra-viagra.com.cn +0.0.0.0 cialis.ourtablets.com +0.0.0.0 cialis-store.shengen.ru +0.0.0.0 co.5.p2l.info +0.0.0.0 co.dcclan.co.uk +0.0.0.0 codeine.ourtablets.com +0.0.0.0 creampie.afdss.info +0.0.0.0 credit-card-application.now-cash.com +0.0.0.0 credit-cards.shengen.ru +0.0.0.0 ct.5.p2l.info +0.0.0.0 cuiland.info +0.0.0.0 cyclobenzaprine.1.p2l.info +0.0.0.0 cyclobenzaprine.ourtablets.com +0.0.0.0 dal.d.la +0.0.0.0 danger-phentermine.allforyourlife.com +0.0.0.0 darvocet.ourtablets.com +0.0.0.0 dc.5.p2l.info +0.0.0.0 de.5.p2l.info +0.0.0.0 debt.shengen.ru +0.0.0.0 def.5.p2l.info +0.0.0.0 demimoorenude.1stOK.com +0.0.0.0 deniserichards.1stOK.com +0.0.0.0 detox-kit.com +0.0.0.0 detox.shengen.ru +0.0.0.0 diazepam.ourtablets.com +0.0.0.0 diazepam.razma.net +0.0.0.0 diazepam.shengen.ru +0.0.0.0 didrex.1.p2l.info +0.0.0.0 diet-pills.hut1.ru +0.0.0.0 digital-cable-descrambler.planet-high-heels.com +0.0.0.0 dir.opank.com +0.0.0.0 dos.velek.com +0.0.0.0 drewbarrymore.ca.tt +0.0.0.0 drugdetox.shengen.ru +0.0.0.0 drug-online.petrovka.info +0.0.0.0 drug-testing.shengen.ru +0.0.0.0 eb.dd.bluelinecomputers.be +0.0.0.0 eb.prout.be +0.0.0.0 ed.at.is13.de +0.0.0.0 ed.at.thamaster.de +0.0.0.0 e-dot.hut1.ru +0.0.0.0 efam4.info +0.0.0.0 effexor-xr.1.p2l.info +0.0.0.0 e-hosting.hut1.ru +0.0.0.0 ei.imbucurator-de-prost.com +0.0.0.0 eminemticket.freespaces.com +0.0.0.0 en.dd.blueline.be +0.0.0.0 enpresse.1.p2l.info +0.0.0.0 en.ultrex.ru +0.0.0.0 epson-printer-ink.beesearch.info +0.0.0.0 erectile.byethost33.com +0.0.0.0 esgic.1.p2l.info +0.0.0.0 fahrrad.bikesshop.de +0.0.0.0 famous-pics.com +0.0.0.0 famvir.1.p2l.info +0.0.0.0 farmius.org +0.0.0.0 fee-hydrocodone.bebto.com +0.0.0.0 female-v.1.p2l.info +0.0.0.0 femaleviagra.findmenow.info +0.0.0.0 fg.softguy.com +0.0.0.0 findmenow.info +0.0.0.0 fioricet.1.p2l.info +0.0.0.0 fioricet.3.p2l.info +0.0.0.0 fioricet.4.p2l.info +0.0.0.0 fioricet-online.blogspot.com +0.0.0.0 firstfinda.info +0.0.0.0 fl.5.p2l.info +0.0.0.0 flexeril.1.p2l.info +0.0.0.0 flextra.1.p2l.info +0.0.0.0 flonase.1.p2l.info +0.0.0.0 flonase.3.p2l.info +0.0.0.0 flonase.4.p2l.info +0.0.0.0 florineff.ql.st +0.0.0.0 flower-online.petrovka.info +0.0.0.0 fluoxetine.1.p2l.info +0.0.0.0 fo4n.com +0.0.0.0 forex-broker.hut1.ru +0.0.0.0 forex-chart.hut1.ru +0.0.0.0 forex-market.hut1.ru +0.0.0.0 forex-news.hut1.ru +0.0.0.0 forex-online.hut1.ru +0.0.0.0 forex-signal.hut1.ru +0.0.0.0 forex-trade.hut1.ru +0.0.0.0 forex-trading-benefits.blogspot.com +0.0.0.0 forextrading.hut1.ru +0.0.0.0 freechat.llil.de +0.0.0.0 free.hostdepartment.com +0.0.0.0 free-money.host.sk +0.0.0.0 free-viagra.polybuild.ru +0.0.0.0 free-virus-scan.100gal.net +0.0.0.0 ga.5.p2l.info +0.0.0.0 game-online-video.petrovka.info +0.0.0.0 gaming-online.petrovka.info +0.0.0.0 gastrointestinal.1.p2l.info +0.0.0.0 gen-hydrocodone.polybuild.ru +0.0.0.0 getcarisoprodol.polybuild.ru +0.0.0.0 gocarisoprodol.polybuild.ru +0.0.0.0 gsm-mobile-phone.beesearch.info +0.0.0.0 gu.5.p2l.info +0.0.0.0 guerria-skateboard-tommy.tabrays.com +0.0.0.0 gwynethpaltrow.ca.tt +0.0.0.0 h1.ripway.com +0.0.0.0 hair-dos.resourcesarchive.com +0.0.0.0 halleberrynude.ca.tt +0.0.0.0 heathergraham.ca.tt +0.0.0.0 herpes.1.p2l.info +0.0.0.0 herpes.3.p2l.info +0.0.0.0 herpes.4.p2l.info +0.0.0.0 hf.themafia.info +0.0.0.0 hi.5.p2l.info +0.0.0.0 hi.pacehillel.org +0.0.0.0 holobumo.info +0.0.0.0 homehre.bravehost.com +0.0.0.0 homehre.ifrance.com +0.0.0.0 homehre.tripod.com +0.0.0.0 hoodia.kogaryu.com +0.0.0.0 hotel-las-vegas.gloses.net +0.0.0.0 hydrocodone-buy-online.blogspot.com +0.0.0.0 hydrocodone.irondel.swisshost.by +0.0.0.0 hydrocodone.on.to +0.0.0.0 hydrocodone.shengen.ru +0.0.0.0 hydrocodone.t-amo.net +0.0.0.0 hydrocodone.visa-usa.ru +0.0.0.0 hydro.polybuild.ru +0.0.0.0 ia.5.p2l.info +0.0.0.0 ia.warnet-thunder.net +0.0.0.0 ibm-notebook-battery.wp-club.net +0.0.0.0 id.5.p2l.info +0.0.0.0 il.5.p2l.info +0.0.0.0 imitrex.1.p2l.info +0.0.0.0 imitrex.3.p2l.info +0.0.0.0 imitrex.4.p2l.info +0.0.0.0 in.5.p2l.info +0.0.0.0 ionamin.1.p2l.info +0.0.0.0 ionamin.t35.com +0.0.0.0 irondel.swisshost.by +0.0.0.0 japanese-girl-xxx.com +0.0.0.0 java-games.bestxs.de +0.0.0.0 jg.hack-inter.net +0.0.0.0 job-online.petrovka.info +0.0.0.0 jobs-online.petrovka.info +0.0.0.0 kitchen-island.mensk.us +0.0.0.0 konstantin.freespaces.com +0.0.0.0 ks.5.p2l.info +0.0.0.0 ky.5.p2l.info +0.0.0.0 la.5.p2l.info +0.0.0.0 lamictal.about-tabs.com +0.0.0.0 lamisil.about-tabs.com +0.0.0.0 levitra.1.p2l.info +0.0.0.0 levitra.3.p2l.info +0.0.0.0 levitra.4.p2l.info +0.0.0.0 lexapro.1.p2l.info +0.0.0.0 lexapro.3.p2l.info +0.0.0.0 lexapro.4.p2l.info +0.0.0.0 loan.aol.msk.su +0.0.0.0 loan.maybachexelero.org +0.0.0.0 loestrin.1.p2l.info +0.0.0.0 lo.ljkeefeco.com +0.0.0.0 lol.to +0.0.0.0 lortab-cod.hut1.ru +0.0.0.0 lortab.hut1.ru +0.0.0.0 ma.5.p2l.info +0.0.0.0 mailforfreedom.com +0.0.0.0 make-money.shengen.ru +0.0.0.0 maps-antivert58.eksuziv.net +0.0.0.0 maps-spyware251-300.eksuziv.net +0.0.0.0 marketing.beesearch.info +0.0.0.0 mb.5.p2l.info +0.0.0.0 mba-online.petrovka.info +0.0.0.0 md.5.p2l.info +0.0.0.0 me.5.p2l.info +0.0.0.0 medical.carway.net +0.0.0.0 mens.1.p2l.info +0.0.0.0 meridia.1.p2l.info +0.0.0.0 meridia.3.p2l.info +0.0.0.0 meridia.4.p2l.info +0.0.0.0 meridiameridia.3xforum.ro +0.0.0.0 mesotherapy.jino-net.ru +0.0.0.0 mi.5.p2l.info +0.0.0.0 micardiss.ql.st +0.0.0.0 microsoft-sql-server.wp-club.net +0.0.0.0 mn.5.p2l.info +0.0.0.0 mo.5.p2l.info +0.0.0.0 moc.silk.com +0.0.0.0 mortgage-memphis.hotmail.ru +0.0.0.0 mortgage-rates.now-cash.com +0.0.0.0 mp.5.p2l.info +0.0.0.0 mrjeweller.us +0.0.0.0 ms.5.p2l.info +0.0.0.0 mt.5.p2l.info +0.0.0.0 multimedia-projector.katrina.ru +0.0.0.0 muscle-relaxers.1.p2l.info +0.0.0.0 music102.awardspace.com +0.0.0.0 mydaddy.b0x.com +0.0.0.0 myphentermine.polybuild.ru +0.0.0.0 nasacort.1.p2l.info +0.0.0.0 nasonex.1.p2l.info +0.0.0.0 nb.5.p2l.info +0.0.0.0 nc.5.p2l.info +0.0.0.0 nd.5.p2l.info +0.0.0.0 ne.5.p2l.info +0.0.0.0 nellyticket.beast-space.com +0.0.0.0 nelsongod.ca +0.0.0.0 nexium.1.p2l.info +0.0.0.0 nextel-ringtone.komi.su +0.0.0.0 nextel-ringtone.spb.su +0.0.0.0 nf.5.p2l.info +0.0.0.0 nh.5.p2l.info +0.0.0.0 nj.5.p2l.info +0.0.0.0 nm.5.p2l.info +0.0.0.0 nordette.1.p2l.info +0.0.0.0 nordette.3.p2l.info +0.0.0.0 nordette.4.p2l.info +0.0.0.0 norton-antivirus-trial.searchservice.info +0.0.0.0 notebook-memory.searchservice.info +0.0.0.0 ns.5.p2l.info +0.0.0.0 nv.5.p2l.info +0.0.0.0 ny.5.p2l.info +0.0.0.0 o8.aus.cc +0.0.0.0 ofni.al0ne.info +0.0.0.0 oh.5.p2l.info +0.0.0.0 ok.5.p2l.info +0.0.0.0 on.5.p2l.info +0.0.0.0 online-auto-insurance.petrovka.info +0.0.0.0 online-bingo.petrovka.info +0.0.0.0 online-broker.petrovka.info +0.0.0.0 online-cash.petrovka.info +0.0.0.0 online-casino.shengen.ru +0.0.0.0 online-casino.webpark.pl +0.0.0.0 online-cigarettes.hitslog.net +0.0.0.0 online-college.petrovka.info +0.0.0.0 online-degree.petrovka.info +0.0.0.0 online-florist.petrovka.info +0.0.0.0 online-forex.hut1.ru +0.0.0.0 online-forex-trading-systems.blogspot.com +0.0.0.0 online-gaming.petrovka.info +0.0.0.0 online-job.petrovka.info +0.0.0.0 online-loan.petrovka.info +0.0.0.0 online-mortgage.petrovka.info +0.0.0.0 online-personal.petrovka.info +0.0.0.0 online-personals.petrovka.info +0.0.0.0 online-pharmacy-online.blogspot.com +0.0.0.0 online-pharmacy.petrovka.info +0.0.0.0 online-phentermine.petrovka.info +0.0.0.0 online-poker-gambling.petrovka.info +0.0.0.0 online-poker-game.petrovka.info +0.0.0.0 online-poker.shengen.ru +0.0.0.0 online-prescription.petrovka.info +0.0.0.0 online-school.petrovka.info +0.0.0.0 online-schools.petrovka.info +0.0.0.0 online-single.petrovka.info +0.0.0.0 online-tarot-reading.beesearch.info +0.0.0.0 online-travel.petrovka.info +0.0.0.0 online-university.petrovka.info +0.0.0.0 online-viagra.petrovka.info +0.0.0.0 online-xanax.petrovka.info +0.0.0.0 onlypreteens.com +0.0.0.0 only-valium.go.to +0.0.0.0 only-valium.shengen.ru +0.0.0.0 or.5.p2l.info +0.0.0.0 oranla.info +0.0.0.0 orderadipex.findmenow.info +0.0.0.0 order-hydrocodone.polybuild.ru +0.0.0.0 order-phentermine.polybuild.ru +0.0.0.0 order-valium.polybuild.ru +0.0.0.0 ortho-tri-cyclen.1.p2l.info +0.0.0.0 pa.5.p2l.info +0.0.0.0 pacific-poker.e-online-poker-4u.net +0.0.0.0 pain-relief.1.p2l.info +0.0.0.0 paintball-gun.tripod.com +0.0.0.0 patio-furniture.dreamhoster.com +0.0.0.0 paxil.1.p2l.info +0.0.0.0 pay-day-loans.beesearch.info +0.0.0.0 payday-loans.now-cash.com +0.0.0.0 pctuzing.php5.cz +0.0.0.0 pd1.funnyhost.com +0.0.0.0 pe.5.p2l.info +0.0.0.0 peter-north-cum-shot.blogspot.com +0.0.0.0 pets.finaltips.com +0.0.0.0 pharmacy-canada.forsearch.net +0.0.0.0 pharmacy.hut1.ru +0.0.0.0 pharmacy-news.blogspot.com +0.0.0.0 pharmacy-online.petrovka.info +0.0.0.0 phendimetrazine.1.p2l.info +0.0.0.0 phentermine.1.p2l.info +0.0.0.0 phentermine.3.p2l.info +0.0.0.0 phentermine.4.p2l.info +0.0.0.0 phentermine.aussie7.com +0.0.0.0 phentermine-buy-online.hitslog.net +0.0.0.0 phentermine-buy.petrovka.info +0.0.0.0 phentermine-online.iscool.nl +0.0.0.0 phentermine-online.petrovka.info +0.0.0.0 phentermine.petrovka.info +0.0.0.0 phentermine.polybuild.ru +0.0.0.0 phentermine.shengen.ru +0.0.0.0 phentermine.t-amo.net +0.0.0.0 phentermine.webpark.pl +0.0.0.0 phone-calling-card.exnet.su +0.0.0.0 plavix.shengen.ru +0.0.0.0 play-poker-free.forsearch.net +0.0.0.0 poker-games.e-online-poker-4u.net +0.0.0.0 pop.egi.biz +0.0.0.0 pr.5.p2l.info +0.0.0.0 prescription-drugs.easy-find.net +0.0.0.0 prescription-drugs.shengen.ru +0.0.0.0 preteenland.com +0.0.0.0 preteensite.com +0.0.0.0 prevacid.1.p2l.info +0.0.0.0 prevent-asian-flu.com +0.0.0.0 prilosec.1.p2l.info +0.0.0.0 propecia.1.p2l.info +0.0.0.0 protonix.shengen.ru +0.0.0.0 psorias.atspace.com +0.0.0.0 purchase.hut1.ru +0.0.0.0 qc.5.p2l.info +0.0.0.0 qz.informs.com +0.0.0.0 refinance.shengen.ru +0.0.0.0 relenza.asian-flu-vaccine.com +0.0.0.0 renova.1.p2l.info +0.0.0.0 replacement-windows.gloses.net +0.0.0.0 re.rutan.org +0.0.0.0 resanium.com +0.0.0.0 retin-a.1.p2l.info +0.0.0.0 ri.5.p2l.info +0.0.0.0 rise-media.ru +0.0.0.0 root.dns.bz +0.0.0.0 roulette-online.petrovka.info +0.0.0.0 router.googlecom.biz +0.0.0.0 s32.bilsay.com +0.0.0.0 samsclub33.pochta.ru +0.0.0.0 sc10.net +0.0.0.0 sc.5.p2l.info +0.0.0.0 sd.5.p2l.info +0.0.0.0 search4you.50webs.com +0.0.0.0 search-phentermine.hpage.net +0.0.0.0 searchpill.boom.ru +0.0.0.0 seasonale.1.p2l.info +0.0.0.0 shop.kauffes.de +0.0.0.0 single-online.petrovka.info +0.0.0.0 sk.5.p2l.info +0.0.0.0 skelaxin.1.p2l.info +0.0.0.0 skelaxin.3.p2l.info +0.0.0.0 skelaxin.4.p2l.info +0.0.0.0 skin-care.1.p2l.info +0.0.0.0 skocz.pl +0.0.0.0 sleep-aids.1.p2l.info +0.0.0.0 sleeper-sofa.dreamhoster.com +0.0.0.0 slf5cyd.info +0.0.0.0 sobolev.net.ru +0.0.0.0 soma.1.p2l.info +0.0.0.0 soma.3xforum.ro +0.0.0.0 soma-store.visa-usa.ru +0.0.0.0 sonata.1.p2l.info +0.0.0.0 sport-betting-online.hitslog.net +0.0.0.0 spyware-removers.shengen.ru +0.0.0.0 spyware-scan.100gal.net +0.0.0.0 spyware.usafreespace.com +0.0.0.0 sq7.co.uk +0.0.0.0 sql-server-driver.beesearch.info +0.0.0.0 starlix.ql.st +0.0.0.0 stop-smoking.1.p2l.info +0.0.0.0 supplements.1.p2l.info +0.0.0.0 sx.nazari.org +0.0.0.0 sx.z0rz.com +0.0.0.0 ta.at.ic5mp.net +0.0.0.0 ta.at.user-mode-linux.net +0.0.0.0 tamiflu-in-canada.asian-flu-vaccine.com +0.0.0.0 tamiflu-no-prescription.asian-flu-vaccine.com +0.0.0.0 tamiflu-purchase.asian-flu-vaccine.com +0.0.0.0 tamiflu-without-prescription.asian-flu-vaccine.com +0.0.0.0 tenuate.1.p2l.info +0.0.0.0 texas-hold-em.e-online-poker-4u.net +0.0.0.0 texas-holdem.shengen.ru +0.0.0.0 ticket20.tripod.com +0.0.0.0 tizanidine.1.p2l.info +0.0.0.0 tn.5.p2l.info +0.0.0.0 topmeds10.com +0.0.0.0 top.pcanywhere.net +0.0.0.0 toyota.cyberealhosting.com +0.0.0.0 tramadol.1.p2l.info +0.0.0.0 tramadol2006.3xforum.ro +0.0.0.0 tramadol.3.p2l.info +0.0.0.0 tramadol.4.p2l.info +0.0.0.0 travel-insurance-quotes.beesearch.info +0.0.0.0 triphasil.1.p2l.info +0.0.0.0 triphasil.3.p2l.info +0.0.0.0 triphasil.4.p2l.info +0.0.0.0 tx.5.p2l.info +0.0.0.0 uf2aasn.111adfueo.us +0.0.0.0 ultracet.1.p2l.info +0.0.0.0 ultram.1.p2l.info +0.0.0.0 united-airline-fare.100pantyhose.com +0.0.0.0 university-online.petrovka.info +0.0.0.0 urlcut.net +0.0.0.0 urshort.net +0.0.0.0 us.kopuz.com +0.0.0.0 ut.5.p2l.info +0.0.0.0 utairway.com +0.0.0.0 va.5.p2l.info +0.0.0.0 vacation.toppick.info +0.0.0.0 valium.este.ru +0.0.0.0 valium.hut1.ru +0.0.0.0 valium.ourtablets.com +0.0.0.0 valium.polybuild.ru +0.0.0.0 valiumvalium.3xforum.ro +0.0.0.0 valtrex.1.p2l.info +0.0.0.0 valtrex.3.p2l.info +0.0.0.0 valtrex.4.p2l.info +0.0.0.0 valtrex.7h.com +0.0.0.0 vaniqa.1.p2l.info +0.0.0.0 vi.5.p2l.info +0.0.0.0 viagra.1.p2l.info +0.0.0.0 viagra.3.p2l.info +0.0.0.0 viagra.4.p2l.info +0.0.0.0 viagra-online.petrovka.info +0.0.0.0 viagra-pill.blogspot.com +0.0.0.0 viagra.polybuild.ru +0.0.0.0 viagra-soft-tabs.1.p2l.info +0.0.0.0 viagra-store.shengen.ru +0.0.0.0 viagraviagra.3xforum.ro +0.0.0.0 vicodin-online.petrovka.info +0.0.0.0 vicodin-store.shengen.ru +0.0.0.0 vicodin.t-amo.net +0.0.0.0 viewtools.com +0.0.0.0 vioxx.1.p2l.info +0.0.0.0 vitalitymax.1.p2l.info +0.0.0.0 vt.5.p2l.info +0.0.0.0 vxv.phre.net +0.0.0.0 w0.drag0n.org +0.0.0.0 wa.5.p2l.info +0.0.0.0 water-bed.8p.org.uk +0.0.0.0 web-hosting.hitslog.net +0.0.0.0 webhosting.hut1.ru +0.0.0.0 weborg.hut1.ru +0.0.0.0 weight-loss.1.p2l.info +0.0.0.0 weight-loss.3.p2l.info +0.0.0.0 weight-loss.4.p2l.info +0.0.0.0 weight-loss.hut1.ru +0.0.0.0 wellbutrin.1.p2l.info +0.0.0.0 wellbutrin.3.p2l.info +0.0.0.0 wellbutrin.4.p2l.info +0.0.0.0 wellnessmonitor.bravehost.com +0.0.0.0 wi.5.p2l.info +0.0.0.0 world-trade-center.hawaiicity.com +0.0.0.0 wp-club.net +0.0.0.0 ws01.do.nu +0.0.0.0 ws02.do.nu +0.0.0.0 ws03.do.nu +0.0.0.0 ws03.home.sapo.pt +0.0.0.0 ws04.do.nu +0.0.0.0 ws04.home.sapo.pt +0.0.0.0 ws05.home.sapo.pt +0.0.0.0 ws06.home.sapo.pt +0.0.0.0 wv.5.p2l.info +0.0.0.0 www.31d.net +0.0.0.0 www3.ddns.ms +0.0.0.0 www4.at.debianbase.de +0.0.0.0 www4.epac.to +0.0.0.0 www5.3-a.net +0.0.0.0 www69.bestdeals.at +0.0.0.0 www69.byinter.net +0.0.0.0 www69.dynu.com +0.0.0.0 www69.findhere.org +0.0.0.0 www69.fw.nu +0.0.0.0 www69.ugly.as +0.0.0.0 www6.ezua.com +0.0.0.0 www6.ns1.name +0.0.0.0 www7.ygto.com +0.0.0.0 www8.ns01.us +0.0.0.0 www99.bounceme.net +0.0.0.0 www99.fdns.net +0.0.0.0 www99.zapto.org +0.0.0.0 www9.compblue.com +0.0.0.0 www9.servequake.com +0.0.0.0 www9.trickip.org +0.0.0.0 www.adspoll.com +0.0.0.0 www.adult-top-list.com +0.0.0.0 www.aektschen.de +0.0.0.0 www.aeqs.com +0.0.0.0 www.alladultdirectories.com +0.0.0.0 www.alladultdirectory.net +0.0.0.0 www.arbeitssuche-web.de +0.0.0.0 www.bestrxpills.com +0.0.0.0 www.bigsister.cxa.de +0.0.0.0 www.bigsister-puff.cxa.de +0.0.0.0 www.bitlocker.net +0.0.0.0 www.cheap-laptops-notebook-computers.info +0.0.0.0 www.cheap-online-stamp.cast.cc +0.0.0.0 www.codez-knacken.de +0.0.0.0 www.computerxchange.com +0.0.0.0 www.credit-dreams.com +0.0.0.0 www.edle-stuecke.de +0.0.0.0 www.exe-file.de +0.0.0.0 www.exttrem.de +0.0.0.0 www.fetisch-pornos.cxa.de +0.0.0.0 www.ficken-ficken-ficken.cxa.de +0.0.0.0 www.ficken-xxx.cxa.de +0.0.0.0 www.financial-advice-books.com +0.0.0.0 www.finanzmarkt2004.de +0.0.0.0 www.furnitureulimited.com +0.0.0.0 www.gewinnspiele-slotmachine.de +0.0.0.0 www.hardware4freaks.de +0.0.0.0 www.healthyaltprods.com +0.0.0.0 www.heimlich-gefilmt.cxa.de +0.0.0.0 www.huberts-kochseite.de +0.0.0.0 www.huren-verzeichnis.is4all.de +0.0.0.0 www.kaaza-legal.de +0.0.0.0 www.kajahdfssa.net +0.0.0.0 www.keyofhealth.com +0.0.0.0 www.kitchentablegang.org +0.0.0.0 www.km69.de +0.0.0.0 www.koch-backrezepte.de +0.0.0.0 www.kvr-systems.de +0.0.0.0 www.lesben-pornos.cxa.de +0.0.0.0 www.links-private-krankenversicherung.de +0.0.0.0 www.littledevildoubt.com +0.0.0.0 www.mailforfreedom.com +0.0.0.0 www.masterspace.biz +0.0.0.0 www.medical-research-books.com +0.0.0.0 www.microsoft2010.com +0.0.0.0 www.nelsongod.ca +0.0.0.0 www.nextstudent.com +0.0.0.0 www.ntdesk.de +0.0.0.0 www.nutten-verzeichnis.cxa.de +0.0.0.0 www.obesitycheck.com +0.0.0.0 www.pawnauctions.net +0.0.0.0 www.pills-home.com +0.0.0.0 www.poker4spain.com +0.0.0.0 www.poker-new.com +0.0.0.0 www.poker-unique.com +0.0.0.0 www.porno-lesben.cxa.de +0.0.0.0 www.prevent-asian-flu.com +0.0.0.0 www.randppro-cuts.com +0.0.0.0 www.romanticmaui.net +0.0.0.0 www.salldo.de +0.0.0.0 www.samsclub33.pochta.ru +0.0.0.0 www.schwarz-weisses.de +0.0.0.0 www.schwule-boys-nackt.cxa.de +0.0.0.0 www.shopping-artikel.de +0.0.0.0 www.showcaserealestate.net +0.0.0.0 www.skattabrain.com +0.0.0.0 www.softcha.com +0.0.0.0 www.striemline.de +0.0.0.0 www.talentbroker.net +0.0.0.0 www.the-discount-store.com +0.0.0.0 www.topmeds10.com +0.0.0.0 www.uniqueinternettexasholdempoker.com +0.0.0.0 www.viagra-home.com +0.0.0.0 www.vthought.com +0.0.0.0 www.vtoyshop.com +0.0.0.0 www.vulcannonibird.de +0.0.0.0 www.webabrufe.de +0.0.0.0 www.wilddreams.info +0.0.0.0 www.willcommen.de +0.0.0.0 www.xcr-286.com +0.0.0.0 wy.5.p2l.info +0.0.0.0 x25.2mydns.com +0.0.0.0 x25.plorp.com +0.0.0.0 x4.lov3.net +0.0.0.0 x6x.a.la +0.0.0.0 x888x.myserver.org +0.0.0.0 x8x.dyndns.dk +0.0.0.0 x8x.trickip.net +0.0.0.0 xanax-online.dot.de +0.0.0.0 xanax-online.run.to +0.0.0.0 xanax-online.sms2.us +0.0.0.0 xanax.ourtablets.com +0.0.0.0 xanax-store.shengen.ru +0.0.0.0 xanax.t-amo.net +0.0.0.0 xanaxxanax.3xforum.ro +0.0.0.0 x-box.t35.com +0.0.0.0 xcr-286.com +0.0.0.0 xenical.1.p2l.info +0.0.0.0 xenical.3.p2l.info +0.0.0.0 xenical.4.p2l.info +0.0.0.0 x-hydrocodone.info +0.0.0.0 xoomer.alice.it +0.0.0.0 x-phentermine.info +0.0.0.0 xr.h4ck.la +0.0.0.0 yasmin.1.p2l.info +0.0.0.0 yasmin.3.p2l.info +0.0.0.0 yasmin.4.p2l.info +0.0.0.0 yt.5.p2l.info +0.0.0.0 zanaflex.1.p2l.info +0.0.0.0 zebutal.1.p2l.info +0.0.0.0 zocor.about-tabs.com +0.0.0.0 zoloft.1.p2l.info +0.0.0.0 zoloft.3.p2l.info +0.0.0.0 zoloft.4.p2l.info +0.0.0.0 zoloft.about-tabs.com +0.0.0.0 zyban.1.p2l.info +0.0.0.0 zyban.about-tabs.com +0.0.0.0 zyban-store.shengen.ru +0.0.0.0 zyprexa.about-tabs.com +0.0.0.0 zyrtec.1.p2l.info +0.0.0.0 zyrtec.3.p2l.info +0.0.0.0 zyrtec.4.p2l.info +# + +# + +# Phorm contextual advertising sites +0.0.0.0 a.oix.com +0.0.0.0 a.oix.net +0.0.0.0 a.openinternetexchange.com +0.0.0.0 a.phormlabs.com +0.0.0.0 a.webwise.com +0.0.0.0 a.webwise.net +0.0.0.0 b.oix.net +0.0.0.0 br.phorm.com +0.0.0.0 bt.phorm.com +0.0.0.0 bt.webwise.com +0.0.0.0 b.webwise.net +0.0.0.0 c.webwise.com +0.0.0.0 c.webwise.net +0.0.0.0 d.oix.com +0.0.0.0 d.phormlabs.com +0.0.0.0 ig.fp.oix.net +0.0.0.0 invite.gezinti.com +0.0.0.0 kentsucks.youcanoptout.com +0.0.0.0 kr.phorm.com +0.0.0.0 mail.youcanoptout.com +0.0.0.0 mail.youcanoptout.net +0.0.0.0 mail.youcanoptout.org +0.0.0.0 monitor.phorm.com +0.0.0.0 mx01.openinternetexchange.com +0.0.0.0 mx01.openinternetexchange.net +0.0.0.0 mx01.webwise.com +0.0.0.0 mx03.phorm.com +0.0.0.0 navegador.oi.com.br +0.0.0.0 navegador.telefonica.com.br +0.0.0.0 ns1.oix.com +0.0.0.0 ns1.openinternetexchange.com +0.0.0.0 ns1.phorm.com +0.0.0.0 ns2.oix.com +0.0.0.0 ns2.openinternetexchange.com +0.0.0.0 ns2.phorm.com +0.0.0.0 ns2.youcanoptout.com +0.0.0.0 ns3.openinternetexchange.com +0.0.0.0 oi.webnavegador.com.br +0.0.0.0 oixcrv-lab.net +0.0.0.0 oixcrv.net +0.0.0.0 oixcrv-stage.net +0.0.0.0 oix.phorm.com +0.0.0.0 oixpre.net +0.0.0.0 oixpre-stage.net +0.0.0.0 oixssp-lab.net +0.0.0.0 oixssp.net +0.0.0.0 oix-stage.net +0.0.0.0 openinternetexchange.com +0.0.0.0 openinternetexchange.net +0.0.0.0 phorm.kr +0.0.0.0 phormlabs.com +0.0.0.0 prm-ext.phorm.com +0.0.0.0 romdiscover.com +0.0.0.0 rtc.romdiscover.com +0.0.0.0 stats.oix.com +0.0.0.0 stopphoulplay.com +0.0.0.0 stopphoulplay.net +0.0.0.0 telefonica.webnavegador.com.br +0.0.0.0 webnavegador.com.br +0.0.0.0 webwise.com +0.0.0.0 webwise.net +0.0.0.0 w.oix.net +0.0.0.0 www.gezinti.com +0.0.0.0 www.gozatar.com +0.0.0.0 www.oix.com +0.0.0.0 www.openinternetexchange.com +0.0.0.0 www.phormlabs.com +0.0.0.0 www.stopphoulplay.com +0.0.0.0 www.youcanoptout.com +0.0.0.0 www.youcanoptout.net +0.0.0.0 www.youcanoptout.org +0.0.0.0 xxyyzz.youcanoptout.com +0.0.0.0 youcanoptout.com +0.0.0.0 youcanoptout.net +0.0.0.0 youcanoptout.org +# diff --git a/src/gevent/tests/https_svn_python_org_root.pem b/src/gevent/tests/https_svn_python_org_root.pem new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/gevent/tests/https_svn_python_org_root.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/gevent/tests/keycert.pem b/src/gevent/tests/keycert.pem new file mode 100644 index 0000000..2f46fcf --- /dev/null +++ b/src/gevent/tests/keycert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/gevent/tests/known_failures.py b/src/gevent/tests/known_failures.py new file mode 100644 index 0000000..6396c0e --- /dev/null +++ b/src/gevent/tests/known_failures.py @@ -0,0 +1,398 @@ +# This is a list of known failures (=bugs). +# The tests listed there must fail (or testrunner.py will report error) unless they are prefixed with FLAKY +# in which cases the result of them is simply ignored +from __future__ import print_function +import os +import sys +import struct + +from gevent.testing.sysinfo import RUNNING_ON_APPVEYOR as APPVEYOR +from gevent.testing.sysinfo import RUNNING_ON_TRAVIS as TRAVIS +from gevent.testing.sysinfo import RUN_LEAKCHECKS as LEAKTEST +from gevent.testing.sysinfo import RUN_COVERAGE as COVERAGE +from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM + +from gevent.testing.sysinfo import PYPY +from gevent.testing.sysinfo import PY3 +from gevent.testing.sysinfo import PY35 + +from gevent.testing.sysinfo import LIBUV + +IGNORED_TESTS = [] + +FAILING_TESTS = [ + + # Sometimes fails with AssertionError: ...\nIOError: close() called during concurrent operation on the same file object.\n' + # Sometimes it contains "\nUnhandled exception in thread started by \nsys.excepthook is missing\nlost sys.stderr\n" + "FLAKY test__subprocess_interrupted.py", + # test__issue6 (see comments in test file) is really flaky on both Travis and Appveyor; + # on Travis we could just run the test again (but that gets old fast), but on appveyor + # we don't have that option without a new commit---and sometimes we really need a build + # to succeed in order to get a release wheel + 'FLAKY test__issue6.py', +] + + +if sys.platform == 'win32': + IGNORED_TESTS = [ + # fork watchers don't get called on windows + # because fork is not a concept windows has. + # See this file for a detailed explanation. + 'test__core_fork.py', + ] + # other Windows-related issues (need investigating) + FAILING_TESTS += [ + 'FLAKY test__greenletset.py', + # This has been seen to fail on Py3 and Py2 due to socket reuse + # errors, probably timing related again. + 'FLAKY test___example_servers.py', + ] + + if APPVEYOR: + FAILING_TESTS += [ + # These both run on port 9000 and can step on each other...seems like the + # appveyor containers aren't fully port safe? Or it takes longer + # for the processes to shut down? Or we run them in a different order + # in the process pool than we do other places? + 'FLAKY test__example_udp_client.py', + 'FLAKY test__example_udp_server.py', + # This one sometimes times out, often after output "The process with PID XXX could not be + # terminated. Reason: There is no running instance of the task." + 'FLAKY test__example_portforwarder.py', + # This one sometimes randomly closes connections, but no indication + # of a server crash, only a client side close. + 'FLAKY test__server_pywsgi.py', + ] + + if PYPY and LIBUV: + IGNORED_TESTS += [ + # This one seems to just stop right after + # patching is done. It passes on a local win 10 vm, and the main + # test_threading_2.py does as well. + # Based on the printouts we added, it appears to not even + # finish importing: + # https://ci.appveyor.com/project/denik/gevent/build/1.0.1277/job/tpvhesij5gldjxqw#L1190 + # Ignored because it takes two minutes to time out. + 'test_threading.py', + ] + + if PY3: + FAILING_TESTS += [ + # test_set_and_clear in Py3 relies on 5 threads all starting and + # coming to an Event wait point while a sixth thread sleeps for a half + # second. The sixth thread then does something and checks that + # the 5 threads were all at the wait point. But the timing is sometimes + # too tight for appveyor. This happens even if Event isn't + # monkey-patched + 'FLAKY test_threading.py', + + # Starting in November 2018, on Python 3.7.0, we observe this test crashing. + # I can't reproduce locally. + # | C:\Python37-x64\python.exe -u -mgevent.tests.test__greenness + # 127.0.0.1 - - [09/Nov/2018 16:34:12] code 501, message Unsupported method ('GET') + # 127.0.0.1 - - [09/Nov/2018 16:34:12] "GET / HTTP/1.1" 501 - + # . + # ---------------------------------------------------------------------- + # Ran 1 test in 0.031s + + # OK + # Windows fatal exception: access violation + + # Current thread 0x000003c8 (most recent call first): + # File "c:\projects\gevent\src\gevent\threadpool.py", line 261 in _worker + + # Thread 0x00000600 (most recent call first): + # File "c:\projects\gevent\src\gevent\libuv\watcher.py", line 577 in send + # File "c:\projects\gevent\src\gevent\threadpool.py", line 408 in set + # File "c:\projects\gevent\src\gevent\threadpool.py", line 290 in _worker + + # Thread 0x000007d4 (most recent call first): + # File "C:\Python37-x64\lib\weakref.py", line 356 in remove + + # ! C:\Python37-x64\python.exe -u -mgevent.tests.test__greenness [code 3221225477] [took 1.3s] + # We have also seen this for Python 3.6.6 Nov 13 2018: + # | C:\Python36-x64\python.exe -u -mgevent.tests.test__backdoor + # ss.s.s + # ---------------------------------------------------------------------- + # Ran 6 tests in 0.953s + + # OK (skipped=4) + # Windows fatal exception: access violation + + # Thread 0x00000aec (most recent call first): + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get + # File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker + + # Thread 0x00000548 (most recent call first): + + # Thread 0x000003d0 (most recent call first): + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get + # File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker + + # Thread 0x00000ad0 (most recent call first): + + # Thread 0x00000588 (most recent call first): + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get + # File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker + + # Thread 0x00000a54 (most recent call first): + + # Thread 0x00000768 (most recent call first): + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get + # File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker + + # Current thread 0x00000894 (most recent call first): + # File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 261 in _worker + + # Thread 0x00000634 (most recent call first): + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 84 in wait + # File "C:\Python36-x64\lib\site-packages\gevent\_threading.py", line 166 in get + # File "C:\Python36-x64\lib\site-packages\gevent\threadpool.py", line 270 in _worker + + # Thread 0x00000538 (most recent call first): + + # Thread 0x0000049c (most recent call first): + # File "C:\Python36-x64\lib\weakref.py", line 356 in remove + + # ! C:\Python36-x64\python.exe -u -mgevent.tests.test__backdoor [code 3221225477] [Ran 6 tests in 2.1s] + + # Note the common factors: + # - The test is finished (successfully) and we're apparently exiting the VM, + # doing GC + # - A weakref is being cleaned up + + # weakref.py line 356 remove() is in WeakKeyDictionary. We only use WeakKeyDictionary + # in gevent._ident.IdentRegistry, which is only used in two places: + # gevent.hub.hub_ident_registry, which has weak references to Hub objects, + # and gevent.greenlet.Greenlet.minimal_ident, which uses its parent Hub's + # IdentRegistry to get its own identifier. So basically they have weak references + # to Hub and arbitrary Greenlets. + + # Our attempted solution: stop using a module-level IdentRegistry to get + # Hub idents, and reduce how often we auto-generate one for greenlets. + # Commenting out the tests, lets see if it works. + #'FLAKY test__greenness.py', + #'FLAKY test__backdoor.py', + ] + + if not PY35: + # Py35 added socket.socketpair, all other releases + # are missing it. No reason to even test it. + IGNORED_TESTS += [ + 'test__socketpair.py', + ] + + if struct.calcsize('P') * 8 == 64: + # could be a problem of appveyor - not sure + # ====================================================================== + # ERROR: test_af (__main__.TestIPv6Environment) + # ---------------------------------------------------------------------- + # File "C:\Python27-x64\lib\ftplib.py", line 135, in connect + # self.sock = socket.create_connection((self.host, self.port), self.timeout) + # File "c:\projects\gevent\gevent\socket.py", line 73, in create_connection + # raise err + # error: [Errno 10049] [Error 10049] The requested address is not valid in its context. + # XXX: On Jan 3 2016 this suddenly started passing on Py27/64; no idea why, the python version + # was 2.7.11 before and after. + FAILING_TESTS.append('FLAKY test_ftplib.py') + + if PY3: + pass + + +if LEAKTEST: + FAILING_TESTS += [ + 'FLAKY test__backdoor.py', + 'FLAKY test__socket_errors.py', + ] + + if os.environ.get("TRAVIS") == "true": + FAILING_TESTS += [ + # On Travis, this very frequently fails due to timing + 'FLAKY test_signal.py', + ] + + +if PYPY: + FAILING_TESTS += [ + ## Different in PyPy: + + ## Not implemented: + + ## --- + + ## BUGS: + + ## UNKNOWN: + # AssertionError: '>>> ' != '' + # test__backdoor.py:52 + 'FLAKY test__backdoor.py', + ] + + if RESOLVER_NOT_SYSTEM: + + FAILING_TESTS += [ + # A few errors and differences: + # AssertionError: ('255.255.255.255', 'http') != gaierror(4, 'ARES_ENOTFOUND: Domain name not found') + # AssertionError: OverflowError('port must be 0-65535.',) != ('readthedocs.org', '65535') + # AssertionError: Lists differ: + # (10, 1, 6, '', ('2607:f8b0:4004:810::200e', 80, 0L, 0L)) + # (10, 1, 6, '', ('2607:f8b0:4004:805::200e', 80, 0, 0)) + 'test__socket_dns.py', + ] + + if LIBUV: + IGNORED_TESTS += [ + # This hangs for no apparent reason when run by the testrunner, + # even wher maked standalone + # when run standalone from the command line, it's fine. + # Issue in pypy2 6.0? + 'test__monkey_sigchld_2.py', + ] + + if TRAVIS: + FAILING_TESTS += [ + # This fails to get the correct results, sometimes. I can't reproduce locally + 'FLAKY test__example_udp_server.py', + 'FLAKY test__example_udp_client.py', + ] + + if LIBUV: + IGNORED_TESTS += [ + # XXX: Re-enable this when we can investigate more. + # This has started crashing with a SystemError. + # I cannot reproduce with the same version on macOS + # and I cannot reproduce with the same version in a Linux vm. + # Commenting out individual tests just moves the crash around. + # https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception + 'test__pywsgi.py', + ] + + IGNORED_TESTS += [ + # XXX Re-enable these when we have more time to investigate. + # This test, which normally takes ~60s, sometimes + # hangs forever after running several tests. I cannot reproduce, + # it seems highly load dependent. Observed with both libev and libuv. + 'test__threadpool.py', + # This test, which normally takes 4-5s, sometimes + # hangs forever after running two tests. I cannot reproduce, + # it seems highly load dependent. Observed with both libev and libuv. + 'test__threading_2.py', + ] + + if PY3 and TRAVIS: + FAILING_TESTS += [ + ## --- + + ## Unknown; can't reproduce locally on OS X + 'FLAKY test_subprocess.py', # timeouts on one test. + + 'FLAKY test_ssl.py', + ] + + +if LIBUV: + if sys.platform.startswith("darwin"): + FAILING_TESTS += [ + ] + +if PY3: + # No idea / TODO + FAILING_TESTS += [ + 'FLAKY test__socket_dns.py', + ] + + + +if sys.version_info[:2] >= (3, 4) and APPVEYOR: + FAILING_TESTS += [ + # Timing issues on appveyor + 'FLAKY test_selectors.py' + ] + + +if COVERAGE: + # The gevent concurrency plugin tends to slow things + # down and get us past our default timeout value. These + # tests in particular are sensitive to it + FAILING_TESTS += [ + 'FLAKY test__issue302monkey.py', + 'FLAKY test__example_portforwarder.py', + 'FLAKY test__threading_vs_settrace.py', + ] + +FAILING_TESTS = [x.strip() for x in set(FAILING_TESTS) if x.strip()] + + +# A mapping from test file basename to a dictionary of +# options that will be applied on top of the DEFAULT_RUN_OPTIONS. +TEST_FILE_OPTIONS = { + +} + + +# tests that don't do well when run on busy box +RUN_ALONE = [ + 'test__threadpool.py', + 'test__examples.py', +] + + + +if APPVEYOR or TRAVIS: + RUN_ALONE += [ + # Partial workaround for the _testcapi issue on PyPy, + # but also because signal delivery can sometimes be slow, and this + # spawn processes of its own + 'test_signal.py', + ] + + if LEAKTEST and PY3: + # On a heavily loaded box, these can all take upwards of 200s + RUN_ALONE += [ + 'test__pool.py', + 'test__pywsgi.py', + 'test__queue.py', + ] + + if PYPY: + # This often takes much longer on PyPy on CI. + TEST_FILE_OPTIONS['test__threadpool.py'] = {'timeout': 180} + TEST_FILE_OPTIONS['test__threading_2.py'] = {'timeout': 180} + if PY3: + RUN_ALONE += [ + # Sometimes shows unexpected timeouts + 'test_socket.py', + ] + if LIBUV: + RUN_ALONE += [ + # https://bitbucket.org/pypy/pypy/issues/2769/systemerror-unexpected-internal-exception + 'test__pywsgi.py', + ] + +# tests that can't be run when coverage is enabled +IGNORE_COVERAGE = [ + # Hangs forever + 'test__threading_vs_settrace.py', + # times out + 'test_socket.py', + # Doesn't get the exceptions it expects + 'test_selectors.py', + # XXX ? + 'test__issue302monkey.py', + "test_subprocess.py", +] + +if PYPY: + IGNORE_COVERAGE += [ + # Tends to timeout + 'test__refcount.py', + 'test__greenletset.py' + ] + +if __name__ == '__main__': + print('known_failures:\n', FAILING_TESTS) diff --git a/src/gevent/tests/lock_tests.py b/src/gevent/tests/lock_tests.py new file mode 100644 index 0000000..2c5a91b --- /dev/null +++ b/src/gevent/tests/lock_tests.py @@ -0,0 +1,767 @@ +""" +Various tests for synchronization primitives. +""" +# pylint:disable=no-member,abstract-method +import sys +import time +try: + from thread import start_new_thread, get_ident +except ImportError: + from _thread import start_new_thread, get_ident +import threading +import unittest + +try: + from test import support +except ImportError: + from test import test_support as support + +from gevent.testing.testcase import TimeAssertMixin + +def _wait(): + # A crude wait/yield function not relying on synchronization primitives. + time.sleep(0.01) + +class Bunch(object): + """ + A bunch of threads. + """ + def __init__(self, f, n, wait_before_exit=False): + """ + Construct a bunch of `n` threads running the same function `f`. + If `wait_before_exit` is True, the threads won't terminate until + do_finish() is called. + """ + self.f = f + self.n = n + self.started = [] + self.finished = [] + self._can_exit = not wait_before_exit + def task(): + tid = get_ident() + self.started.append(tid) + try: + f() + finally: + self.finished.append(tid) + while not self._can_exit: + _wait() + for _ in range(n): + start_new_thread(task, ()) + + def wait_for_started(self): + while len(self.started) < self.n: + _wait() + + def wait_for_finished(self): + while len(self.finished) < self.n: + _wait() + + def do_finish(self): + self._can_exit = True + + +class BaseTestCase(TimeAssertMixin, unittest.TestCase): + def setUp(self): + self._threads = support.threading_setup() + + def tearDown(self): + support.threading_cleanup(*self._threads) + support.reap_children() + + +class BaseLockTests(BaseTestCase): + """ + Tests for both recursive and non-recursive locks. + """ + + def locktype(self): + raise NotImplementedError() + + def test_constructor(self): + lock = self.locktype() + del lock + + def test_acquire_destroy(self): + lock = self.locktype() + lock.acquire() + del lock + + def test_acquire_release(self): + lock = self.locktype() + lock.acquire() + lock.release() + del lock + + def test_try_acquire(self): + lock = self.locktype() + self.assertTrue(lock.acquire(False)) + lock.release() + + def test_try_acquire_contended(self): + lock = self.locktype() + lock.acquire() + result = [] + def f(): + result.append(lock.acquire(False)) + Bunch(f, 1).wait_for_finished() + self.assertFalse(result[0]) + lock.release() + + def test_acquire_contended(self): + lock = self.locktype() + lock.acquire() + N = 5 + def f(): + lock.acquire() + lock.release() + + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(len(b.finished), 0) + lock.release() + b.wait_for_finished() + self.assertEqual(len(b.finished), N) + + def test_with(self): + lock = self.locktype() + def f(): + lock.acquire() + lock.release() + def _with(err=None): + with lock: + if err is not None: + raise err # pylint:disable=raising-bad-type + _with() + # Check the lock is unacquired + Bunch(f, 1).wait_for_finished() + self.assertRaises(TypeError, _with, TypeError) + # Check the lock is unacquired + Bunch(f, 1).wait_for_finished() + + def test_thread_leak(self): + # The lock shouldn't leak a Thread instance when used from a foreign + # (non-threading) thread. + lock = self.locktype() + def f(): + lock.acquire() + lock.release() + n = len(threading.enumerate()) + # We run many threads in the hope that existing threads ids won't + # be recycled. + Bunch(f, 15).wait_for_finished() + self.assertEqual(n, len(threading.enumerate())) + + +class LockTests(BaseLockTests): # pylint:disable=abstract-method + """ + Tests for non-recursive, weak locks + (which can be acquired and released from different threads). + """ + def test_reacquire(self): + # Lock needs to be released before re-acquiring. + lock = self.locktype() + phase = [] + def f(): + lock.acquire() + phase.append(None) + lock.acquire() + phase.append(None) + start_new_thread(f, ()) + while not phase: + _wait() + _wait() + self.assertEqual(len(phase), 1) + lock.release() + while len(phase) == 1: + _wait() + self.assertEqual(len(phase), 2) + + def test_different_thread(self): + # Lock can be released from a different thread. + lock = self.locktype() + lock.acquire() + def f(): + lock.release() + b = Bunch(f, 1) + b.wait_for_finished() + lock.acquire() + lock.release() + + +class RLockTests(BaseLockTests): + """ + Tests for recursive locks. + """ + def test_reacquire(self): + lock = self.locktype() + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + + def test_release_unacquired(self): + # Cannot release an unacquired lock + lock = self.locktype() + self.assertRaises(RuntimeError, lock.release) + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + self.assertRaises(RuntimeError, lock.release) + + def test_different_thread(self): + # Cannot release from a different thread + lock = self.locktype() + def f(): + lock.acquire() + b = Bunch(f, 1, True) + try: + self.assertRaises(RuntimeError, lock.release) + finally: + b.do_finish() + + def test__is_owned(self): + lock = self.locktype() + self.assertFalse(lock._is_owned()) + lock.acquire() + self.assertTrue(lock._is_owned()) + lock.acquire() + self.assertTrue(lock._is_owned()) + result = [] + def f(): + result.append(lock._is_owned()) + Bunch(f, 1).wait_for_finished() + self.assertFalse(result[0]) + lock.release() + self.assertTrue(lock._is_owned()) + lock.release() + self.assertFalse(lock._is_owned()) + + +class EventTests(BaseTestCase): + """ + Tests for Event objects. + """ + + def eventtype(self): + raise NotImplementedError() + + def test_is_set(self): + evt = self.eventtype() + self.assertFalse(evt.is_set()) + evt.set() + self.assertTrue(evt.is_set()) + evt.set() + self.assertTrue(evt.is_set()) + evt.clear() + self.assertFalse(evt.is_set()) + evt.clear() + self.assertFalse(evt.is_set()) + + def _check_notify(self, evt): + # All threads get notified + N = 5 + results1 = [] + results2 = [] + def f(): + evt.wait() + results1.append(evt.is_set()) + evt.wait() + results2.append(evt.is_set()) + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(len(results1), 0) + evt.set() + b.wait_for_finished() + self.assertEqual(results1, [True] * N) + self.assertEqual(results2, [True] * N) + + def test_notify(self): + evt = self.eventtype() + self._check_notify(evt) + # Another time, after an explicit clear() + evt.set() + evt.clear() + self._check_notify(evt) + + def test_timeout(self): + evt = self.eventtype() + results1 = [] + results2 = [] + N = 5 + def f(): + evt.wait(0.0) + results1.append(evt.is_set()) + t1 = time.time() + evt.wait(0.2) + r = evt.is_set() + t2 = time.time() + results2.append((r, t2 - t1)) + Bunch(f, N).wait_for_finished() + self.assertEqual(results1, [False] * N) + for r, dt in results2: + self.assertFalse(r) + self.assertTimeWithinRange(dt, 0.18, 10) + # The event is set + results1 = [] + results2 = [] + evt.set() + Bunch(f, N).wait_for_finished() + self.assertEqual(results1, [True] * N) + for r, dt in results2: + self.assertTrue(r) + + +class ConditionTests(BaseTestCase): + """ + Tests for condition variables. + """ + + def condtype(self, *args): + raise NotImplementedError() + + def test_acquire(self): + cond = self.condtype() + # Be default we have an RLock: the condition can be acquired multiple + # times. + cond.acquire() + cond.acquire() + cond.release() + cond.release() + lock = threading.Lock() + cond = self.condtype(lock) + cond.acquire() + self.assertFalse(lock.acquire(False)) + cond.release() + self.assertTrue(lock.acquire(False)) + self.assertFalse(cond.acquire(False)) + lock.release() + with cond: + self.assertFalse(lock.acquire(False)) + + def test_unacquired_wait(self): + cond = self.condtype() + self.assertRaises(RuntimeError, cond.wait) + + def test_unacquired_notify(self): + cond = self.condtype() + self.assertRaises(RuntimeError, cond.notify) + + def _check_notify(self, cond): + N = 5 + results1 = [] + results2 = [] + phase_num = 0 + def f(): + cond.acquire() + cond.wait() + cond.release() + results1.append(phase_num) + cond.acquire() + cond.wait() + cond.release() + results2.append(phase_num) + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(results1, []) + # Notify 3 threads at first + cond.acquire() + cond.notify(3) + _wait() + phase_num = 1 + cond.release() + while len(results1) < 3: + _wait() + self.assertEqual(results1, [1] * 3) + self.assertEqual(results2, []) + # Notify 5 threads: they might be in their first or second wait + cond.acquire() + cond.notify(5) + _wait() + phase_num = 2 + cond.release() + while len(results1) + len(results2) < 8: + _wait() + self.assertEqual(results1, [1] * 3 + [2] * 2) + self.assertEqual(results2, [2] * 3) + # Notify all threads: they are all in their second wait + cond.acquire() + cond.notify_all() + _wait() + phase_num = 3 + cond.release() + while len(results2) < 5: + _wait() + self.assertEqual(results1, [1] * 3 + [2] * 2) + self.assertEqual(results2, [2] * 3 + [3] * 2) + b.wait_for_finished() + + def test_notify(self): + cond = self.condtype() + self._check_notify(cond) + # A second time, to check internal state is still ok. + self._check_notify(cond) + + def test_timeout(self): + cond = self.condtype() + results = [] + N = 5 + def f(): + cond.acquire() + t1 = time.time() + cond.wait(0.2) + t2 = time.time() + cond.release() + results.append(t2 - t1) + Bunch(f, N).wait_for_finished() + self.assertEqual(len(results), 5) + for dt in results: + # XXX: libuv sometimes produces 0.19958 + self.assertTimeWithinRange(dt, 0.19, 2.0) + + +class BaseSemaphoreTests(BaseTestCase): + """ + Common tests for {bounded, unbounded} semaphore objects. + """ + + def semtype(self, *args): + raise NotImplementedError() + + def test_constructor(self): + self.assertRaises(ValueError, self.semtype, value=-1) + # Py3 doesn't have sys.maxint + self.assertRaises(ValueError, self.semtype, + value=-getattr(sys, 'maxint', getattr(sys, 'maxsize', None))) + + def test_acquire(self): + sem = self.semtype(1) + sem.acquire() + sem.release() + sem = self.semtype(2) + sem.acquire() + sem.acquire() + sem.release() + sem.release() + + def test_acquire_destroy(self): + sem = self.semtype() + sem.acquire() + del sem + + def test_acquire_contended(self): + sem = self.semtype(7) + sem.acquire() + #N = 10 + results1 = [] + results2 = [] + phase_num = 0 + def f(): + sem.acquire() + results1.append(phase_num) + sem.acquire() + results2.append(phase_num) + b = Bunch(f, 10) + b.wait_for_started() + while len(results1) + len(results2) < 6: + _wait() + self.assertEqual(results1 + results2, [0] * 6) + phase_num = 1 + for _ in range(7): + sem.release() + while len(results1) + len(results2) < 13: + _wait() + self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) + phase_num = 2 + for _ in range(6): + sem.release() + while len(results1) + len(results2) < 19: + _wait() + self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + # Final release, to let the last thread finish + sem.release() + b.wait_for_finished() + + def test_try_acquire(self): + sem = self.semtype(2) + self.assertTrue(sem.acquire(False)) + self.assertTrue(sem.acquire(False)) + self.assertFalse(sem.acquire(False)) + sem.release() + self.assertTrue(sem.acquire(False)) + + def test_try_acquire_contended(self): + sem = self.semtype(4) + sem.acquire() + results = [] + def f(): + results.append(sem.acquire(False)) + results.append(sem.acquire(False)) + Bunch(f, 5).wait_for_finished() + # There can be a thread switch between acquiring the semaphore and + # appending the result, therefore results will not necessarily be + # ordered. + self.assertEqual(sorted(results), [False] * 7 + [True] * 3) + + def test_default_value(self): + # The default initial value is 1. + sem = self.semtype() + sem.acquire() + def f(): + sem.acquire() + sem.release() + b = Bunch(f, 1) + b.wait_for_started() + _wait() + self.assertFalse(b.finished) + sem.release() + b.wait_for_finished() + + def test_with(self): + sem = self.semtype(2) + def _with(err=None): + with sem: + self.assertTrue(sem.acquire(False)) + sem.release() + with sem: + self.assertFalse(sem.acquire(False)) + if err: + raise err # pylint:disable=raising-bad-type + _with() + self.assertTrue(sem.acquire(False)) + sem.release() + self.assertRaises(TypeError, _with, TypeError) + self.assertTrue(sem.acquire(False)) + sem.release() + +class SemaphoreTests(BaseSemaphoreTests): + """ + Tests for unbounded semaphores. + """ + + def test_release_unacquired(self): + # Unbounded releases are allowed and increment the semaphore's value + sem = self.semtype(1) + sem.release() + sem.acquire() + sem.acquire() + sem.release() + + +class BoundedSemaphoreTests(BaseSemaphoreTests): + """ + Tests for bounded semaphores. + """ + + def test_release_unacquired(self): + # Cannot go past the initial value + sem = self.semtype() + self.assertRaises(ValueError, sem.release) + sem.acquire() + sem.release() + self.assertRaises(ValueError, sem.release) + +class BarrierTests(BaseTestCase): + """ + Tests for Barrier objects. + """ + N = 5 + defaultTimeout = 2.0 + + def setUp(self): + self.barrier = self.barriertype(self.N, timeout=self.defaultTimeout) + def tearDown(self): + self.barrier.abort() + + def run_threads(self, f): + b = Bunch(f, self.N-1) + f() + b.wait_for_finished() + + def multipass(self, results, n): + m = self.barrier.parties + self.assertEqual(m, self.N) + for i in range(n): + results[0].append(True) + self.assertEqual(len(results[1]), i * m) + self.barrier.wait() + results[1].append(True) + self.assertEqual(len(results[0]), (i + 1) * m) + self.barrier.wait() + self.assertEqual(self.barrier.n_waiting, 0) + self.assertFalse(self.barrier.broken) + + def test_barrier(self, passes=1): + """ + Test that a barrier is passed in lockstep + """ + results = [[], []] + def f(): + self.multipass(results, passes) + self.run_threads(f) + + def test_barrier_10(self): + """ + Test that a barrier works for 10 consecutive runs + """ + return self.test_barrier(10) + + def test_wait_return(self): + """ + test the return value from barrier.wait + """ + results = [] + def f(): + r = self.barrier.wait() + results.append(r) + + self.run_threads(f) + self.assertEqual(sum(results), sum(range(self.N))) + + def test_action(self): + """ + Test the 'action' callback + """ + results = [] + def action(): + results.append(True) + barrier = self.barriertype(self.N, action) + def f(): + barrier.wait() + self.assertEqual(len(results), 1) + + self.run_threads(f) + + def test_abort(self): + """ + Test that an abort will put the barrier in a broken state + """ + results1 = [] + results2 = [] + def f(): + try: + i = self.barrier.wait() + if i == self.N//2: + raise RuntimeError + self.barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + except RuntimeError: + self.barrier.abort() + + self.run_threads(f) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(self.barrier.broken) + + def test_reset(self): + """ + Test that a 'reset' on a barrier frees the waiting threads + """ + results1 = [] + results2 = [] + results3 = [] + def f(): + i = self.barrier.wait() + if i == self.N//2: + # Wait until the other threads are all in the barrier. + while self.barrier.n_waiting < self.N-1: + time.sleep(0.001) + self.barrier.reset() + else: + try: + self.barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + # Now, pass the barrier again + self.barrier.wait() + results3.append(True) + + self.run_threads(f) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + + def test_abort_and_reset(self): + """ + Test that a barrier can be reset after being broken. + """ + results1 = [] + results2 = [] + results3 = [] + barrier2 = self.barriertype(self.N) + def f(): + try: + i = self.barrier.wait() + if i == self.N//2: + raise RuntimeError + self.barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + except RuntimeError: + self.barrier.abort() + + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + if barrier2.wait() == self.N//2: + self.barrier.reset() + barrier2.wait() + self.barrier.wait() + results3.append(True) + + self.run_threads(f) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + def test_timeout(self): + """ + Test wait(timeout) + """ + def f(): + i = self.barrier.wait() + if i == self.N // 2: + # One thread is late! + time.sleep(1.0) + # Default timeout is 2.0, so this is shorter. + self.assertRaises(threading.BrokenBarrierError, + self.barrier.wait, 0.5) + self.run_threads(f) + + def test_default_timeout(self): + """ + Test the barrier's default timeout + """ + # create a barrier with a low default timeout + barrier = self.barriertype(self.N, timeout=0.3) + def f(): + i = barrier.wait() + if i == self.N // 2: + # One thread is later than the default timeout of 0.3s. + time.sleep(1.0) + self.assertRaises(threading.BrokenBarrierError, barrier.wait) + self.run_threads(f) + + def test_single_thread(self): + b = self.barriertype(1) + b.wait() + b.wait() + + +if __name__ == '__main__': + print("This module contains no tests; it is used by other test cases like test_threading_2") diff --git a/src/gevent/tests/monkey_package/__main__.py b/src/gevent/tests/monkey_package/__main__.py new file mode 100644 index 0000000..316d16f --- /dev/null +++ b/src/gevent/tests/monkey_package/__main__.py @@ -0,0 +1,5 @@ +from __future__ import print_function +# This file makes this directory into a package. +# it exists to test 'python -m gevent.monkey monkey_package' +print(__file__) +print(__name__) diff --git a/src/gevent/tests/monkey_package/issue302monkey.py b/src/gevent/tests/monkey_package/issue302monkey.py new file mode 100644 index 0000000..73b5d1f --- /dev/null +++ b/src/gevent/tests/monkey_package/issue302monkey.py @@ -0,0 +1,28 @@ +from __future__ import print_function +import socket +import sys +if sys.argv[1] == 'patched': + print('gevent' in repr(socket.socket)) +else: + assert sys.argv[1] == 'stdlib' + print('gevent' not in repr(socket.socket)) +print(__file__) + +if sys.version_info[:2] == (2, 7): + # Prior to gevent 1.3, 'python -m gevent.monkey' guaranteed this to be + # None for all python versions. + print(__package__ == None) +else: + if sys.argv[1] == 'patched': + # __package__ is handled differently, for some reason, and + # runpy doesn't let us override it. When we call it, it + # becomes ''. This appears to be against the documentation for + # runpy, which says specifically "If the supplied path + # directly references a script file (whether as source or as + # precompiled byte code), then __file__ will be set to the + # supplied path, and __spec__, __cached__, __loader__ and + # __package__ will all be set to None." + print(__package__ == '') + else: + # but the interpreter sets it to None + print(__package__ == None) diff --git a/src/gevent/tests/monkey_package/script.py b/src/gevent/tests/monkey_package/script.py new file mode 100644 index 0000000..8584fb7 --- /dev/null +++ b/src/gevent/tests/monkey_package/script.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +Test script file, to be used directly as a file. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +# We need some global imports +from textwrap import dedent + +def use_import(): + return dedent(" text") + +if __name__ == '__main__': + print(__file__) + print(__name__) + print(use_import()) diff --git a/src/gevent/tests/nullcert.pem b/src/gevent/tests/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/gevent/tests/server.crt b/src/gevent/tests/server.crt new file mode 100644 index 0000000..1379e1d --- /dev/null +++ b/src/gevent/tests/server.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICYzCCAcwCCQD5jx1Aa0dytjANBgkqhkiG9w0BAQQFADB2MQswCQYDVQQGEwJU +UzENMAsGA1UECBMEVGVzdDENMAsGA1UEBxMEVGVzdDEWMBQGA1UEChMNVGVzdCBF +dmVudGxldDENMAsGA1UECxMEVGVzdDENMAsGA1UEAxMEVGVzdDETMBEGCSqGSIb3 +DQEJARYEVGVzdDAeFw0wODA3MDgyMTExNDJaFw0xMDAyMDgwODE1MTBaMHYxCzAJ +BgNVBAYTAlRTMQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQHEwRUZXN0MRYwFAYDVQQK +Ew1UZXN0IEV2ZW50bGV0MQ0wCwYDVQQLEwRUZXN0MQ0wCwYDVQQDEwRUZXN0MRMw +EQYJKoZIhvcNAQkBFgRUZXN0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDM +WcyeIiHQuEGQxgTIvu0aOW4iRFAyUEi8pLWNCxMEHglF8k6OxFVq7XWZMDnDFVnb +ZjmQh5Tc21Ae6cXzxXln578fROXHEzXo3Is8HUlq3ug1yYOGHjxw++Opjf1uoHwP +EBUKsz/flS7knuscgFM9FO05KSPn2wHnZeIDta4yTwIDAQABMA0GCSqGSIb3DQEB +BAUAA4GBAKM71aP0r26gEEEBzovfXm1IwKav6R9/xiWsJ4pFsUXVotcaIjcVBDG1 +Z7tz688hokb+GNxsTI2gNfqanqUnfP9wZxnKRmfTSOvb5aWHIiaiMXSgjiPlqBcm +6mnSeEbSMM9cw479wWhh1YqY8tf3gYJa+sxznVWLSfVLpsjRMphe +-----END CERTIFICATE----- diff --git a/src/gevent/tests/server.key b/src/gevent/tests/server.key new file mode 100644 index 0000000..24cd8e5 --- /dev/null +++ b/src/gevent/tests/server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDMWcyeIiHQuEGQxgTIvu0aOW4iRFAyUEi8pLWNCxMEHglF8k6O +xFVq7XWZMDnDFVnbZjmQh5Tc21Ae6cXzxXln578fROXHEzXo3Is8HUlq3ug1yYOG +Hjxw++Opjf1uoHwPEBUKsz/flS7knuscgFM9FO05KSPn2wHnZeIDta4yTwIDAQAB +AoGBAKWfvq0IIvok7Ncm92ew/0D6/R1+2rT8xwdGQ/Nt31q98WwkqLEjxctlbKPd +J2PLIUomf0955BhhFH4JoSwjiHJQ6uishY7srjQQDX/Dxdi5wZAyxYCIVW/kAA9N +/u2s75hSD3s/rqAwOZ182DwAPIqJc4KQoYzvlKERSMDT1PJhAkEA5SUFsiSzBEMX +FyZ++ZMMs1vHrTu5oTK7WHznh9lk7dvsnp9BoUPqhiu8iJ7Q23zj0u5asz2czu11 +nnczXgU6XwJBAORM5Ib4I7nAsoUWn9wDiTwVQeE+D9P1ac9p7EHm7XXuf8o2irRZ +wYYfpXXsjk496YfyQFcQRMk0tU0gegCP7hECQFWRWqwoajUoPIInnPjjwbVki48U +I4CfqjgkBG3Fb5wnKRgezmpDK1vJD1FRRRsBay4EVhhi5KCdKfPv/V2ZxC8CQQCu +U5SxBytofJ8UhxkcTErvaR/8GYLGi//21GAGVop+YdaMlydE3cCrZODYcgCb+CSp +nS7KDG8p4KiMMz9VzJGxAkEAv85K6Sa3H8g9h7LwopBZ5tFNZUaFWo7lEP7DDMH0 +eckZTb1JVpyT/8zrDtsis4WlV9zVkVHxkIaad503BjqvEQ== +-----END RSA PRIVATE KEY----- diff --git a/src/gevent/tests/sha256.pem b/src/gevent/tests/sha256.pem new file mode 100644 index 0000000..01878e9 --- /dev/null +++ b/src/gevent/tests/sha256.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFxzCCA6+gAwIBAgIJALnlnf5uzTkIMA0GCSqGSIb3DQEBCwUAMEsxCzAJBgNV +BAYTAkRFMRcwFQYDVQQKEw5zY2hva29rZWtzLm9yZzEjMCEGCSqGSIb3DQEJARYU +aGFubm9Ac2Nob2tva2Vrcy5vcmcwHhcNMTAwMTI3MDAyMTI1WhcNMjAwMTI1MDAy +MTI1WjBLMQswCQYDVQQGEwJERTEXMBUGA1UEChMOc2Nob2tva2Vrcy5vcmcxIzAh +BgkqhkiG9w0BCQEWFGhhbm5vQHNjaG9rb2tla3Mub3JnMIICIjANBgkqhkiG9w0B +AQEFAAOCAg8AMIICCgKCAgEApJ4ODPwEooMW35dQPlBqdvcfkEvjhcsA7jmJfFqN +e/1T34zT44X9+KnMBSG2InacbD7eyFgjfaENFsZ87YkEBDIFZ/SHotLJZORQ8PUj +YoxPG4mjKN+yL2WthNcYbRyJreTbbDroNMuw6tkTSxeSXyYFQrKMCUfErVbZa/d5 +RvfFVk+Au9dVUFhed/Stn5cv+a0ffvpyA7ygihm1kMFICbvPeI0846tmC2Ph7rM5 +pYQyNBDOVpULODTk5Wu6jiiJJygvJWCZ1FdpsdBs5aKWHWdRhX++quGuflTTjH5d +qaIka4op9H7XksYphTDXmV+qHnva5jbPogwutDQcVsGBQcJaLmQqhsQK13bf4khE +iWJvfBLfHn8OOpY25ZwwuigJIwifNCxQeeT1FrLmyuYNhz2phPpzx065kqSUSR+A +Iw8DPE6e65UqMDKqZnID3dQeiQaFrHEV+Ibo0U/tD0YSBw5p33TMh0Es33IBWMac +m7x4hIFWdhl8W522u6qOrTswY3s8vB7blNWqMc9n7oWH8ybFf7EgKeDVtEN9AyBE +0WotXIEZWI+WvDbU1ACJXau9sQhYP/eerg7Zwr3iGUy4IQ5oUJibnjtcE+z8zmDN +pE6YcMCLJyLjXiQ3iHG9mNXzw7wPnslTbEEEukrfSlHGgW8Dm+VrNyW0JUM1bntx +vbMCAwEAAaOBrTCBqjAdBgNVHQ4EFgQUCedv7pDTuXtCxm4HTw9hUtrTvsowewYD +VR0jBHQwcoAUCedv7pDTuXtCxm4HTw9hUtrTvsqhT6RNMEsxCzAJBgNVBAYTAkRF +MRcwFQYDVQQKEw5zY2hva29rZWtzLm9yZzEjMCEGCSqGSIb3DQEJARYUaGFubm9A +c2Nob2tva2Vrcy5vcmeCCQC55Z3+bs05CDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4ICAQBHKAxA7WA/MEFjet03K8ouzEOr6Jrk2fZOuRhoDZ+9gr4FtaJB +P3Hh5D00kuSOvDnwsvCohxeNd1KTMAwVmVoH+NZkHERn3UXniUENlp18koI1ehlr +CZbXbzzE9Te9BelliSFA63q0cq0yJN1x9GyabU34XkAouCAmOqfSpKNZWZHGBHPF +bbYnZrHEMcsye6vKeTOcg1GqUHGrQM2WK0QaOwnCQv2RblI9VN+SeRoUJ44qTXdW +TwIYStsIPesacNcAQTStnHgKqIPx4zCwdx5xo8zONbXJfocqwyFqiAofvb9dN1nW +g1noVBcXB+oRBZW5CjFw87U88itq39i9+BWl835DWLBW2pVmx1QTLGv0RNgs/xVx +mWnjH4nNHvrjn6pRmqHZTk/SS0Hkl2qtDsynVxIl8EiMTfWSU3DBTuD2J/RSzuOE +eKtAbaoXkXE31jCl4FEZLITIZd8UkXacb9rN304tAK92L76JOAV+xOZxFRipmvx4 ++A9qQXgLhtP4VaDajb44V/kCKPSA0Vm3apehke9Wl8dDtagfos1e6MxSu3EVLXRF +SP2U777V77pdMSd0f/7cerKn5FjrxW1v1FaP1oIGniMk4qQNTgA/jvvhjybsPlVA +jsfnhWGbh1voJa0RQcMiRMsxpw2P1KNOEu37W2eq/vFghVztZJQUmb5iNw== +-----END CERTIFICATE----- diff --git a/src/gevent/tests/test__GreenletExit.py b/src/gevent/tests/test__GreenletExit.py new file mode 100644 index 0000000..acad1bb --- /dev/null +++ b/src/gevent/tests/test__GreenletExit.py @@ -0,0 +1,4 @@ +from gevent import GreenletExit + +assert issubclass(GreenletExit, BaseException) +assert not issubclass(GreenletExit, Exception) diff --git a/src/gevent/tests/test___config.py b/src/gevent/tests/test___config.py new file mode 100644 index 0000000..316e03a --- /dev/null +++ b/src/gevent/tests/test___config.py @@ -0,0 +1,154 @@ +# Copyright 2018 gevent contributors. See LICENSE for details. + +import os +import unittest +import sys + +from gevent import _config + +class TestResolver(unittest.TestCase): + + old_resolver = None + + def setUp(self): + if 'GEVENT_RESOLVER' in os.environ: + self.old_resolver = os.environ['GEVENT_RESOLVER'] + del os.environ['GEVENT_RESOLVER'] + + def tearDown(self): + if self.old_resolver: + os.environ['GEVENT_RESOLVER'] = self.old_resolver + + def test_key(self): + self.assertEqual(_config.Resolver.environment_key, 'GEVENT_RESOLVER') + + def test_default(self): + from gevent.resolver.thread import Resolver + + conf = _config.Resolver() + self.assertEqual(conf.get(), Resolver) + + def test_env(self): + from gevent.resolver.blocking import Resolver + + os.environ['GEVENT_RESOLVER'] = 'foo,bar,block,dnspython' + + conf = _config.Resolver() + self.assertEqual(conf.get(), Resolver) + + os.environ['GEVENT_RESOLVER'] = 'dnspython' + + # The existing value is unchanged + self.assertEqual(conf.get(), Resolver) + + # A new object reflects it + conf = _config.Resolver() + from gevent.resolver.dnspython import Resolver as DResolver + self.assertEqual(conf.get(), DResolver) + + def test_set_str_long(self): + from gevent.resolver.blocking import Resolver + conf = _config.Resolver() + conf.set('gevent.resolver.blocking.Resolver') + + self.assertEqual(conf.get(), Resolver) + + def test_set_str_short(self): + from gevent.resolver.blocking import Resolver + conf = _config.Resolver() + conf.set('block') + + self.assertEqual(conf.get(), Resolver) + + def test_set_class(self): + from gevent.resolver.blocking import Resolver + conf = _config.Resolver() + conf.set(Resolver) + + self.assertEqual(conf.get(), Resolver) + + + def test_set_through_config(self): + from gevent.resolver.thread import Resolver as Default + from gevent.resolver.blocking import Resolver + + conf = _config.Config() + self.assertEqual(conf.resolver, Default) + + conf.resolver = 'block' + self.assertEqual(conf.resolver, Resolver) + +class TestFunctions(unittest.TestCase): + + def test_validate_bool(self): + self.assertTrue(_config.validate_bool('on')) + self.assertTrue(_config.validate_bool('1')) + self.assertFalse(_config.validate_bool('off')) + self.assertFalse(_config.validate_bool('0')) + self.assertFalse(_config.validate_bool('')) + + with self.assertRaises(ValueError): + _config.validate_bool(' hmm ') + + def test_validate_invalid(self): + with self.assertRaises(ValueError): + _config.validate_invalid(self) + +class TestConfig(unittest.TestCase): + + def test__dir__(self): + self.assertEqual(sorted(_config.config.settings), + sorted(dir(_config.config))) + + def test_getattr(self): + # Bypass the property that might be set here + self.assertIsNotNone(_config.config.__getattr__('resolver')) + + def test__getattr__invalid(self): + with self.assertRaises(AttributeError): + getattr(_config.config, 'no_such_setting') + + def test_set_invalid(self): + with self.assertRaises(AttributeError): + _config.config.set('no such setting', True) + +class TestImportableSetting(unittest.TestCase): + + assertRaisesRegex = getattr(unittest.TestCase, 'assertRaisesRegex', + unittest.TestCase.assertRaisesRegexp) + def test_empty_list(self): + i = _config.ImportableSetting() + with self.assertRaisesRegex(ImportError, + "Cannot import from empty list"): + i._import_one_of([]) + + def test_path_not_supported(self): + import warnings + i = _config.ImportableSetting() + path = list(sys.path) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + with self.assertRaisesRegex(ImportError, + "Cannot import 'foo/bar/gevent.no_such_module'"): + i._import_one('foo/bar/gevent.no_such_module') + + # We restored the path + self.assertEqual(path, sys.path) + + # We did not issue a warning + self.assertEqual(len(w), 0) + + def test_non_string(self): + i = _config.ImportableSetting() + self.assertIs(i._import_one(self), self) + + def test_get_options(self): + i = _config.ImportableSetting() + self.assertEqual({}, i.get_options()) + + i.shortname_map = {'foo': 'bad/path'} + options = i.get_options() + self.assertIn('foo', options) + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test___example_servers.py b/src/gevent/tests/test___example_servers.py new file mode 100644 index 0000000..2b3453c --- /dev/null +++ b/src/gevent/tests/test___example_servers.py @@ -0,0 +1,141 @@ +import sys + +try: + from urllib import request as urllib2 +except ImportError: + import urllib2 +from unittest import SkipTest + +import socket +import ssl + +import gevent.testing as greentest +from gevent.testing import DEFAULT_XPC_SOCKET_TIMEOUT +from gevent.testing import util +from gevent.testing import params + +@greentest.skipOnCI("Timing issues sometimes lead to a connection refused") +class Test_wsgiserver(util.TestServer): + server = 'wsgiserver.py' + URL = 'http://%s:8088' % (params.DEFAULT_LOCAL_HOST_ADDR,) + PORT = 8088 + not_found_message = b'

Not Found

' + ssl_ctx = None + _use_ssl = False + + def read(self, path='/'): + url = self.URL + path + try: + kwargs = {} + if self.ssl_ctx is not None: + kwargs = {'context': self.ssl_ctx} + + response = urllib2.urlopen(url, None, + DEFAULT_XPC_SOCKET_TIMEOUT, + **kwargs) + except urllib2.HTTPError: + response = sys.exc_info()[1] + result = '%s %s' % (response.code, response.msg), response.read() + # XXX: It looks like under PyPy this isn't directly closing the socket + # when SSL is in use. It takes a GC cycle to make that true. + response.close() + return result + + def _test_hello(self): + status, data = self.read('/') + self.assertEqual(status, '200 OK') + self.assertEqual(data, b"hello world") + + def _test_not_found(self): + status, data = self.read('/xxx') + self.assertEqual(status, '404 Not Found') + self.assertEqual(data, self.not_found_message) + + def _do_test_a_blocking_client(self): + # We spawn this in a separate server because if it's broken + # the whole server hangs + with self.running_server(): + # First, make sure we can talk to it. + self._test_hello() + # Now create a connection and only partway finish + # the transaction + sock = socket.create_connection(('localhost', self.PORT)) + ssl_sock = None + if self._use_ssl: + ssl_sock = ssl.wrap_socket(sock) + sock_file = ssl_sock.makefile(mode='rwb') + else: + sock_file = sock.makefile(mode='rwb') + # write an incomplete request + sock_file.write(b'GET /xxx HTTP/1.0\r\n') + sock_file.flush() + # Leave it open and not doing anything + # while the other request runs to completion. + # This demonstrates that a blocking client + # doesn't hang the whole server + self._test_hello() + + # now finish the original request + sock_file.write(b'\r\n') + sock_file.flush() + line = sock_file.readline() + self.assertEqual(line, b'HTTP/1.1 404 Not Found\r\n') + + sock_file.close() + if ssl_sock is not None: + ssl_sock.close() + sock.close() + + def test_a_blocking_client(self): + self._do_test_a_blocking_client() + +@greentest.skipOnCI("Timing issues sometimes lead to a connection refused") +class Test_wsgiserver_ssl(Test_wsgiserver): + server = 'wsgiserver_ssl.py' + URL = 'https://%s:8443' % (params.DEFAULT_LOCAL_HOST_ADDR,) + PORT = 8443 + _use_ssl = True + + if hasattr(ssl, '_create_unverified_context'): + # Disable verification for our self-signed cert + # on Python >= 2.7.9 and 3.4 + ssl_ctx = ssl._create_unverified_context() + + +@greentest.skipOnCI("Timing issues sometimes lead to a connection refused") +class Test_webproxy(Test_wsgiserver): + server = 'webproxy.py' + + def _run_all_tests(self): + status, data = self.read('/') + self.assertEqual(status, '200 OK') + self.assertIn(b"gevent example", data) + status, data = self.read('/http://www.google.com') + self.assertEqual(status, '200 OK') + self.assertIn(b'google', data.lower()) + + def test_a_blocking_client(self): + # Not applicable + raise SkipTest("Not applicable") + + +# class Test_webpy(Test_wsgiserver): +# server = 'webpy.py' +# not_found_message = 'not found' +# +# def _test_hello(self): +# status, data = self.read('/') +# self.assertEqual(status, '200 OK') +# assert "Hello, world" in data, repr(data) +# +# def _test_long(self): +# start = time.time() +# status, data = self.read('/long') +# delay = time.time() - start +# assert 10 - 0.5 < delay < 10 + 0.5, delay +# self.assertEqual(status, '200 OK') +# self.assertEqual(data, 'Hello, 10 seconds later') + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test___ident.py b/src/gevent/tests/test___ident.py new file mode 100644 index 0000000..aa82d41 --- /dev/null +++ b/src/gevent/tests/test___ident.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +# copyright 2018 gevent contributors. See LICENSE for details. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gc + + +import gevent.testing as greentest +from gevent._ident import IdentRegistry +from gevent._compat import PYPY + +class Target(object): + pass + +class TestIdent(greentest.TestCase): + + def setUp(self): + self.reg = IdentRegistry() + + def tearDown(self): + self.reg = None + + def test_basic(self): + target = Target() + self.assertEqual(0, self.reg.get_ident(target)) + self.assertEqual(1, len(self.reg)) + + self.assertEqual(0, self.reg.get_ident(target)) + self.assertEqual(1, len(self.reg)) + + target2 = Target() + self.assertEqual(1, self.reg.get_ident(target2)) + self.assertEqual(2, len(self.reg)) + + self.assertEqual(1, self.reg.get_ident(target2)) + self.assertEqual(2, len(self.reg)) + + self.assertEqual(0, self.reg.get_ident(target)) + + # When an object dies, we can re-use + # its id. Under PyPy we need to collect garbage first. + del target + if PYPY: + for _ in range(3): + gc.collect() + + self.assertEqual(1, len(self.reg)) + + target3 = Target() + self.assertEqual(1, self.reg.get_ident(target2)) + self.assertEqual(0, self.reg.get_ident(target3)) + self.assertEqual(2, len(self.reg)) + + @greentest.skipOnPyPy("This would need to GC very frequently") + def test_circle(self): + keep_count = 3 + keepalive = [None] * keep_count + + for i in range(1000): + target = Target() + # Drop an old one. + keepalive[i % keep_count] = target + self.assertLessEqual(self.reg.get_ident(target), keep_count) + + +@greentest.skipOnPurePython("Needs C extension") +class TestCExt(greentest.TestCase): + + def test_c_extension(self): + self.assertEqual(IdentRegistry.__module__, + 'gevent.__ident') + + + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test___monitor.py b/src/gevent/tests/test___monitor.py new file mode 100644 index 0000000..e5a003a --- /dev/null +++ b/src/gevent/tests/test___monitor.py @@ -0,0 +1,386 @@ +# Copyright 2018 gevent contributors. See LICENSE for details. + +import gc +import unittest + + +from greenlet import gettrace +from greenlet import settrace + +from gevent.monkey import get_original +from gevent._compat import thread_mod_name +from gevent._compat import NativeStrIO + +from gevent.testing.skipping import skipOnPyPyOnWindows + +from gevent import _monitor as monitor +from gevent import config as GEVENT_CONFIG + +get_ident = get_original(thread_mod_name, 'get_ident') + +class MockHub(object): + _threadpool = None + _resolver = None + + def __init__(self): + self.thread_ident = get_ident() + self.exception_stream = NativeStrIO() + self.dead = False + + def __bool__(self): + return not self.dead + + __nonzero__ = __bool__ + + def handle_error(self, *args): # pylint:disable=unused-argument + raise # pylint:disable=misplaced-bare-raise + + @property + def loop(self): + return self + + def reinit(self): + "mock loop.reinit" + +class _AbstractTestPeriodicMonitoringThread(object): + # Makes sure we don't actually spin up a new monitoring thread. + + # pylint:disable=no-member + + def setUp(self): + super(_AbstractTestPeriodicMonitoringThread, self).setUp() + self._orig_start_new_thread = monitor.start_new_thread + self._orig_thread_sleep = monitor.thread_sleep + monitor.thread_sleep = lambda _s: gc.collect() # For PyPy + self.tid = 0xDEADBEEF + def start_new_thread(_f, _a): + r = self.tid + self.tid += 1 + return r + + monitor.start_new_thread = start_new_thread + self.hub = MockHub() + self.pmt = monitor.PeriodicMonitoringThread(self.hub) + self.hub.periodic_monitoring_thread = self.pmt + self.pmt_default_funcs = self.pmt.monitoring_functions()[:] + self.len_pmt_default_funcs = len(self.pmt_default_funcs) + + def tearDown(self): + monitor.start_new_thread = self._orig_start_new_thread + monitor.thread_sleep = self._orig_thread_sleep + prev = self.pmt._greenlet_tracer.previous_trace_function + self.pmt.kill() + assert gettrace() is prev, (gettrace(), prev) + settrace(None) + super(_AbstractTestPeriodicMonitoringThread, self).tearDown() + + +class TestPeriodicMonitoringThread(_AbstractTestPeriodicMonitoringThread, + unittest.TestCase): + + def test_constructor(self): + self.assertEqual(0xDEADBEEF, self.pmt.monitor_thread_ident) + self.assertEqual(gettrace(), self.pmt._greenlet_tracer) + + @skipOnPyPyOnWindows("psutil doesn't install on PyPy on Win") + def test_get_process(self): + proc = self.pmt._get_process() + self.assertIsNotNone(proc) + self.assertIs(proc, self.pmt._get_process()) + + def test_hub_wref(self): + self.assertIs(self.hub, self.pmt.hub) + del self.hub + + gc.collect() + self.assertIsNone(self.pmt.hub) + + # And it killed itself. + self.assertFalse(self.pmt.should_run) + self.assertIsNone(gettrace()) + + + def test_add_monitoring_function(self): + + self.assertRaises(ValueError, self.pmt.add_monitoring_function, None, 1) + self.assertRaises(ValueError, self.pmt.add_monitoring_function, lambda: None, -1) + + def f(): + "Does nothing" + + # Add + self.pmt.add_monitoring_function(f, 1) + self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions())) + self.assertEqual(1, self.pmt.monitoring_functions()[1].period) + + # Update + self.pmt.add_monitoring_function(f, 2) + self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions())) + self.assertEqual(2, self.pmt.monitoring_functions()[1].period) + + # Remove + self.pmt.add_monitoring_function(f, None) + self.assertEqual(self.len_pmt_default_funcs, len(self.pmt.monitoring_functions())) + + def test_calculate_sleep_time(self): + self.assertEqual( + self.pmt.monitoring_functions()[0].period, + self.pmt.calculate_sleep_time()) + + # Pretend that GEVENT_CONFIG.max_blocking_time was set to 0, + # to disable this monitor. + self.pmt._calculated_sleep_time = 0 + self.assertEqual( + self.pmt.inactive_sleep_time, + self.pmt.calculate_sleep_time() + ) + + # Getting the list of monitoring functions will also + # do this, if it looks like it has changed + self.pmt.monitoring_functions()[0].period = -1 + self.pmt._calculated_sleep_time = 0 + self.pmt.monitoring_functions() + self.assertEqual( + self.pmt.monitoring_functions()[0].period, + self.pmt.calculate_sleep_time()) + self.assertEqual( + self.pmt.monitoring_functions()[0].period, + self.pmt._calculated_sleep_time) + + def test_call_destroyed_hub(self): + # Add a function that destroys the hub so we break out (eventually) + # This clears the wref, which eventually calls kill() + def f(_hub): + _hub = None + self.hub = None + gc.collect() + + self.pmt.add_monitoring_function(f, 0.1) + self.pmt() + self.assertFalse(self.pmt.should_run) + + def test_call_dead_hub(self): + # Add a function that makes the hub go false (e.g., it quit) + # This causes the function to kill itself. + def f(hub): + hub.dead = True + self.pmt.add_monitoring_function(f, 0.1) + self.pmt() + self.assertFalse(self.pmt.should_run) + + def test_call_SystemExit(self): + # breaks the loop + def f(_hub): + raise SystemExit() + + self.pmt.add_monitoring_function(f, 0.1) + self.pmt() + + def test_call_other_error(self): + class MyException(Exception): + pass + + def f(_hub): + raise MyException() + + self.pmt.add_monitoring_function(f, 0.1) + with self.assertRaises(MyException): + self.pmt() + + def test_hub_reinit(self): + import os + from gevent.hub import reinit + self.pmt.pid = -1 + old_tid = self.pmt.monitor_thread_ident + + reinit(self.hub) + + self.assertEqual(os.getpid(), self.pmt.pid) + self.assertEqual(old_tid + 1, self.pmt.monitor_thread_ident) + + + +class TestPeriodicMonitorBlocking(_AbstractTestPeriodicMonitoringThread, + unittest.TestCase): + + def test_previous_trace(self): + self.pmt.kill() + self.assertIsNone(gettrace()) + + called = [] + def f(*args): + called.append(args) + + settrace(f) + + self.pmt = monitor.PeriodicMonitoringThread(self.hub) + self.assertEqual(gettrace(), self.pmt._greenlet_tracer) + self.assertIs(self.pmt._greenlet_tracer.previous_trace_function, f) + + self.pmt._greenlet_tracer('event', ('args',)) + + self.assertEqual([('event', ('args',))], called) + + def test__greenlet_tracer(self): + self.assertEqual(0, self.pmt._greenlet_tracer.greenlet_switch_counter) + # Unknown event still counts as a switch (should it?) + self.pmt._greenlet_tracer('unknown', None) + self.assertEqual(1, self.pmt._greenlet_tracer.greenlet_switch_counter) + self.assertIsNone(self.pmt._greenlet_tracer.active_greenlet) + + origin = object() + target = object() + + self.pmt._greenlet_tracer('switch', (origin, target)) + self.assertEqual(2, self.pmt._greenlet_tracer.greenlet_switch_counter) + self.assertIs(target, self.pmt._greenlet_tracer.active_greenlet) + + # Unknown event removes active greenlet + self.pmt._greenlet_tracer('unknown', ()) + self.assertEqual(3, self.pmt._greenlet_tracer.greenlet_switch_counter) + self.assertIsNone(self.pmt._greenlet_tracer.active_greenlet) + + def test_monitor_blocking(self): + # Initially there's no active greenlet and no switches, + # so nothing is considered blocked + from gevent.events import subscribers + from gevent.events import IEventLoopBlocked + from zope.interface.verify import verifyObject + events = [] + subscribers.append(events.append) + + self.assertFalse(self.pmt.monitor_blocking(self.hub)) + + # Give it an active greenlet + origin = object() + target = object() + self.pmt._greenlet_tracer('switch', (origin, target)) + + # We've switched, so we're not blocked + self.assertFalse(self.pmt.monitor_blocking(self.hub)) + self.assertFalse(events) + + # Again without switching is a problem. + self.assertTrue(self.pmt.monitor_blocking(self.hub)) + self.assertTrue(events) + verifyObject(IEventLoopBlocked, events[0]) + del events[:] + + # But we can order it not to be a problem + self.pmt.ignore_current_greenlet_blocking() + self.assertFalse(self.pmt.monitor_blocking(self.hub)) + self.assertFalse(events) + + # And back again + self.pmt.monitor_current_greenlet_blocking() + self.assertTrue(self.pmt.monitor_blocking(self.hub)) + + # A bad thread_ident in the hub doesn't mess things up + self.hub.thread_ident = -1 + self.assertTrue(self.pmt.monitor_blocking(self.hub)) + + +class MockProcess(object): + + def __init__(self, rss): + self.rss = rss + + def memory_full_info(self): + return self + + +@skipOnPyPyOnWindows("psutil doesn't install on PyPy on Win") +class TestPeriodicMonitorMemory(_AbstractTestPeriodicMonitoringThread, + unittest.TestCase): + + rss = 0 + + def setUp(self): + super(TestPeriodicMonitorMemory, self).setUp() + self._old_max = GEVENT_CONFIG.max_memory_usage + GEVENT_CONFIG.max_memory_usage = None + + self.pmt._get_process = lambda: MockProcess(self.rss) + + def tearDown(self): + GEVENT_CONFIG.max_memory_usage = self._old_max + super(TestPeriodicMonitorMemory, self).tearDown() + + def test_can_monitor_and_install(self): + + # We run tests with psutil installed, and we have access to our + # process. + self.assertTrue(self.pmt.can_monitor_memory_usage()) + # No warning, adds a function + + self.pmt.install_monitor_memory_usage() + self.assertEqual(self.len_pmt_default_funcs + 1, len(self.pmt.monitoring_functions())) + + def test_cannot_monitor_and_install(self): + import warnings + self.pmt._get_process = lambda: None + self.assertFalse(self.pmt.can_monitor_memory_usage()) + + # This emits a warning, visible by default + with warnings.catch_warnings(record=True) as ws: + self.pmt.install_monitor_memory_usage() + + self.assertEqual(1, len(ws)) + self.assertIs(monitor.MonitorWarning, ws[0].category) + + def test_monitor_no_allowed(self): + self.assertEqual(-1, self.pmt.monitor_memory_usage(None)) + + def test_monitor_greater(self): + from gevent import events + + self.rss = 2 + GEVENT_CONFIG.max_memory_usage = 1 + + # Initial event + event = self.pmt.monitor_memory_usage(None) + self.assertIsInstance(event, events.MemoryUsageThresholdExceeded) + self.assertEqual(2, event.mem_usage) + self.assertEqual(1, event.max_allowed) + self.assertIsInstance(event.memory_info, MockProcess) + + # No growth, no event + event = self.pmt.monitor_memory_usage(None) + self.assertIsNone(event) + + # Growth, event + self.rss = 3 + event = self.pmt.monitor_memory_usage(None) + self.assertIsInstance(event, events.MemoryUsageThresholdExceeded) + self.assertEqual(3, event.mem_usage) + + # Shrinking below gets us back + self.rss = 1 + event = self.pmt.monitor_memory_usage(None) + self.assertIsInstance(event, events.MemoryUsageUnderThreshold) + self.assertEqual(1, event.mem_usage) + + # coverage + repr(event) + + # No change, no event + event = self.pmt.monitor_memory_usage(None) + self.assertIsNone(event) + + # Growth, event + self.rss = 3 + event = self.pmt.monitor_memory_usage(None) + self.assertIsInstance(event, events.MemoryUsageThresholdExceeded) + self.assertEqual(3, event.mem_usage) + + + def test_monitor_initial_below(self): + self.rss = 1 + GEVENT_CONFIG.max_memory_usage = 10 + + + event = self.pmt.monitor_memory_usage(None) + self.assertIsNone(event) + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test___monkey_patching.py b/src/gevent/tests/test___monkey_patching.py new file mode 100644 index 0000000..ea82cc5 --- /dev/null +++ b/src/gevent/tests/test___monkey_patching.py @@ -0,0 +1,113 @@ +import sys +import os +import glob + +import atexit +# subprocess: include in subprocess tests + +from gevent.testing import util + +TIMEOUT = 120 + +# XXX: Generalize this so other packages can use it. + +def find_stdlib_tests(): + setup_py = util.search_for_setup_py(a_file=__file__) + greentest = os.path.join(setup_py, 'src', 'greentest') + + + directory = '%s.%s' % sys.version_info[:2] + full_directory = '%s.%s.%s' % sys.version_info[:3] + if hasattr(sys, 'pypy_version_info'): + directory += 'pypy' + full_directory += 'pypy' + + directory = os.path.join(greentest, directory) + full_directory = os.path.join(greentest, full_directory) + + return directory, full_directory + +def get_python_version(): + version = '%s.%s.%s' % sys.version_info[:3] + if sys.version_info[3] == 'alpha': + version += 'a%s' % sys.version_info[4] + elif sys.version_info[3] == 'beta': + version += 'b%s' % sys.version_info[4] + + return version + +def get_absolute_pythonpath(): + paths = [os.path.abspath(p) for p in os.environ.get('PYTHONPATH', '').split(os.pathsep)] + return os.pathsep.join(paths) + + +def TESTRUNNER(tests=None): + try: + test_dir, version_test_dir = find_stdlib_tests() + except util.NoSetupPyFound as e: + util.log("WARNING: No setup.py and src/greentest found: %r", e, + color="suboptimal-behaviour") + return + + if not os.path.exists(test_dir): + util.log('WARNING: No test directory found at %s', test_dir, + color="suboptimal-behaviour") + return + + with open(os.path.join(test_dir, 'version')) as f: + preferred_version = f.read().strip() + + running_version = get_python_version() + if preferred_version != running_version: + util.log('WARNING: The tests in %s/ are from version %s and your Python is %s', + test_dir, preferred_version, running_version, + color="suboptimal-behaviour") + + version_tests = glob.glob('%s/test_*.py' % version_test_dir) + version_tests = sorted(version_tests) + if not tests: + tests = glob.glob('%s/test_*.py' % test_dir) + tests = sorted(tests) + + PYTHONPATH = (os.getcwd() + os.pathsep + get_absolute_pythonpath()).rstrip(':') + + tests = [os.path.basename(x) for x in tests] + version_tests = [os.path.basename(x) for x in version_tests] + + options = { + 'cwd': test_dir, + 'timeout': TIMEOUT, + 'setenv': { + 'PYTHONPATH': PYTHONPATH, + # debug produces resource tracking warnings for the + # CFFI backends. On Python 2, many of the stdlib tests + # rely on refcounting to close sockets so they produce + # lots of noise. Python 3 is not completely immune; + # test_ftplib.py tends to produce warnings---and the Python 3 + # test framework turns those into test failures! + 'GEVENT_DEBUG': 'error', + } + } + + if tests and not sys.platform.startswith("win"): + atexit.register(os.system, 'rm -f */@test*') + + basic_args = [sys.executable, '-u', '-W', 'ignore', '-m', 'gevent.testing.monkey_test'] + for filename in tests: + if filename in version_tests: + util.log("Overriding %s from %s with file from %s", filename, test_dir, version_test_dir) + continue + yield basic_args + [filename], options.copy() + + options['cwd'] = version_test_dir + for filename in version_tests: + yield basic_args + [filename], options.copy() + + +def main(): + from gevent.testing import testrunner + return testrunner.run_many(list(TESTRUNNER(sys.argv[1:]))) + + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__all__.py b/src/gevent/tests/test__all__.py new file mode 100644 index 0000000..0efbe2c --- /dev/null +++ b/src/gevent/tests/test__all__.py @@ -0,0 +1,241 @@ +"""Check __all__, __implements__, __extensions__, __imports__ of the modules""" +from __future__ import print_function + +import sys +import unittest +import types +import importlib +import warnings + +from gevent.testing import six +from gevent.testing.modules import walk_modules +from gevent.testing.sysinfo import PLATFORM_SPECIFIC_SUFFIXES + +from gevent._patcher import MAPPING + +class ANY(object): + def __contains__(self, item): + return True + +ANY = ANY() + +NOT_IMPLEMENTED = { + 'socket': ['CAPI'], + 'thread': ['allocate', 'exit_thread', 'interrupt_main', 'start_new'], + 'select': ANY, + 'os': ANY, + 'threading': ANY, + 'builtins' if six.PY3 else '__builtin__': ANY, + 'signal': ANY, +} + +COULD_BE_MISSING = { + 'socket': ['create_connection', 'RAND_add', 'RAND_egd', 'RAND_status'], + 'subprocess': ['_posixsubprocess'], +} + +# Things without an __all__ should generally be internal implementation +# helpers +NO_ALL = { + 'gevent.threading', + 'gevent._util', + 'gevent._compat', + 'gevent._socketcommon', + 'gevent._fileobjectcommon', + 'gevent._fileobjectposix', + 'gevent._tblib', + 'gevent._corecffi', + 'gevent._patcher', + 'gevent._ffi', +} + +ALLOW_IMPLEMENTS = [ + 'gevent._queue', +] + +# A list of modules that may contain things that aren't actually, technically, +# extensions, but that need to be in __extensions__ anyway due to the way, +# for example, monkey patching, needs to work. +EXTRA_EXTENSIONS = [] +if sys.platform.startswith('win'): + EXTRA_EXTENSIONS.append('gevent.signal') + + +class Test(unittest.TestCase): + + stdlib_has_all = False + stdlib_all = () + stdlib_name = None + stdlib_module = None + module = None + __implements__ = __extensions__ = __imports__ = () + + def check_all(self): + "Check that __all__ is present and does not contain invalid entries" + if not hasattr(self.module, '__all__'): + self.assertIn(self.modname, NO_ALL) + return + names = {} + six.exec_("from %s import *" % self.modname, names) + names.pop('__builtins__', None) + self.assertEqual(sorted(names), sorted(self.module.__all__)) + + def check_all_formula(self): + "Check __all__ = __implements__ + __extensions__ + __imported__" + all_calculated = self.__implements__ + self.__imports__ + self.__extensions__ + self.assertEqual(sorted(all_calculated), sorted(self.module.__all__)) + + def check_implements_presence_justified(self): + "Check that __implements__ is present only if the module is modeled after a module from stdlib (like gevent.socket)." + if self.modname in ALLOW_IMPLEMENTS: + return + if self.__implements__ is not None and self.stdlib_module is None: + raise AssertionError('%r has __implements__ but no stdlib counterpart (%s)' + % (self.modname, self.stdlib_name)) + + def set_stdlib_all(self): + self.assertIsNotNone(self.stdlib_module) + self.stdlib_has_all = True + self.stdlib_all = getattr(self.stdlib_module, '__all__', None) + if self.stdlib_all is None: + self.stdlib_has_all = False + self.stdlib_all = dir(self.stdlib_module) + self.stdlib_all = [name for name in self.stdlib_all if not name.startswith('_')] + self.stdlib_all = [name for name in self.stdlib_all if not isinstance(getattr(self.stdlib_module, name), types.ModuleType)] + + def check_implements_subset_of_stdlib_all(self): + "Check that __implements__ + __imports__ is a subset of the corresponding standard module __all__ or dir()" + for name in self.__implements__ + self.__imports__: + if name in self.stdlib_all: + continue + if name in COULD_BE_MISSING.get(self.stdlib_name, ()): + continue + if name in dir(self.stdlib_module): # like thread._local which is not in thread.__all__ + continue + raise AssertionError('%r is not found in %r.__all__ nor in dir(%r)' % (name, self.stdlib_module, self.stdlib_module)) + + def check_implements_actually_implements(self): + """Check that the module actually implements the entries from __implements__""" + for name in self.__implements__: + item = getattr(self.module, name) + try: + stdlib_item = getattr(self.stdlib_module, name) + self.assertIsNot(item, stdlib_item) + except AttributeError: + if name not in COULD_BE_MISSING.get(self.stdlib_name, []): + raise + + def check_imports_actually_imports(self): + """Check that the module actually imports the entries from __imports__""" + for name in self.__imports__: + item = getattr(self.module, name) + stdlib_item = getattr(self.stdlib_module, name) + self.assertIs(item, stdlib_item) + + def check_extensions_actually_extend(self): + """Check that the module actually defines new entries in __extensions__""" + if self.modname in EXTRA_EXTENSIONS: + return + for name in self.__extensions__: + if hasattr(self.stdlib_module, name): + raise AssertionError("'%r' is not an extension, it is found in %r" % (name, self.stdlib_module)) + + def check_completeness(self): # pylint:disable=too-many-branches + """Check that __all__ (or dir()) of the corresponsing stdlib is a subset of __all__ of this module""" + missed = [] + for name in self.stdlib_all: + if name not in getattr(self.module, '__all__', []): + missed.append(name) + + # handle stuff like ssl.socket and ssl.socket_error which have no reason to be in gevent.ssl.__all__ + if not self.stdlib_has_all: + for name in missed[:]: + if hasattr(self.module, name): + missed.remove(name) + + # remove known misses + not_implemented = NOT_IMPLEMENTED.get(self.stdlib_name) + if not_implemented is not None: + result = [] + for name in missed: + if name in not_implemented: + # We often don't want __all__ to be set because we wind up + # documenting things that we just copy in from the stdlib. + # But if we implement it, don't print a warning + if getattr(self.module, name, self) is self: + print('IncompleteImplWarning: %s.%s' % (self.modname, name)) + else: + result.append(name) + missed = result + + if missed: + if self.stdlib_has_all: + msg = '''The following items + in %r.__all__ +are missing from %r: + %r''' % (self.stdlib_module, self.module, missed) + else: + msg = '''The following items + in dir(%r) +are missing from %r: + %r''' % (self.stdlib_module, self.module, missed) + raise AssertionError(msg) + + def _test(self, modname): + if modname.endswith(PLATFORM_SPECIFIC_SUFFIXES): + return + + self.modname = modname + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + self.module = importlib.import_module(modname) + + self.check_all() + + self.__implements__ = getattr(self.module, '__implements__', None) + self.__imports__ = getattr(self.module, '__imports__', []) + self.__extensions__ = getattr(self.module, '__extensions__', []) + + self.stdlib_name = MAPPING.get(modname) + self.stdlib_module = None + + if self.stdlib_name is not None: + try: + self.stdlib_module = __import__(self.stdlib_name) + except ImportError: + pass + + self.check_implements_presence_justified() + + if self.stdlib_module is None: + return + + # use __all__ as __implements__ + if self.__implements__ is None: + self.__implements__ = sorted(self.module.__all__) + + self.set_stdlib_all() + self.check_implements_subset_of_stdlib_all() + self.check_implements_actually_implements() + self.check_imports_actually_imports() + self.check_extensions_actually_extend() + self.check_completeness() + + path = modname = orig_modname = None + + for path, modname in walk_modules(include_so=False, recursive=True): + orig_modname = modname + modname = modname.replace('gevent.', '').split('.')[0] + if not modname: + print("WARNING: No such module '%s' at '%s'" % (orig_modname, path), + file=sys.stderr) + continue + exec( + '''def test_%s(self): self._test("%s")''' % ( + orig_modname.replace('.', '_').replace('-', '_'), orig_modname) + ) + del path, modname, orig_modname + + +if __name__ == "__main__": + unittest.main() diff --git a/src/gevent/tests/test__api.py b/src/gevent/tests/test__api.py new file mode 100644 index 0000000..697a3e0 --- /dev/null +++ b/src/gevent/tests/test__api.py @@ -0,0 +1,132 @@ +# Copyright (c) 2008 AG Projects +# Author: Denis Bilenko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import gevent.testing as greentest +import gevent +from gevent import util, socket + +DELAY = 0.1 + + +class Test(greentest.TestCase): + + @greentest.skipOnAppVeyor("Timing causes the state to often be [start,finished]") + def test_killing_dormant(self): + state = [] + + def test(): + try: + state.append('start') + gevent.sleep(DELAY * 3.0) + except: # pylint:disable=bare-except + state.append('except') + # catching GreenletExit + + state.append('finished') + + g = gevent.spawn(test) + gevent.sleep(DELAY / 2) + assert state == ['start'], state + g.kill() + # will not get there, unless switching is explicitly scheduled by kill + self.assertEqual(state, ['start', 'except', 'finished']) + + def test_nested_with_timeout(self): + def func(): + return gevent.with_timeout(0.2, gevent.sleep, 2, timeout_value=1) + self.assertRaises(gevent.Timeout, gevent.with_timeout, 0.1, func) + + def test_sleep_invalid_switch(self): + p = gevent.spawn(util.wrap_errors(AssertionError, gevent.sleep), 2) + gevent.sleep(0) # wait for p to start, because actual order of switching is reversed + switcher = gevent.spawn(p.switch, None) + result = p.get() + assert isinstance(result, AssertionError), result + assert 'Invalid switch' in str(result), repr(str(result)) + switcher.kill() + + if hasattr(socket, 'socketpair'): + + def _test_wait_read_invalid_switch(self, sleep): + sock1, sock2 = socket.socketpair() + try: + p = gevent.spawn(util.wrap_errors(AssertionError, + socket.wait_read), # pylint:disable=no-member + sock1.fileno()) + gevent.get_hub().loop.run_callback(switch_None, p) + if sleep is not None: + gevent.sleep(sleep) + result = p.get() + assert isinstance(result, AssertionError), result + assert 'Invalid switch' in str(result), repr(str(result)) + finally: + sock1.close() + sock2.close() + + def test_invalid_switch_None(self): + self._test_wait_read_invalid_switch(None) + + def test_invalid_switch_0(self): + self._test_wait_read_invalid_switch(0) + + def test_invalid_switch_1(self): + self._test_wait_read_invalid_switch(0.001) + + # we don't test wait_write the same way, because socket is always ready to write + + +def switch_None(g): + g.switch(None) + + +class TestTimers(greentest.TestCase): + + def test_timer_fired(self): + lst = [1] + + def func(): + gevent.spawn_later(0.01, lst.pop) + gevent.sleep(0.02) + + gevent.spawn(func) + # Func has not run yet + self.assertEqual(lst, [1]) + # Run callbacks but don't yield. + gevent.sleep() + + # Let timers fire. Func should be done. + gevent.sleep(0.1) + self.assertEqual(lst, []) + + + def test_spawn_is_not_cancelled(self): + lst = [1] + + def func(): + gevent.spawn(lst.pop) + # exiting immediately, but self.lst.pop must be called + gevent.spawn(func) + gevent.sleep(0.1) + self.assertEqual(lst, []) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__api_timeout.py b/src/gevent/tests/test__api_timeout.py new file mode 100644 index 0000000..98eb152 --- /dev/null +++ b/src/gevent/tests/test__api_timeout.py @@ -0,0 +1,210 @@ +# Copyright (c) 2008 AG Projects +# Author: Denis Bilenko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import sys +import gevent.testing as greentest +import weakref +import time +import gc + +from gevent import sleep +from gevent import Timeout +from gevent import get_hub + + +from gevent.testing.timing import SMALL_TICK as DELAY +from gevent.testing import flaky + + +class Error(Exception): + pass + + +class _UpdateNowProxy(object): + + update_now_calls = 0 + + def __init__(self, loop): + self.loop = loop + + def __getattr__(self, name): + return getattr(self.loop, name) + + def update_now(self): + self.update_now_calls += 1 + self.loop.update_now() + +class _UpdateNowWithTimerProxy(_UpdateNowProxy): + + def timer(self, *_args, **_kwargs): + return _Timer(self) + +class _Timer(object): + + pending = False + active = False + + def __init__(self, loop): + self.loop = loop + + def start(self, *_args, **kwargs): + if kwargs.get("update"): + self.loop.update_now() + self.pending = self.active = True + + def stop(self): + self.active = self.pending = False + + def close(self): + "Does nothing" + + +class Test(greentest.TestCase): + + def test_timeout_calls_update_now(self): + hub = get_hub() + loop = hub.loop + proxy = _UpdateNowWithTimerProxy(loop) + hub.loop = proxy + + try: + with Timeout(DELAY * 2) as t: + self.assertTrue(t.pending) + finally: + hub.loop = loop + + self.assertEqual(1, proxy.update_now_calls) + + def test_sleep_calls_update_now(self): + hub = get_hub() + loop = hub.loop + proxy = _UpdateNowProxy(loop) + hub.loop = proxy + try: + sleep(0.01) + finally: + hub.loop = loop + + self.assertEqual(1, proxy.update_now_calls) + + + @greentest.skipOnAppVeyor("Timing is flaky, especially under Py 3.4/64-bit") + @greentest.skipOnPyPy3OnCI("Timing is flaky, especially under Py 3.4/64-bit") + @greentest.reraises_flaky_timeout(Timeout) + def test_api(self): + # Nothing happens if with-block finishes before the timeout expires + t = Timeout(DELAY * 2) + self.assertFalse(t.pending, t) + with t: + self.assertTrue(t.pending, t) + sleep(DELAY) + # check if timer was actually cancelled + self.assertFalse(t.pending, t) + sleep(DELAY * 2) + + # An exception will be raised if it's not + with self.assertRaises(Timeout) as exc: + with Timeout(DELAY) as t: + sleep(DELAY * 10) + + self.assertIs(exc.exception, t) + + # You can customize the exception raised: + with self.assertRaises(IOError): + with Timeout(DELAY, IOError("Operation takes way too long")): + sleep(DELAY * 10) + + # Providing classes instead of values should be possible too: + with self.assertRaises(ValueError): + with Timeout(DELAY, ValueError): + sleep(DELAY * 10) + + + try: + 1 / 0 + except ZeroDivisionError: + with self.assertRaises(ZeroDivisionError): + with Timeout(DELAY, sys.exc_info()[0]): + sleep(DELAY * 10) + raise AssertionError('should not get there') + raise AssertionError('should not get there') + else: + raise AssertionError('should not get there') + + # It's possible to cancel the timer inside the block: + with Timeout(DELAY) as timer: + timer.cancel() + sleep(DELAY * 2) + + # To silent the exception before exiting the block, pass False as second parameter. + XDELAY = 0.1 + start = time.time() + with Timeout(XDELAY, False): + sleep(XDELAY * 2) + delta = (time.time() - start) + self.assertTimeWithinRange(delta, 0, XDELAY * 2) + + # passing None as seconds disables the timer + with Timeout(None): + sleep(DELAY) + sleep(DELAY) + + def test_ref(self): + err = Error() + err_ref = weakref.ref(err) + with Timeout(DELAY * 2, err): + sleep(DELAY) + del err + gc.collect() + self.assertFalse(err_ref(), err_ref) + + @flaky.reraises_flaky_race_condition() + def test_nested_timeout(self): + with Timeout(DELAY, False): + with Timeout(DELAY * 10, False): + sleep(DELAY * 3 * 20) + raise AssertionError('should not get there') + + with Timeout(DELAY) as t1: + with Timeout(DELAY * 20) as t2: + with self.assertRaises(Timeout) as exc: + sleep(DELAY * 30) + self.assertIs(exc.exception, t1) + + self.assertFalse(t1.pending, t1) + self.assertTrue(t2.pending, t2) + + self.assertFalse(t2.pending) + + with Timeout(DELAY * 20) as t1: + with Timeout(DELAY) as t2: + with self.assertRaises(Timeout) as exc: + sleep(DELAY * 30) + self.assertIs(exc.exception, t2) + + self.assertTrue(t1.pending, t1) + self.assertFalse(t2.pending, t2) + + self.assertFalse(t1.pending) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__ares_host_result.py b/src/gevent/tests/test__ares_host_result.py new file mode 100644 index 0000000..2ea36c8 --- /dev/null +++ b/src/gevent/tests/test__ares_host_result.py @@ -0,0 +1,31 @@ +from __future__ import print_function + +import pickle +import gevent.testing as greentest +try: + from gevent.resolver.cares import ares_host_result +except ImportError: # pragma: no cover + ares_host_result = None + + +@greentest.skipIf(ares_host_result is None, + "Must be able to import ares") +class TestPickle(greentest.TestCase): + # Issue 104: ares.ares_host_result unpickleable + + def _test(self, protocol): + r = ares_host_result('family', ('arg1', 'arg2', )) + dumped = pickle.dumps(r, protocol) + loaded = pickle.loads(dumped) + self.assertEqual(r, loaded) + self.assertEqual(r.family, loaded.family) + + +for i in range(0, pickle.HIGHEST_PROTOCOL): + def make_test(j): + return lambda self: self._test(j) + setattr(TestPickle, 'test' + str(i), make_test(i)) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__ares_timeout.py b/src/gevent/tests/test__ares_timeout.py new file mode 100644 index 0000000..e2c362d --- /dev/null +++ b/src/gevent/tests/test__ares_timeout.py @@ -0,0 +1,52 @@ +from __future__ import print_function + +import errno +import unittest + +import gevent +try: + from gevent.resolver.ares import Resolver +except ImportError as ex: + Resolver = None +from gevent import socket + +import gevent.testing as greentest + +@unittest.skipIf( + Resolver is None, + "Needs ares resolver" +) +class TestTimeout(greentest.TestCase): + + __timeout__ = 30 + + address = ('', 7153) + + def test(self): + listener = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + try: + listener.bind(self.address) + except socket.error as ex: + if ex.errno in (errno.EPERM, errno.EADDRNOTAVAIL) or 'permission denied' in str(ex).lower(): + raise unittest.SkipTest( + 'This test binds on port a port that was already in use or not allowed.\n' + ) + raise + + + def reader(): + while True: + listener.recvfrom(10000) + + gevent.spawn(reader) + + r = Resolver(servers=['127.0.0.1'], timeout=0.001, tries=1, + udp_port=self.address[-1]) + + with self.assertRaisesRegex(socket.gaierror, "ARES_ETIMEOUT"): + r.gethostbyname('www.google.com') + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__backdoor.py b/src/gevent/tests/test__backdoor.py new file mode 100644 index 0000000..2f60ad8 --- /dev/null +++ b/src/gevent/tests/test__backdoor.py @@ -0,0 +1,135 @@ +from __future__ import print_function +import gevent.testing as greentest +import gevent +from gevent import socket +from gevent import backdoor + +def read_until(conn, postfix): + read = b'' + assert isinstance(postfix, bytes) + + while not read.endswith(postfix): + result = conn.recv(1) + if not result: + raise AssertionError('Connection ended before %r. Data read:\n%r' % (postfix, read)) + read += result + + return read if isinstance(read, str) else read.decode('utf-8') + +def readline(conn): + with conn.makefile() as f: + return f.readline() + +class Test(greentest.TestCase): + + __timeout__ = 10 + + _server = None + + def tearDown(self): + if self._server is not None: + self._server.stop() + self.close_on_teardown.remove(self._server.stop) + self._server = None + gevent.sleep() # let spawned greenlets die + super(Test, self).tearDown() + + def _make_server(self, *args, **kwargs): + assert self._server is None + self._server = backdoor.BackdoorServer(('127.0.0.1', 0), *args, **kwargs) + self._close_on_teardown(self._server.stop) + self._server.start() + + def _create_connection(self): + conn = socket.socket() + self._close_on_teardown(conn) + conn.connect(('127.0.0.1', self._server.server_port)) + return conn + + def _close(self, conn, cmd=b'quit()\r\n)'): + conn.sendall(cmd) + line = readline(conn) + self.assertEqual(line, '') + conn.close() + self.close_on_teardown.remove(conn) + + @greentest.skipOnLibuvOnTravisOnCPython27( + "segfaults; " + "See https://github.com/gevent/gevent/pull/1156") + def test_multi(self): + self._make_server() + + def connect(): + conn = self._create_connection() + read_until(conn, b'>>> ') + conn.sendall(b'2+2\r\n') + line = readline(conn) + self.assertEqual(line.strip(), '4', repr(line)) + self._close(conn) + + jobs = [gevent.spawn(connect) for _ in range(10)] + done = gevent.joinall(jobs, raise_error=True) + + self.assertEqual(len(done), len(jobs), done) + + @greentest.skipOnAppVeyor("Times out") + def test_quit(self): + self._make_server() + conn = self._create_connection() + read_until(conn, b'>>> ') + self._close(conn) + + @greentest.skipOnAppVeyor("Times out") + def test_sys_exit(self): + self._make_server() + conn = self._create_connection() + read_until(conn, b'>>> ') + self._close(conn, b'import sys; sys.exit(0)\r\n') + + @greentest.skipOnAppVeyor("Times out") + def test_banner(self): + banner = "Welcome stranger!" # native string + self._make_server(banner=banner) + conn = self._create_connection() + response = read_until(conn, b'>>> ') + self.assertEqual(response[:len(banner)], banner, response) + + self._close(conn) + + @greentest.skipOnAppVeyor("Times out") + def test_builtins(self): + self._make_server() + conn = self._create_connection() + read_until(conn, b'>>> ') + conn.sendall(b'locals()["__builtins__"]\r\n') + response = read_until(conn, b'>>> ') + self.assertTrue(len(response) < 300, msg="locals() unusable: %s..." % response) + + self._close(conn) + + def test_switch_exc(self): + from gevent.queue import Queue, Empty + + def bad(): + q = Queue() + print('switching out, then throwing in') + try: + q.get(block=True, timeout=0.1) + except Empty: + print("Got Empty") + print('switching out') + gevent.sleep(0.1) + print('switched in') + + self._make_server(locals={'bad': bad}) + conn = self._create_connection() + read_until(conn, b'>>> ') + conn.sendall(b'bad()\r\n') + response = read_until(conn, b'>>> ') + response = response.replace('\r\n', '\n') + self.assertEqual('switching out, then throwing in\nGot Empty\nswitching out\nswitched in\n>>> ', response) + + self._close(conn) + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__close_backend_fd.py b/src/gevent/tests/test__close_backend_fd.py new file mode 100644 index 0000000..4ac760e --- /dev/null +++ b/src/gevent/tests/test__close_backend_fd.py @@ -0,0 +1,63 @@ +from __future__ import print_function +import os +import unittest + +import gevent +from gevent import core + + +@unittest.skipUnless( + getattr(core, 'LIBEV_EMBED', False), + "Needs embedded libev. " + "hub.loop.fileno is only defined when " + "we embed libev for some reason. " + "Choosing specific backends is also only supported by libev " + "(not libuv), and besides, libuv has a nasty tendency to " + "abort() the process if its FD gets closed. " +) +class Test(unittest.TestCase): + # NOTE that we extend unittest.TestCase, not greentest.TestCase + # Extending the later causes the wrong hub to get used. + + assertRaisesRegex = getattr(unittest.TestCase, 'assertRaisesRegex', + getattr(unittest.TestCase, 'assertRaisesRegexp')) + + def _check_backend(self, backend): + hub = gevent.get_hub(backend, default=False) + try: + self.assertEqual(hub.loop.backend, backend) + + gevent.sleep(0.001) + fileno = hub.loop.fileno() + if fileno is None: + raise unittest.SkipTest("backend %s lacks fileno" % (backend,)) + + os.close(fileno) + with self.assertRaisesRegex(SystemError, "(libev)"): + gevent.sleep(0.001) + + hub.destroy() + self.assertIn('destroyed', repr(hub)) + finally: + if hub.loop is not None: + hub.destroy() + + def _make_test(count, backend): # pylint:disable=no-self-argument + def test(self): + self._check_backend(backend) + test.__name__ = 'test_' + backend + '_' + str(count) + return test.__name__, test + + count = backend = None + for count in range(2): + for backend in core.supported_backends(): + name, func = _make_test(count, backend) + locals()[name] = func + name = func = None + + del count + del backend + del _make_test + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__compat.py b/src/gevent/tests/test__compat.py new file mode 100644 index 0000000..7679597 --- /dev/null +++ b/src/gevent/tests/test__compat.py @@ -0,0 +1,56 @@ +from __future__ import absolute_import, print_function, division + +import os +import unittest + +class TestFSPath(unittest.TestCase): + + def setUp(self): + self.__path = None + + def __fspath__(self): + if self.__path is not None: + return self.__path + raise AttributeError("Accessing path data") + + def _callFUT(self, arg): + from gevent._compat import _fspath + return _fspath(arg) + + def test_text(self): + s = u'path' + self.assertIs(s, self._callFUT(s)) + + def test_bytes(self): + s = b'path' + self.assertIs(s, self._callFUT(s)) + + def test_None(self): + with self.assertRaises(TypeError): + self._callFUT(None) + + def test_working_path(self): + self.__path = u'text' + self.assertIs(self.__path, self._callFUT(self)) + + self.__path = b'bytes' + self.assertIs(self.__path, self._callFUT(self)) + + def test_failing_path_AttributeError(self): + self.assertIsNone(self.__path) + with self.assertRaises(AttributeError): + self._callFUT(self) + + def test_fspath_non_str(self): + self.__path = object() + with self.assertRaises(TypeError): + self._callFUT(self) + +@unittest.skipUnless(hasattr(os, 'fspath'), "Tests native os.fspath") +class TestNativeFSPath(TestFSPath): + + def _callFUT(self, arg): + return os.fspath(arg) + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__core.py b/src/gevent/tests/test__core.py new file mode 100644 index 0000000..7e107ef --- /dev/null +++ b/src/gevent/tests/test__core.py @@ -0,0 +1,170 @@ + +from __future__ import absolute_import, print_function, division +import sys +import unittest +import gevent.testing as greentest + +from gevent import core + + + +class TestCore(unittest.TestCase): + + def test_get_version(self): + version = core.get_version() # pylint: disable=no-member + self.assertIsInstance(version, str) + self.assertTrue(version) + header_version = core.get_header_version() # pylint: disable=no-member + self.assertIsInstance(header_version, str) + self.assertTrue(header_version) + self.assertEqual(version, header_version) + + +class TestWatchers(unittest.TestCase): + + def _makeOne(self): + return core.loop() # pylint:disable=no-member + + def destroyOne(self, loop): + loop.destroy() + + def setUp(self): + self.loop = self._makeOne() + + def tearDown(self): + self.destroyOne(self.loop) + del self.loop + + def test_io(self): + if sys.platform == 'win32': + # libev raises IOError, libuv raises ValueError + Error = (IOError, ValueError) + win32 = True + else: + Error = ValueError + win32 = False + + with self.assertRaises(Error): + self.loop.io(-1, 1) + + if hasattr(core, 'TIMER'): + # libev + with self.assertRaises(ValueError): + self.loop.io(1, core.TIMER) # pylint:disable=no-member + + # Test we can set events and io before it's started + if not win32: + # We can't do this with arbitrary FDs on windows; + # see libev_vfd.h + io = self.loop.io(1, core.READ) # pylint:disable=no-member + io.fd = 2 + self.assertEqual(io.fd, 2) + io.events = core.WRITE # pylint:disable=no-member + if not hasattr(core, 'libuv'): + # libev + # pylint:disable=no-member + self.assertEqual(core._events_to_str(io.events), 'WRITE|_IOFDSET') + else: + + self.assertEqual(core._events_to_str(io.events), # pylint:disable=no-member + 'WRITE') + io.start(lambda: None) + io.close() + + @greentest.skipOnLibev("libuv-specific") + @greentest.skipOnWindows("Destroying the loop somehow fails") + def test_io_multiplex_events(self): + # pylint:disable=no-member + import socket + sock = socket.socket() + fd = sock.fileno() + + read = self.loop.io(fd, core.READ) + write = self.loop.io(fd, core.WRITE) + + try: + real_watcher = read._watcher_ref + + read.start(lambda: None) + self.assertEqual(real_watcher.events, core.READ) + + write.start(lambda: None) + self.assertEqual(real_watcher.events, core.READ | core.WRITE) + + write.stop() + self.assertEqual(real_watcher.events, core.READ) + + write.start(lambda: None) + self.assertEqual(real_watcher.events, core.READ | core.WRITE) + + read.stop() + self.assertEqual(real_watcher.events, core.WRITE) + + write.stop() + self.assertEqual(real_watcher.events, 0) + finally: + read.close() + write.close() + sock.close() + + def test_timer_constructor(self): + with self.assertRaises(ValueError): + self.loop.timer(1, -1) + + def test_signal_constructor(self): + with self.assertRaises(ValueError): + self.loop.signal(1000) + +class TestWatchersDefault(TestWatchers): + + def _makeOne(self): + return core.loop(default=True) # pylint:disable=no-member + + def destroyOne(self, loop): + return + +# XXX: The crash may be fixed? The hang showed up after the crash was +# reproduced and fixed on linux and OS X. +@greentest.skipOnLibuvOnWin( + "This crashes with PyPy 5.10.0, only on Windows. " + "See https://ci.appveyor.com/project/denik/gevent/build/1.0.1380/job/lrlvid6mkjtyrhn5#L1103 " + "It has also timed out, but only on Appveyor CPython 3.6; local CPython 3.6 does not. " + "See https://ci.appveyor.com/project/denik/gevent/build/1.0.1414/job/yn7yi8b53vtqs8lw#L1523") +class TestWatchersDefaultDestroyed(TestWatchers): + + def _makeOne(self): + # pylint: disable=no-member + l = core.loop(default=True) + l.destroy() + del l + return core.loop(default=True) + +@greentest.skipOnLibuv("Tests for libev-only functions") +class TestLibev(unittest.TestCase): + + def test_flags_conversion(self): + # pylint: disable=no-member + if sys.platform != 'win32': + self.assertEqual(core.loop(2, default=False).backend_int, 2) + self.assertEqual(core.loop('select', default=False).backend, 'select') + self.assertEqual(core._flags_to_int(None), 0) + self.assertEqual(core._flags_to_int(['kqueue', 'SELECT']), core.BACKEND_KQUEUE | core.BACKEND_SELECT) + self.assertEqual(core._flags_to_list(core.BACKEND_PORT | core.BACKEND_POLL), ['port', 'poll']) + self.assertRaises(ValueError, core.loop, ['port', 'blabla']) + self.assertRaises(TypeError, core.loop, object()) + + +class TestEvents(unittest.TestCase): + + def test_events_conversion(self): + self.assertEqual(core._events_to_str(core.READ | core.WRITE), # pylint: disable=no-member + 'READ|WRITE') + + def test_EVENTS(self): + self.assertEqual(str(core.EVENTS), # pylint: disable=no-member + 'gevent.core.EVENTS') + self.assertEqual(repr(core.EVENTS), # pylint: disable=no-member + 'gevent.core.EVENTS') + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__core_async.py b/src/gevent/tests/test__core_async.py new file mode 100644 index 0000000..e1e76cd --- /dev/null +++ b/src/gevent/tests/test__core_async.py @@ -0,0 +1,24 @@ +from __future__ import print_function +import gevent +import gevent.core +import time +try: + import thread +except ImportError: + import _thread as thread + + +hub = gevent.get_hub() +watcher = hub.loop.async_() + +# BWC for <3.7: This should still be an attribute +assert hasattr(hub.loop, 'async') + +gevent.spawn_later(0.1, thread.start_new_thread, watcher.send, ()) + +start = time.time() + +with gevent.Timeout(1.0): # Large timeout for appveyor + hub.wait(watcher) + +print('Watcher %r reacted after %.6f seconds' % (watcher, time.time() - start - 0.1)) diff --git a/src/gevent/tests/test__core_callback.py b/src/gevent/tests/test__core_callback.py new file mode 100644 index 0000000..f2acab5 --- /dev/null +++ b/src/gevent/tests/test__core_callback.py @@ -0,0 +1,31 @@ +import gevent +from gevent.hub import get_hub + +called = [] + + +def f(): + called.append(1) + + +def main(): + loop = get_hub().loop + x = loop.run_callback(f) + + assert x, x + gevent.sleep(0) + assert called == [1], called + assert not x, (x, bool(x)) + + x = loop.run_callback(f) + assert x, x + x.stop() + assert not x, x + gevent.sleep(0) + assert called == [1], called + assert not x, x + + +if __name__ == '__main__': + called[:] = [] + main() diff --git a/src/gevent/tests/test__core_fork.py b/src/gevent/tests/test__core_fork.py new file mode 100644 index 0000000..fee42f5 --- /dev/null +++ b/src/gevent/tests/test__core_fork.py @@ -0,0 +1,58 @@ +from __future__ import print_function +import gevent.monkey +gevent.monkey.patch_all() +import gevent +import os + +import multiprocessing + +hub = gevent.get_hub() +pid = os.getpid() +newpid = None + + +def on_fork(): + global newpid + newpid = os.getpid() + +fork_watcher = hub.loop.fork(ref=False) +fork_watcher.start(on_fork) + + +def run(q): + # libev only calls fork callbacks at the beginning of + # the loop; we use callbacks extensively so it takes *two* + # calls to sleep (with a timer) to actually get wrapped + # around to the beginning of the loop. + gevent.sleep(0.01) + gevent.sleep(0.01) + q.put(newpid) + + +def test(): + # Use a thread to make us multi-threaded + hub.threadpool.apply(lambda: None) + # If the Queue is global, q.get() hangs on Windows; must pass as + # an argument. + q = multiprocessing.Queue() + p = multiprocessing.Process(target=run, args=(q,)) + p.start() + p.join() + p_val = q.get() + + assert p_val is not None, "The fork watcher didn't run" + assert p_val != pid + +if __name__ == '__main__': + # Must call for Windows to fork properly; the fork can't be in the top-level + multiprocessing.freeze_support() + # fork watchers weren't firing in multi-threading processes. + # This test is designed to prove that they are. + # However, it fails on Windows: The fork watcher never runs! + # This makes perfect sense: on Windows, our patches to os.fork() + # that call gevent.hub.reinit() don't get used; os.fork doesn't + # exist and multiprocessing.Process uses the windows-specific _subprocess.CreateProcess() + # to create a whole new process that has no relation to the current process; + # that process then calls multiprocessing.forking.main() to do its work. + # Since no state is shared, a fork watcher cannot exist in that process. + test() diff --git a/src/gevent/tests/test__core_loop_run.py b/src/gevent/tests/test__core_loop_run.py new file mode 100644 index 0000000..ad35e01 --- /dev/null +++ b/src/gevent/tests/test__core_loop_run.py @@ -0,0 +1,25 @@ +from __future__ import print_function +import sys +# Using a direct import of signal (deprecated). +# We are also called from test__core_loop_run_sig_mod.py, +# which has already done 'import gevent.signal' to be sure we work +# when the module has been imported. +from gevent import core, signal +loop = core.loop(default=False) + + +signal = signal(2, sys.stderr.write, 'INTERRUPT!') + +print('must exit immediately...') +loop.run() # must exit immediately +print('...and once more...') +loop.run() # repeating does not fail +print('..done') + +print('must exit after 0.5 seconds.') +timer = loop.timer(0.5) +timer.start(lambda: None) +loop.run() +timer.close() +loop.destroy() +del loop diff --git a/src/gevent/tests/test__core_loop_run_sig_mod.py b/src/gevent/tests/test__core_loop_run_sig_mod.py new file mode 100644 index 0000000..396cbe5 --- /dev/null +++ b/src/gevent/tests/test__core_loop_run_sig_mod.py @@ -0,0 +1,16 @@ +import sys +import gevent.signal +assert gevent.signal # Get gevent.signal as a module, make sure our backwards compatibility kicks in + +import test__core_loop_run # this runs main tests, fails if signal() is not callable. +assert test__core_loop_run # pyflakes + +from gevent.hub import signal as hub_signal +from gevent import signal as top_signal # pylint:disable=reimported + +assert gevent.signal is hub_signal +assert gevent.signal is top_signal +assert hasattr(gevent.signal, 'signal') +s = top_signal(2, sys.stderr.write, 'INTERRUPT') +assert isinstance(s, top_signal) +assert isinstance(s, hub_signal) diff --git a/src/gevent/tests/test__core_stat.py b/src/gevent/tests/test__core_stat.py new file mode 100644 index 0000000..d022d90 --- /dev/null +++ b/src/gevent/tests/test__core_stat.py @@ -0,0 +1,118 @@ +from __future__ import print_function + +import os +import tempfile +import time + +import gevent +import gevent.core + +import gevent.testing as greentest +import gevent.testing.flaky + +#pylint: disable=protected-access + + +DELAY = 0.5 + +WIN = greentest.WIN + +LIBUV = greentest.LIBUV + +class TestCoreStat(greentest.TestCase): + + __timeout__ = greentest.LARGE_TIMEOUT + + def setUp(self): + super(TestCoreStat, self).setUp() + fd, path = tempfile.mkstemp(suffix='.gevent_test_core_stat') + os.close(fd) + self.temp_path = path + self.hub = gevent.get_hub() + # If we don't specify an interval, we default to zero. + # libev interprets that as meaning to use its default interval, + # which is about 5 seconds. If we go below it's minimum check + # threshold, it bumps it up to the minimum. + self.watcher = self.hub.loop.stat(self.temp_path, interval=-1) + + def tearDown(self): + self.watcher.close() + if os.path.exists(self.temp_path): + os.unlink(self.temp_path) + super(TestCoreStat, self).tearDown() + + def _write(self): + with open(self.temp_path, 'wb', buffering=0) as f: + f.write(b'x') + + def _check_attr(self, name, none): + # Deals with the complex behaviour of the 'attr' and 'prev' + # attributes on Windows. This codifies it, rather than simply letting + # the test fail, so we know exactly when and what changes it. + try: + x = getattr(self.watcher, name) + except ImportError: + if WIN: + # the 'posix' module is not available + pass + else: + raise + else: + if WIN and not LIBUV: + # The ImportError is only raised for the first time; + # after that, the attribute starts returning None + self.assertIsNone(x, "Only None is supported on Windows") + if none: + self.assertIsNone(x, name) + else: + self.assertIsNotNone(x, name) + + def _wait_on_greenlet(self, func, *greenlet_args): + start = time.time() + + self.hub.loop.update_now() + greenlet = gevent.spawn_later(DELAY, func, *greenlet_args) + with gevent.Timeout(5 + DELAY + 0.5): + self.hub.wait(self.watcher) + now = time.time() + + self.assertGreaterEqual(now, start, "Time must move forward") + + wait_duration = now - start + reaction = wait_duration - DELAY + + if reaction <= 0.0: + # Sigh. This is especially true on PyPy on Windows + raise gevent.testing.flaky.FlakyTestRaceCondition( + "Bad timer resolution (on Windows?), test is useless. Start %s, now %s" % (start, now)) + + self.assertGreaterEqual( + reaction, 0.0, + 'Watcher %s reacted too early: %.3fs' % (self.watcher, reaction)) + + greenlet.join() + + def test_watcher_basics(self): + watcher = self.watcher + filename = self.temp_path + self.assertEqual(watcher.path, filename) + filenames = filename if isinstance(filename, bytes) else filename.encode('ascii') + self.assertEqual(watcher._paths, filenames) + self.assertEqual(watcher.interval, -1) + + def test_write(self): + self._wait_on_greenlet(self._write) + + self._check_attr('attr', False) + self._check_attr('prev', False) + # The watcher interval changed after it started; -1 is illegal + self.assertNotEqual(self.watcher.interval, -1) + + def test_unlink(self): + self._wait_on_greenlet(os.unlink, self.temp_path) + + self._check_attr('attr', True) + self._check_attr('prev', False) + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__core_timer.py b/src/gevent/tests/test__core_timer.py new file mode 100644 index 0000000..97caa75 --- /dev/null +++ b/src/gevent/tests/test__core_timer.py @@ -0,0 +1,157 @@ +from __future__ import print_function +from gevent import config + +import gevent.testing as greentest +from gevent.testing import TestCase +from gevent.testing import LARGE_TIMEOUT +from gevent.testing.sysinfo import CFFI_BACKEND +from gevent.testing.flaky import reraises_flaky_timeout + + +class Test(TestCase): + __timeout__ = LARGE_TIMEOUT + + repeat = 0 + timer_duration = 0.001 + + def setUp(self): + super(Test, self).setUp() + self.called = [] + self.loop = config.loop(default=False) + self.timer = self.loop.timer(self.timer_duration, repeat=self.repeat) + assert not self.loop.default + + def cleanup(self): + # cleanup instead of tearDown to cooperate well with + # leakcheck.py + self.timer.close() + # cycle the loop so libuv close callbacks fire + self.loop.run() + self.loop.destroy() + self.loop = None + self.timer = None + + def f(self, x=None): + self.called.append(1) + if x is not None: + x.stop() + + def assertTimerInKeepalive(self): + if CFFI_BACKEND: + self.assertIn(self.timer, self.loop._keepaliveset) + + def assertTimerNotInKeepalive(self): + if CFFI_BACKEND: + self.assertNotIn(self.timer, self.loop._keepaliveset) + + def test_main(self): + loop = self.loop + x = self.timer + x.start(self.f) + self.assertTimerInKeepalive() + self.assertTrue(x.active, x) + + with self.assertRaises((AttributeError, ValueError)): + x.priority = 1 + + loop.run() + self.assertEqual(x.pending, 0) + self.assertEqual(self.called, [1]) + self.assertIsNone(x.callback) + self.assertIsNone(x.args) + + if x.priority is not None: + self.assertEqual(x.priority, 0) + x.priority = 1 + self.assertEqual(x.priority, 1) + + x.stop() + self.assertTimerNotInKeepalive() + +class TestAgain(Test): + repeat = 1 + + def test_main(self): + # Again works for a new timer + x = self.timer + x.again(self.f, x) + self.assertTimerInKeepalive() + + self.assertEqual(x.args, (x,)) + + # XXX: On libev, this takes 1 second. On libuv, + # it takes the expected time. + self.loop.run() + + self.assertEqual(self.called, [1]) + + x.stop() + self.assertTimerNotInKeepalive() + + +class TestTimerResolution(Test): + + # On CI, with *all* backends, sometimes we get timer values of + # 0.02 or higher. + @reraises_flaky_timeout(AssertionError) + def test_resolution(self): # pylint:disable=too-many-locals + # Make sure that having an active IO watcher + # doesn't badly throw off our timer resolution. + # (This was a specific problem with libuv) + + # https://github.com/gevent/gevent/pull/1194 + from gevent._compat import perf_counter + + import socket + s = socket.socket() + self._close_on_teardown(s) + fd = s.fileno() + + ran_at_least_once = False + fired_at = [] + + def timer_counter(): + fired_at.append(perf_counter()) + + loop = self.loop + + timer_multiplier = 11 + max_time = self.timer_duration * timer_multiplier + assert max_time < 0.3 + + for _ in range(150): + # in libuv, our signal timer fires every 300ms; depending on + # when this runs, we could artificially get a better + # resolution than we expect. Run it multiple times to be more sure. + io = loop.io(fd, 1) + io.start(lambda events=None: None) + + + now = perf_counter() + del fired_at[:] + timer = self.timer + timer.start(timer_counter) + + loop.run(once=True) + + io.stop() + io.close() + + timer.stop() + + if fired_at: + ran_at_least_once = True + self.assertEqual(1, len(fired_at)) + self.assertTimeWithinRange(fired_at[0] - now, + 0, + max_time) + + + if not greentest.RUNNING_ON_CI: + # Hmm, this always fires locally on mocOS but + # not an Travis? + self.assertTrue(ran_at_least_once) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__core_watcher.py b/src/gevent/tests/test__core_watcher.py new file mode 100644 index 0000000..bd52805 --- /dev/null +++ b/src/gevent/tests/test__core_watcher.py @@ -0,0 +1,124 @@ +from __future__ import absolute_import, print_function + +import gevent.testing as greentest +from gevent import config +from gevent.testing.sysinfo import CFFI_BACKEND + +from gevent.core import READ # pylint:disable=no-name-in-module +from gevent.core import WRITE # pylint:disable=no-name-in-module + + +class Test(greentest.TestCase): + + __timeout__ = None + + def setUp(self): + super(Test, self).setUp() + self.loop = config.loop(default=False) + self.timer = self.loop.timer(0.01) + + def tearDown(self): + if self.timer is not None: + self.timer.close() + if self.loop is not None: + self.loop.destroy() + self.loop = self.timer = None + super(Test, self).tearDown() + + def test_non_callable_to_start(self): + # test that cannot pass non-callable thing to start() + self.assertRaises(TypeError, self.timer.start, None) + self.assertRaises(TypeError, self.timer.start, 5) + + def test_non_callable_after_start(self): + # test that cannot set 'callback' to non-callable thing later either + lst = [] + timer = self.timer + timer.start(lst.append) + + + with self.assertRaises(TypeError): + timer.callback = False + + with self.assertRaises(TypeError): + timer.callback = 5 + + def test_args_can_be_changed_after_start(self): + lst = [] + timer = self.timer + self.timer.start(lst.append) + self.assertEqual(timer.args, ()) + timer.args = (1, 2, 3) + self.assertEqual(timer.args, (1, 2, 3)) + + # Only tuple can be args + with self.assertRaises(TypeError): + timer.args = 5 + with self.assertRaises(TypeError): + timer.args = [4, 5] + + self.assertEqual(timer.args, (1, 2, 3)) + + # None also works, means empty tuple + # XXX why? + timer.args = None + self.assertEqual(timer.args, None) + + + def test_run(self): + loop = self.loop + lst = [] + + self.timer.start(lambda *args: lst.append(args)) + + loop.run() + loop.update_now() + + self.assertEqual(lst, [()]) + + # Even if we lose all references to it, the ref in the callback + # keeps it alive + self.timer.start(reset, self.timer, lst) + self.timer = None + loop.run() + self.assertEqual(lst, [(), 25]) + + def test_invalid_fd(self): + loop = self.loop + + # Negative case caught everywhere. ValueError + # on POSIX, OSError on Windows Py3, IOError on Windows Py2 + with self.assertRaises((ValueError, OSError, IOError)): + loop.io(-1, READ) + + + @greentest.skipOnWindows("Stdout can't be watched on Win32") + def test_reuse_io(self): + loop = self.loop + + # Watchers aren't reused once all outstanding + # refs go away BUT THEY MUST BE CLOSED + tty_watcher = loop.io(1, WRITE) + watcher_handle = tty_watcher._watcher if CFFI_BACKEND else tty_watcher + tty_watcher.close() + del tty_watcher + # XXX: Note there is a cycle in the CFFI code + # from watcher_handle._handle -> watcher_handle. + # So it doesn't go away until a GC runs. + import gc + gc.collect() + + tty_watcher = loop.io(1, WRITE) + self.assertIsNot(tty_watcher._watcher if CFFI_BACKEND else tty_watcher, watcher_handle) + tty_watcher.close() + + +def reset(watcher, lst): + watcher.args = None + watcher.callback = lambda: None + lst.append(25) + watcher.close() + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__destroy.py b/src/gevent/tests/test__destroy.py new file mode 100644 index 0000000..784723a --- /dev/null +++ b/src/gevent/tests/test__destroy.py @@ -0,0 +1,51 @@ +from __future__ import absolute_import, print_function + +import gevent +import unittest + +class TestDestroyHub(unittest.TestCase): + + def test_destroy_hub(self): + # Loop of initial Hub is default loop. + hub = gevent.get_hub() + self.assertTrue(hub.loop.default) + + # Save `gevent.core.loop` object for later comparison. + initloop = hub.loop + + # Increase test complexity via threadpool creation. + # Implicitly creates fork watcher connected to the current event loop. + tp = hub.threadpool + self.assertIsNotNone(tp) + + # Destroy hub. Does not destroy libev default loop if not explicitly told to. + hub.destroy() + + # Create new hub. Must re-use existing libev default loop. + hub = gevent.get_hub() + self.assertTrue(hub.loop.default) + + # Ensure that loop object is identical to the initial one. + self.assertIs(hub.loop, initloop) + + # Destroy hub including default loop. + hub.destroy(destroy_loop=True) + + # Create new hub and explicitly request creation of a new default loop. + hub = gevent.get_hub(default=True) + self.assertTrue(hub.loop.default) + + # `gevent.core.loop` objects as well as libev loop pointers must differ. + self.assertIsNot(hub.loop, initloop) + self.assertIsNot(hub.loop.ptr, initloop.ptr) + self.assertNotEqual(hub.loop.ptr, initloop.ptr) + + # Destroy hub including default loop. The default loop regenerates. + hub.destroy(destroy_loop=True) + hub = gevent.get_hub() + self.assertTrue(hub.loop.default) + + hub.destroy() + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__destroy_default_loop.py b/src/gevent/tests/test__destroy_default_loop.py new file mode 100644 index 0000000..c655b8d --- /dev/null +++ b/src/gevent/tests/test__destroy_default_loop.py @@ -0,0 +1,56 @@ +from __future__ import print_function +import gevent + +import unittest + +class TestDestroyDefaultLoop(unittest.TestCase): + + def test_destroy_gc(self): + # Issue 1098: destroying the default loop + # while using the C extension could crash + # the interpreter when it exits + + # Create the hub greenlet. This creates one loop + # object pointing to the default loop. + gevent.get_hub() + + # Get a new loop object, but using the default + # C loop + loop = gevent.config.loop(default=True) + self.assertTrue(loop.default) + # Destroy it + loop.destroy() + # It no longer claims to be the default + self.assertFalse(loop.default) + + # Delete it + del loop + # Delete the hub. This prompts garbage + # collection of it and its loop object. + # (making this test more repeatable; the exit + # crash only happened when that greenlet object + # was collected at exit time, which was most common + # in CPython 3.5) + from gevent._hub_local import set_hub + set_hub(None) + + + + def test_destroy_two(self): + # Get two new loop object, but using the default + # C loop + loop1 = gevent.config.loop(default=True) + loop2 = gevent.config.loop(default=True) + self.assertTrue(loop1.default) + self.assertTrue(loop2.default) + # Destroy the first + loop1.destroy() + # It no longer claims to be the default + self.assertFalse(loop1.default) + + # Destroy the second. This doesn't crash. + loop2.destroy() + + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__doctests.py b/src/gevent/tests/test__doctests.py new file mode 100644 index 0000000..731a9ee --- /dev/null +++ b/src/gevent/tests/test__doctests.py @@ -0,0 +1,133 @@ +from __future__ import print_function + +import doctest +import functools +import os +import re +import sys +import unittest + + + +# Ignore tracebacks: ZeroDivisionError + + +def myfunction(*_args, **_kwargs): + pass + + +class RENormalizingOutputChecker(doctest.OutputChecker): + """ + Pattern-normalizing output checker. Inspired by one used in zope.testing. + """ + + def __init__(self, patterns): + self.transformers = [functools.partial(re.sub, replacement) for re, replacement in patterns] + + def check_output(self, want, got, optionflags): + if got == want: + return True + + for transformer in self.transformers: + want = transformer(want) + got = transformer(got) + + return doctest.OutputChecker.check_output(self, want, got, optionflags) + +FORBIDDEN_MODULES = set() + + +class Modules(object): + + def __init__(self, allowed_modules): + from gevent.testing import walk_modules + self.allowed_modules = allowed_modules + self.modules = set() + + for path, module in walk_modules(recursive=True): + self.add_module(module, path) + + + def add_module(self, name, path): + if self.allowed_modules and name not in self.allowed_modules: + return + if name in FORBIDDEN_MODULES: + return + self.modules.add((name, path)) + + def __bool__(self): + return bool(self.modules) + + __nonzero__ = __bool__ + + def __iter__(self): + return iter(self.modules) + + +def main(): # pylint:disable=too-many-locals + cwd = os.getcwd() + # Use pure_python to get the correct module source and docstrings + os.environ['PURE_PYTHON'] = '1' + + import gevent + from gevent import socket + + + from gevent.testing import util + from gevent.testing import sysinfo + + if sysinfo.WIN: + FORBIDDEN_MODULES.update({ + # Uses commands only found on posix + 'gevent.subprocess', + }) + + try: + allowed_modules = sys.argv[1:] + sys.path.append('.') + + globs = { + 'myfunction': myfunction, + 'gevent': gevent, + 'socket': socket, + } + + modules = Modules(allowed_modules) + + if not modules: + sys.exit('No modules found matching %s' % ' '.join(allowed_modules)) + + suite = unittest.TestSuite() + checker = RENormalizingOutputChecker(( + # Normalize subprocess.py: BSD ls is in the example, gnu ls outputs + # 'cannot access' + (re.compile( + "ls: cannot access 'non_existent_file': No such file or directory"), + "ls: non_existent_file: No such file or directory"), + # Python 3 bytes add a "b". + (re.compile(r'b(".*?")'), r"\1"), + (re.compile(r"b('.*?')"), r"\1"), + )) + + tests_count = 0 + modules_count = 0 + for m, path in sorted(modules): + with open(path, 'rb') as f: + contents = f.read() + if re.search(br'^\s*>>> ', contents, re.M): + s = doctest.DocTestSuite(m, extraglobs=globs, checker=checker) + test_count = len(s._tests) + util.log('%s (from %s): %s tests', m, path, test_count) + suite.addTest(s) + modules_count += 1 + tests_count += test_count + + util.log('Total: %s tests in %s modules', tests_count, modules_count) + # TODO: Pass this off to unittest.main() + runner = unittest.TextTestRunner(verbosity=2) + runner.run(suite) + finally: + os.chdir(cwd) + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__environ.py b/src/gevent/tests/test__environ.py new file mode 100644 index 0000000..fae25ab --- /dev/null +++ b/src/gevent/tests/test__environ.py @@ -0,0 +1,17 @@ +import os +import sys +import gevent +import gevent.core +import subprocess + +if sys.argv[1:] == []: + os.environ['GEVENT_BACKEND'] = 'select' + popen = subprocess.Popen([sys.executable, __file__, '1']) + assert popen.wait() == 0, popen.poll() +else: # pragma: no cover + hub = gevent.get_hub() + if 'select' in gevent.core.supported_backends(): + assert hub.loop.backend == 'select', hub.loop.backend + else: + # libuv isn't configurable + assert hub.loop.backend == 'default', hub.loop.backend diff --git a/src/gevent/tests/test__event.py b/src/gevent/tests/test__event.py new file mode 100644 index 0000000..4c95fee --- /dev/null +++ b/src/gevent/tests/test__event.py @@ -0,0 +1,254 @@ +from __future__ import absolute_import, print_function, division + +import weakref + +import gevent +from gevent.event import Event, AsyncResult + +import gevent.testing as greentest + +from gevent.testing.six import xrange +from gevent.testing.timing import AbstractGenericGetTestCase +from gevent.testing.timing import AbstractGenericWaitTestCase +from gevent.testing.timing import SMALL_TICK +from gevent.testing.timing import SMALL_TICK_MAX_ADJ + +DELAY = SMALL_TICK + SMALL_TICK_MAX_ADJ + + +class TestEventWait(AbstractGenericWaitTestCase): + + def wait(self, timeout): + Event().wait(timeout=timeout) + + def test_cover(self): + str(Event()) + + +class TestWaitEvent(AbstractGenericWaitTestCase): + + def wait(self, timeout): + gevent.wait([Event()], timeout=timeout) + + def test_set_during_wait(self): + # https://github.com/gevent/gevent/issues/771 + # broke in the refactoring. we must not add new links + # while we're running the callback + + event = Event() + + def setter(): + event.set() + + def waiter(): + s = gevent.spawn(setter) + # let the setter set() the event; + # when this method returns we'll be running in the Event._notify_links callback + # (that is, it switched to us) + res = event.wait() + self.assertTrue(res) + self.assertTrue(event.ready()) + s.join() # make sure it's dead + # Clear the event. Now we can't wait for the event without + # another set to happen. + event.clear() + self.assertFalse(event.ready()) + + # Before the bug fix, this would return "immediately" with + # event in the result list, because the _notify_links loop would + # immediately add the waiter and call it + o = gevent.wait((event,), timeout=0.01) + self.assertFalse(event.ready()) + self.assertNotIn(event, o) + + gevent.spawn(waiter).join() + + +class TestAsyncResultWait(AbstractGenericWaitTestCase): + + def wait(self, timeout): + AsyncResult().wait(timeout=timeout) + + +class TestWaitAsyncResult(AbstractGenericWaitTestCase): + + def wait(self, timeout): + gevent.wait([AsyncResult()], timeout=timeout) + + +class TestAsyncResultGet(AbstractGenericGetTestCase): + + def wait(self, timeout): + AsyncResult().get(timeout=timeout) + +class MyException(Exception): + pass + +class TestAsyncResult(greentest.TestCase): + + def test_link(self): + ar = AsyncResult() + self.assertRaises(TypeError, ar.rawlink, None) + ar.unlink(None) # doesn't raise + ar.unlink(None) # doesn't raise + str(ar) # cover + + def test_set_exc(self): + log = [] + e = AsyncResult() + self.assertEqual(e.exc_info, ()) + self.assertEqual(e.exception, None) + + def waiter(): + with self.assertRaises(MyException) as exc: + e.get() + log.append(('caught', exc.exception)) + gevent.spawn(waiter) + obj = MyException() + e.set_exception(obj) + gevent.sleep(0) + self.assertEqual(log, [('caught', obj)]) + + def test_set(self): + event1 = AsyncResult() + timer_exc = MyException('interrupted') + + # Notice that this test is racy: + # After DELAY, we set the event. We also try to immediately + # raise the exception with a timer of 0 --- but that depends + # on cycling the loop. Hence the fairly large value for DELAY. + g = gevent.spawn_later(DELAY, event1.set, 'hello event1') + self._close_on_teardown(g.kill) + with gevent.Timeout.start_new(0, timer_exc): + with self.assertRaises(MyException) as exc: + event1.get() + self.assertIs(timer_exc, exc.exception) + + def test_set_with_timeout(self): + event2 = AsyncResult() + + X = object() + result = gevent.with_timeout(DELAY, event2.get, timeout_value=X) + self.assertIs( + result, X, + 'Nobody sent anything to event2 yet it received %r' % (result, )) + + def test_nonblocking_get(self): + ar = AsyncResult() + self.assertRaises(gevent.Timeout, ar.get, block=False) + self.assertRaises(gevent.Timeout, ar.get_nowait) + + +class TestAsyncResultAsLinkTarget(greentest.TestCase): + error_fatal = False + + def test_set(self): + g = gevent.spawn(lambda: 1) + s1, s2, s3 = AsyncResult(), AsyncResult(), AsyncResult() + g.link(s1) + g.link_value(s2) + g.link_exception(s3) + self.assertEqual(s1.get(), 1) + self.assertEqual(s2.get(), 1) + X = object() + result = gevent.with_timeout(DELAY, s3.get, timeout_value=X) + self.assertIs(result, X) + + def test_set_exception(self): + def func(): + raise greentest.ExpectedException('TestAsyncResultAsLinkTarget.test_set_exception') + g = gevent.spawn(func) + s1, s2, s3 = AsyncResult(), AsyncResult(), AsyncResult() + g.link(s1) + g.link_value(s2) + g.link_exception(s3) + self.assertRaises(greentest.ExpectedException, s1.get) + X = object() + result = gevent.with_timeout(DELAY, s2.get, timeout_value=X) + self.assertIs(result, X) + self.assertRaises(greentest.ExpectedException, s3.get) + + +class TestEvent_SetThenClear(greentest.TestCase): + N = 1 + + def test(self): + e = Event() + waiters = [gevent.spawn(e.wait) for i in range(self.N)] + gevent.sleep(0.001) + e.set() + e.clear() + for greenlet in waiters: + greenlet.join() + + +class TestEvent_SetThenClear100(TestEvent_SetThenClear): + N = 100 + + +class TestEvent_SetThenClear1000(TestEvent_SetThenClear): + N = 1000 + + +class TestWait(greentest.TestCase): + N = 5 + count = None + timeout = 1 + period = timeout / 100.0 + + def _sender(self, events, asyncs): + while events or asyncs: + gevent.sleep(self.period) + if events: + events.pop().set() + gevent.sleep(self.period) + if asyncs: + asyncs.pop().set() + + @greentest.skipOnAppVeyor("Not all results have arrived sometimes due to timer issues") + def test(self): + events = [Event() for _ in xrange(self.N)] + asyncs = [AsyncResult() for _ in xrange(self.N)] + max_len = len(events) + len(asyncs) + sender = gevent.spawn(self._sender, events, asyncs) + results = gevent.wait(events + asyncs, count=self.count, timeout=self.timeout) + if self.timeout is None: + expected_len = max_len + else: + expected_len = min(max_len, self.timeout / self.period) + if self.count is None: + self.assertTrue(sender.ready(), sender) + else: + expected_len = min(self.count, expected_len) + self.assertFalse(sender.ready(), sender) + sender.kill() + self.assertEqual(expected_len, len(results), (expected_len, len(results), results)) + + +class TestWait_notimeout(TestWait): + timeout = None + + +class TestWait_count1(TestWait): + count = 1 + + +class TestWait_count2(TestWait): + count = 2 + +class TestEventBasics(greentest.TestCase): + + def test_weakref(self): + # Event objects should allow weakrefs + e = Event() + r = weakref.ref(e) + self.assertIs(e, r()) + del e + del r + + +del AbstractGenericGetTestCase +del AbstractGenericWaitTestCase + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__events.py b/src/gevent/tests/test__events.py new file mode 100644 index 0000000..f95da3a --- /dev/null +++ b/src/gevent/tests/test__events.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 gevent. See LICENSE. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +import unittest + +from gevent import events +from zope.interface import verify + +class TestImplements(unittest.TestCase): + + def test_event_loop_blocked(self): + verify.verifyClass(events.IEventLoopBlocked, events.EventLoopBlocked) + + def test_mem_threshold(self): + verify.verifyClass(events.IMemoryUsageThresholdExceeded, + events.MemoryUsageThresholdExceeded) + verify.verifyObject(events.IMemoryUsageThresholdExceeded, + events.MemoryUsageThresholdExceeded(0, 0, 0)) + + def test_mem_decreased(self): + verify.verifyClass(events.IMemoryUsageUnderThreshold, + events.MemoryUsageUnderThreshold) + verify.verifyObject(events.IMemoryUsageUnderThreshold, + events.MemoryUsageUnderThreshold(0, 0, 0, 0)) + + +class TestEvents(unittest.TestCase): + + def test_is_zope(self): + from zope import event + self.assertIs(events.subscribers, event.subscribers) + self.assertIs(events.notify, event.notify) + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__example_echoserver.py b/src/gevent/tests/test__example_echoserver.py new file mode 100644 index 0000000..128678a --- /dev/null +++ b/src/gevent/tests/test__example_echoserver.py @@ -0,0 +1,40 @@ +from gevent.socket import create_connection, timeout +import gevent.testing as greentest +import gevent + +from gevent.testing import util +from gevent.testing import params + +class Test(util.TestServer): + server = 'echoserver.py' + + def _run_all_tests(self): + def test_client(message): + if greentest.PY3: + kwargs = {'buffering': 1} + else: + kwargs = {'bufsize': 1} + kwargs['mode'] = 'rb' + conn = create_connection((params.DEFAULT_LOCAL_HOST_ADDR, 16000)) + conn.settimeout(greentest.DEFAULT_XPC_SOCKET_TIMEOUT) + rfile = conn.makefile(**kwargs) + + welcome = rfile.readline() + self.assertIn(b'Welcome', welcome) + + conn.sendall(message) + received = rfile.read(len(message)) + self.assertEqual(received, message) + + self.assertRaises(timeout, conn.recv, 1) + + rfile.close() + conn.close() + + client1 = gevent.spawn(test_client, b'hello\r\n') + client2 = gevent.spawn(test_client, b'world\r\n') + gevent.joinall([client1, client2], raise_error=True) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__example_portforwarder.py b/src/gevent/tests/test__example_portforwarder.py new file mode 100644 index 0000000..cd6292c --- /dev/null +++ b/src/gevent/tests/test__example_portforwarder.py @@ -0,0 +1,66 @@ +from __future__ import print_function, absolute_import +from gevent import monkey; monkey.patch_all(subprocess=True) +import signal +import sys +import socket +from time import sleep + +import gevent +from gevent.server import StreamServer + +import gevent.testing as greentest +from gevent.testing import util + +@greentest.skipOnLibuvOnCIOnPyPy("Timing issues sometimes lead to connection refused") +class Test(util.TestServer): + server = 'portforwarder.py' + args = ['127.0.0.1:10011', '127.0.0.1:10012'] + + if sys.platform.startswith('win'): + from subprocess import CREATE_NEW_PROCESS_GROUP + # Must be in a new process group to use CTRL_C_EVENT, otherwise + # we get killed too + start_kwargs = {'creationflags': CREATE_NEW_PROCESS_GROUP} + + def after(self): + if sys.platform == 'win32': + self.assertIsNotNone(self.popen.poll()) + else: + self.assertEqual(self.popen.poll(), 0) + + def _run_all_tests(self): + log = [] + + def handle(sock, _address): + while True: + data = sock.recv(1024) + print('got %r' % data) + if not data: + break + log.append(data) + + server = StreamServer(self.args[1], handle) + server.start() + try: + conn = socket.create_connection(('127.0.0.1', 10011)) + conn.sendall(b'msg1') + sleep(0.1) + # On Windows, SIGTERM actually abruptly terminates the process; + # it can't be caught. However, CTRL_C_EVENT results in a KeyboardInterrupt + # being raised, so we can shut down properly. + self.popen.send_signal(getattr(signal, 'CTRL_C_EVENT', signal.SIGTERM)) + sleep(0.1) + + conn.sendall(b'msg2') + conn.close() + + with gevent.Timeout(2.1): + self.popen.wait() + finally: + server.close() + + self.assertEqual([b'msg1', b'msg2'], log) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__example_udp_client.py b/src/gevent/tests/test__example_udp_client.py new file mode 100644 index 0000000..d4dacec --- /dev/null +++ b/src/gevent/tests/test__example_udp_client.py @@ -0,0 +1,31 @@ +from gevent import monkey +monkey.patch_all(subprocess=True) + +import sys +from gevent.server import DatagramServer + +from gevent.testing.util import run +from gevent.testing import util +from gevent.testing import main + +class Test_udp_client(util.TestServer): + + def test(self): + log = [] + + def handle(message, address): + log.append(message) + server.sendto(b'reply-from-server', address) + + server = DatagramServer('127.0.0.1:9001', handle) + server.start() + try: + run([sys.executable, '-W', 'ignore', '-u', 'udp_client.py', 'Test_udp_client'], + timeout=10, cwd=self.cwd) + finally: + server.close() + self.assertEqual(log, [b'Test_udp_client']) + + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__example_udp_server.py b/src/gevent/tests/test__example_udp_server.py new file mode 100644 index 0000000..23c4057 --- /dev/null +++ b/src/gevent/tests/test__example_udp_server.py @@ -0,0 +1,22 @@ +import socket + +from gevent.testing import util +from gevent.testing import main + + +class Test(util.TestServer): + server = 'udp_server.py' + + def _run_all_tests(self): + sock = socket.socket(type=socket.SOCK_DGRAM) + try: + sock.connect(('127.0.0.1', 9000)) + sock.send(b'Test udp_server') + data, _address = sock.recvfrom(8192) + self.assertEqual(data, b'Received 15 bytes') + finally: + sock.close() + + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__examples.py b/src/gevent/tests/test__examples.py new file mode 100644 index 0000000..3310be9 --- /dev/null +++ b/src/gevent/tests/test__examples.py @@ -0,0 +1,87 @@ +import sys +import os +import glob +import time +import unittest + +import gevent.testing as greentest +from gevent.testing import util + +this_dir = os.path.dirname(__file__) + +def _find_files_to_ignore(): + old_dir = os.getcwd() + try: + os.chdir(this_dir) + + result = [ + 'wsgiserver.py', + 'wsgiserver_ssl.py', + 'webproxy.py', + 'webpy.py', + 'unixsocket_server.py', + 'unixsocket_client.py', + 'psycopg2_pool.py', + 'geventsendfile.py', + ] + result += [x[14:] for x in glob.glob('test__example_*.py')] + + finally: + os.chdir(old_dir) + + return result + +default_time_range = (2, 4) +time_ranges = { + 'concurrent_download.py': (0, 30), + 'processes.py': (0, 4) +} + +class _AbstractTestMixin(util.ExampleMixin): + time_range = (2, 4) + filename = None + + def test_runs(self): + start = time.time() + min_time, max_time = self.time_range + if util.run([sys.executable, '-u', self.filename], + timeout=max_time, + cwd=self.cwd, + quiet=True, + buffer_output=True, + nested=True, + setenv={'GEVENT_DEBUG': 'error'}): + self.fail("Failed example: " + self.filename) + else: + took = time.time() - start + self.assertGreaterEqual(took, min_time) + +def _build_test_classes(): + result = {} + try: + example_dir = util.ExampleMixin().cwd + except unittest.SkipTest: + util.log("WARNING: No examples dir found", color='suboptimal-behaviour') + return result + + ignore = _find_files_to_ignore() + for filename in glob.glob(example_dir + '/*.py'): + bn = os.path.basename(filename) + if bn in ignore: + continue + tc = type( + 'Test_' + bn, + (_AbstractTestMixin, greentest.TestCase), + { + 'filename': bn, + 'time_range': time_ranges.get(bn, _AbstractTestMixin.time_range) + } + ) + result[tc.__name__] = tc + return result + +for k, v in _build_test_classes().items(): + locals()[k] = v + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__exc_info.py b/src/gevent/tests/test__exc_info.py new file mode 100644 index 0000000..7656ab8 --- /dev/null +++ b/src/gevent/tests/test__exc_info.py @@ -0,0 +1,58 @@ +import gevent +import sys +import gevent.testing as greentest +from gevent.testing import six +from gevent.testing import ExpectedException as ExpectedError + +if not six.PY3: + sys.exc_clear() + +class RawException(Exception): + pass + + +def hello(err): + assert sys.exc_info() == (None, None, None), sys.exc_info() + raise err + + +def hello2(): + try: + hello(ExpectedError('expected exception in hello')) + except ExpectedError: + pass + + +class Test(greentest.TestCase): + + def test1(self): + error = RawException('hello') + expected_error = ExpectedError('expected exception in hello') + try: + raise error + except RawException: + self.expect_one_error() + g = gevent.spawn(hello, expected_error) + g.join() + self.assert_error(ExpectedError, expected_error) + self.assertIsInstance(g.exception, ExpectedError) + + try: + raise + except: # pylint:disable=bare-except + ex = sys.exc_info()[1] + self.assertIs(ex, error) + + def test2(self): + timer = gevent.get_hub().loop.timer(0) + timer.start(hello2) + try: + gevent.sleep(0.1) + self.assertEqual(sys.exc_info(), (None, None, None)) + finally: + timer.close() + + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__execmodules.py b/src/gevent/tests/test__execmodules.py new file mode 100644 index 0000000..1844a30 --- /dev/null +++ b/src/gevent/tests/test__execmodules.py @@ -0,0 +1,39 @@ +import unittest +import warnings + +from gevent.testing.modules import walk_modules +from gevent.testing import main +from gevent.testing.sysinfo import NON_APPLICABLE_SUFFIXES + + +from gevent.testing import six + + +class TestExec(unittest.TestCase): + pass + + +def make_exec_test(path, module): + + def test(_): + with open(path, 'rb') as f: + src = f.read() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + six.exec_(src, {'__file__': path}) + + name = "test_" + module.replace(".", "_") + test.__name__ = name + setattr(TestExec, name, test) + +def make_all_tests(): + for path, module in walk_modules(recursive=True): + if module.endswith(NON_APPLICABLE_SUFFIXES): + continue + make_exec_test(path, module) + + +make_all_tests() + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__fileobject.py b/src/gevent/tests/test__fileobject.py new file mode 100644 index 0000000..d3bdef8 --- /dev/null +++ b/src/gevent/tests/test__fileobject.py @@ -0,0 +1,278 @@ +from __future__ import print_function +import os +import sys +import tempfile +import gc +import unittest + +import gevent +from gevent import fileobject + +import gevent.testing as greentest +from gevent.testing.sysinfo import PY3 +from gevent.testing.flaky import reraiseFlakyTestRaceConditionLibuv +from gevent.testing.skipping import skipOnLibuvOnCIOnPyPy + + +try: + ResourceWarning +except NameError: + class ResourceWarning(Warning): + "Python 2 fallback" + + +def writer(fobj, line): + for character in line: + fobj.write(character) + fobj.flush() + fobj.close() + + +def close_fd_quietly(fd): + try: + os.close(fd) + except (IOError, OSError): + pass + +class TestFileObjectBlock(greentest.TestCase): + + def _getTargetClass(self): + return fileobject.FileObjectBlock + + def _makeOne(self, *args, **kwargs): + return self._getTargetClass()(*args, **kwargs) + + def _test_del(self, **kwargs): + r, w = os.pipe() + self.addCleanup(close_fd_quietly, r) + self.addCleanup(close_fd_quietly, w) + + self._do_test_del((r, w), **kwargs) + + def _do_test_del(self, pipe, **kwargs): + r, w = pipe + s = self._makeOne(w, 'wb', **kwargs) + s.write(b'x') + try: + s.flush() + except IOError: + # Sometimes seen on Windows/AppVeyor + print("Failed flushing fileobject", repr(s), file=sys.stderr) + import traceback + traceback.print_exc() + + import warnings + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + # Deliberately getting ResourceWarning with FileObject(Thread) under Py3 + del s + gc.collect() # PyPy + + if kwargs.get("close", True): + with self.assertRaises((OSError, IOError)): + # expected, because FileObject already closed it + os.close(w) + else: + os.close(w) + + with self._makeOne(r, 'rb') as fobj: + self.assertEqual(fobj.read(), b'x') + + def test_del(self): + # Close should be true by default + self._test_del() + + def test_del_close(self): + self._test_del(close=True) + + @skipOnLibuvOnCIOnPyPy("This appears to crash on libuv/pypy/travis.") + # No idea why, can't duplicate locally. + def test_seek(self): + fileno, path = tempfile.mkstemp('.gevent.test__fileobject.test_seek') + self.addCleanup(os.remove, path) + + s = b'a' * 1024 + os.write(fileno, b'B' * 15) + os.write(fileno, s) + os.close(fileno) + + with open(path, 'rb') as f: + f.seek(15) + native_data = f.read(1024) + + with open(path, 'rb') as f_raw: + try: + f = self._makeOne(f_raw, 'rb', close=False) + except ValueError: + # libuv on Travis can raise EPERM + # from FileObjectPosix. I can't produce it on mac os locally, + # don't know what the issue is. This started happening on Jan 19, + # in the branch that caused all watchers to be explicitly closed. + # That shouldn't have any effect on io watchers, though, which were + # already being explicitly closed. + reraiseFlakyTestRaceConditionLibuv() + + if PY3 or hasattr(f, 'seekable'): + # On Python 3, all objects should have seekable. + # On Python 2, only our custom objects do. + self.assertTrue(f.seekable()) + f.seek(15) + self.assertEqual(15, f.tell()) + + # Note that a duplicate close() of the underlying + # file descriptor can look like an OSError from this line + # as we exit the with block + fileobj_data = f.read(1024) + + self.assertEqual(native_data, s) + self.assertEqual(native_data, fileobj_data) + + def test_close_pipe(self): + # Issue #190, 203 + r, w = os.pipe() + x = self._makeOne(r) + y = self._makeOne(w, 'w') + x.close() + y.close() + + +class ConcurrentFileObjectMixin(object): + # Additional tests for fileobjects that cooperate + # and we have full control of the implementation + + def test_read1(self): + # Issue #840 + r, w = os.pipe() + x = self._makeOne(r) + y = self._makeOne(w, 'w') + self._close_on_teardown(x) + self._close_on_teardown(y) + self.assertTrue(hasattr(x, 'read1')) + + def test_bufsize_0(self): + # Issue #840 + r, w = os.pipe() + x = self._makeOne(r, 'rb', bufsize=0) + y = self._makeOne(w, 'wb', bufsize=0) + self._close_on_teardown(x) + self._close_on_teardown(y) + y.write(b'a') + b = x.read(1) + self.assertEqual(b, b'a') + + y.writelines([b'2']) + b = x.read(1) + self.assertEqual(b, b'2') + + def test_newlines(self): + import warnings + r, w = os.pipe() + lines = [b'line1\n', b'line2\r', b'line3\r\n', b'line4\r\nline5', b'\nline6'] + g = gevent.spawn(writer, self._makeOne(w, 'wb'), lines) + + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + # U is deprecated in Python 3, shows up on FileObjectThread + fobj = self._makeOne(r, 'rU') + result = fobj.read() + fobj.close() + self.assertEqual('line1\nline2\nline3\nline4\nline5\nline6', result) + finally: + g.kill() + + +class TestFileObjectThread(ConcurrentFileObjectMixin, + TestFileObjectBlock): + + def _getTargetClass(self): + return fileobject.FileObjectThread + + # FileObjectThread uses os.fdopen() when passed a file-descriptor, + # which returns an object with a destructor that can't be + # bypassed, so we can't even create one that way + def test_del_noclose(self): + with self.assertRaisesRegex(TypeError, + 'FileObjectThread does not support close=False on an fd.'): + self._test_del(close=False) + + # We don't test this with FileObjectThread. Sometimes the + # visibility of the 'close' operation, which happens in a + # background thread, doesn't make it to the foreground + # thread in a timely fashion, leading to 'os.close(4) must + # not succeed' in test_del_close. We have the same thing + # with flushing and closing in test_newlines. Both of + # these are most commonly (only?) observed on Py27/64-bit. + # They also appear on 64-bit 3.6 with libuv + + def test_del(self): + raise unittest.SkipTest("Race conditions") + + def test_del_close(self): + raise unittest.SkipTest("Race conditions") + + +@unittest.skipUnless( + hasattr(fileobject, 'FileObjectPosix'), + "Needs FileObjectPosix" +) +class TestFileObjectPosix(ConcurrentFileObjectMixin, + TestFileObjectBlock): + + def _getTargetClass(self): + return fileobject.FileObjectPosix + + def test_seek_raises_ioerror(self): + # https://github.com/gevent/gevent/issues/1323 + + # Get a non-seekable file descriptor + r, w = os.pipe() + + self.addCleanup(close_fd_quietly, r) + self.addCleanup(close_fd_quietly, w) + + with self.assertRaises(OSError) as ctx: + os.lseek(r, 0, os.SEEK_SET) + os_ex = ctx.exception + + with self.assertRaises(IOError) as ctx: + f = self._makeOne(r, 'r', close=False) + # Seek directly using the underlying GreenFileDescriptorIO; + # the buffer may do different things, depending + # on the version of Python (especially 3.7+) + f.fileio.seek(0) + io_ex = ctx.exception + + self.assertEqual(io_ex.errno, os_ex.errno) + self.assertEqual(io_ex.strerror, os_ex.strerror) + self.assertEqual(io_ex.args, os_ex.args) + self.assertEqual(str(io_ex), str(os_ex)) + + +class TestTextMode(unittest.TestCase): + + def test_default_mode_writes_linesep(self): + # See https://github.com/gevent/gevent/issues/1282 + # libuv 1.x interferes with the default line mode on + # Windows. + # First, make sure we initialize gevent + gevent.get_hub() + + fileno, path = tempfile.mkstemp('.gevent.test__fileobject.test_default') + self.addCleanup(os.remove, path) + + os.close(fileno) + + with open(path, "w") as f: + f.write("\n") + + with open(path, "rb") as f: + data = f.read() + + self.assertEqual(data, os.linesep.encode('ascii')) + + + +if __name__ == '__main__': + sys.argv.append('-v') + greentest.main() diff --git a/src/gevent/tests/test__getaddrinfo_import.py b/src/gevent/tests/test__getaddrinfo_import.py new file mode 100644 index 0000000..098ea91 --- /dev/null +++ b/src/gevent/tests/test__getaddrinfo_import.py @@ -0,0 +1,7 @@ +# a deadlock is possible if we import a module that runs Gevent's getaddrinfo +# with a unicode hostname, which starts Python's getaddrinfo on a thread, which +# attempts to import encodings.idna but blocks on the import lock. verify +# that Gevent avoids this deadlock. + +import getaddrinfo_module +del getaddrinfo_module # fix pyflakes diff --git a/src/gevent/tests/test__greenio.py b/src/gevent/tests/test__greenio.py new file mode 100644 index 0000000..08c4d6a --- /dev/null +++ b/src/gevent/tests/test__greenio.py @@ -0,0 +1,128 @@ +# Copyright (c) 2006-2007, Linden Research, Inc. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +import sys + +import gevent +from gevent import socket + +from gevent.testing import TestCase, main, tcp_listener +from gevent.testing import skipOnPyPy +from gevent.testing import params + + + +PYPY = hasattr(sys, 'pypy_version_info') +PY3 = sys.version_info[0] >= 3 + + +def _write_to_closed(f, s): + try: + r = f.write(s) + except ValueError: + assert PY3 + else: + assert r is None, r + + +class TestGreenIo(TestCase): + + def test_close_with_makefile(self): + + def accept_close_early(listener): + # verify that the makefile and the socket are truly independent + # by closing the socket prior to using the made file + try: + conn, _ = listener.accept() + fd = conn.makefile(mode='wb') + conn.close() + fd.write(b'hello\n') + fd.close() + _write_to_closed(fd, b'a') + self.assertRaises(socket.error, conn.send, b'b') + finally: + listener.close() + + def accept_close_late(listener): + # verify that the makefile and the socket are truly independent + # by closing the made file and then sending a character + try: + conn, _ = listener.accept() + fd = conn.makefile(mode='wb') + fd.write(b'hello') + fd.close() + conn.send(b'\n') + conn.close() + _write_to_closed(fd, b'a') + self.assertRaises(socket.error, conn.send, b'b') + finally: + listener.close() + + def did_it_work(server): + client = socket.create_connection((params.DEFAULT_CONNECT, server.getsockname()[1])) + fd = client.makefile(mode='rb') + client.close() + self.assertEqual(fd.readline(), b'hello\n') + self.assertFalse(fd.read()) + fd.close() + + server = tcp_listener() + server_greenlet = gevent.spawn(accept_close_early, server) + did_it_work(server) + server_greenlet.kill() + + server = tcp_listener() + server_greenlet = gevent.spawn(accept_close_late, server) + did_it_work(server) + server_greenlet.kill() + + @skipOnPyPy("GC is different") + def test_del_closes_socket(self): + def accept_once(listener): + # delete/overwrite the original conn + # object, only keeping the file object around + # closing the file object should close everything + + # XXX: This is not exactly true on Python 3. + # This produces a ResourceWarning. + oconn = None + try: + conn, _ = listener.accept() + if PY3: + oconn = conn + conn = conn.makefile(mode='wb') + conn.write(b'hello\n') + conn.close() + _write_to_closed(conn, b'a') + finally: + listener.close() + if oconn is not None: + oconn.close() + + server = tcp_listener() + gevent.spawn(accept_once, server) + client = socket.create_connection((params.DEFAULT_CONNECT, server.getsockname()[1])) + with gevent.Timeout.start_new(0.5): + fd = client.makefile() + client.close() + self.assertEqual(fd.read(), 'hello\n') + self.assertEqual(fd.read(), '') + + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__greenlet.py b/src/gevent/tests/test__greenlet.py new file mode 100644 index 0000000..208482d --- /dev/null +++ b/src/gevent/tests/test__greenlet.py @@ -0,0 +1,831 @@ +# Copyright (c) 2008-2009 AG Projects +# Author: Denis Bilenko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +import re +import unittest + +import gevent.testing as greentest +import gevent + +from gevent import sleep, with_timeout, getcurrent +from gevent import greenlet +from gevent.event import AsyncResult +from gevent.queue import Queue, Channel + +from gevent.testing.timing import AbstractGenericWaitTestCase +from gevent.testing.timing import AbstractGenericGetTestCase +from gevent.testing import timing + +DELAY = timing.SMALL_TICK +greentest.TestCase.error_fatal = False + + +class ExpectedError(greentest.ExpectedException): + pass + + +class TestLink(greentest.TestCase): + + def test_link_to_asyncresult(self): + p = gevent.spawn(lambda: 100) + event = AsyncResult() + p.link(event) + self.assertEqual(event.get(), 100) + + for _ in range(3): + event2 = AsyncResult() + p.link(event2) + self.assertEqual(event2.get(), 100) + + def test_link_to_asyncresult_exception(self): + err = ExpectedError('test_link_to_asyncresult_exception') + p = gevent.spawn(lambda: getcurrent().throw(err)) + event = AsyncResult() + p.link(event) + with self.assertRaises(ExpectedError) as exc: + event.get() + + self.assertIs(exc.exception, err) + + for _ in range(3): + event2 = AsyncResult() + p.link(event2) + with self.assertRaises(ExpectedError) as exc: + event2.get() + self.assertIs(exc.exception, err) + + def test_link_to_queue(self): + p = gevent.spawn(lambda: 100) + q = Queue() + p.link(q.put) + self.assertEqual(q.get().get(), 100) + + for _ in range(3): + p.link(q.put) + self.assertEqual(q.get().get(), 100) + + def test_link_to_channel(self): + p1 = gevent.spawn(lambda: 101) + p2 = gevent.spawn(lambda: 102) + p3 = gevent.spawn(lambda: 103) + q = Channel() + p1.link(q.put) + p2.link(q.put) + p3.link(q.put) + results = [q.get().get(), q.get().get(), q.get().get()] + assert sorted(results) == [101, 102, 103], results + + +class TestUnlink(greentest.TestCase): + switch_expected = False + + def _test_func(self, p, link): + link(dummy_test_func) + self.assertEqual(1, p.has_links()) + + p.unlink(dummy_test_func) + self.assertEqual(0, p.has_links()) + + link(self.setUp) + self.assertEqual(1, p.has_links()) + + p.unlink(self.setUp) + self.assertEqual(0, p.has_links()) + + p.kill() + + def test_func_link(self): + p = gevent.spawn(dummy_test_func) + self._test_func(p, p.link) + + def test_func_link_value(self): + p = gevent.spawn(dummy_test_func) + self._test_func(p, p.link_value) + + def test_func_link_exception(self): + p = gevent.spawn(dummy_test_func) + self._test_func(p, p.link_exception) + + +class LinksTestCase(greentest.TestCase): + + link_method = None + + def link(self, p, listener=None): + getattr(p, self.link_method)(listener) + + def set_links(self, p): + event = AsyncResult() + self.link(p, event) + + queue = Queue(1) + self.link(p, queue.put) + + callback_flag = ['initial'] + self.link(p, lambda *args: callback_flag.remove('initial')) + + for _ in range(10): + self.link(p, AsyncResult()) + self.link(p, Queue(1).put) + + return event, queue, callback_flag + + def set_links_timeout(self, link): + # stuff that won't be touched + event = AsyncResult() + link(event) + + queue = Channel() + link(queue.put) + return event, queue + + def check_timed_out(self, event, queue): + assert with_timeout(DELAY, event.get, timeout_value=X) is X, repr(event.get()) + assert with_timeout(DELAY, queue.get, timeout_value=X) is X, queue.get() + + +def return25(): + return 25 + + + +class TestReturn_link(LinksTestCase): + link_method = 'link' + + p = None + + def cleanup(self): + self.p.unlink_all() + self.p = None + + def test_return(self): + self.p = gevent.spawn(return25) + for _ in range(3): + self._test_return(self.p, 25) + self.p.kill() + + def _test_return(self, p, result): + event, queue, callback_flag = self.set_links(p) + + # stuff that will time out because there's no unhandled exception: + xxxxx = self.set_links_timeout(p.link_exception) + + sleep(DELAY * 2) + self.assertFalse(p) + + self.assertEqual(event.get(), result) + self.assertEqual(queue.get().get(), result) + + sleep(DELAY) + self.assertFalse(callback_flag) + + self.check_timed_out(*xxxxx) + + def _test_kill(self, p): + event, queue, callback_flag = self.set_links(p) + xxxxx = self.set_links_timeout(p.link_exception) + + p.kill() + sleep(DELAY) + self.assertFalse(p) + + + self.assertIsInstance(event.get(), gevent.GreenletExit) + self.assertIsInstance(queue.get().get(), gevent.GreenletExit) + + sleep(DELAY) + self.assertFalse(callback_flag) + + self.check_timed_out(*xxxxx) + + def test_kill(self): + p = self.p = gevent.spawn(sleep, DELAY) + for _ in range(3): + self._test_kill(p) + + +class TestReturn_link_value(TestReturn_link): + link_method = 'link_value' + + +class TestRaise_link(LinksTestCase): + link_method = 'link' + + def _test_raise(self, p): + event, queue, callback_flag = self.set_links(p) + xxxxx = self.set_links_timeout(p.link_value) + + sleep(DELAY) + assert not p, p + + self.assertRaises(ExpectedError, event.get) + self.assertEqual(queue.get(), p) + sleep(DELAY) + assert not callback_flag, callback_flag + + self.check_timed_out(*xxxxx) + + def test_raise(self): + p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_raise'))) + for _ in range(3): + self._test_raise(p) + + +class TestRaise_link_exception(TestRaise_link): + link_method = 'link_exception' + + +class TestStuff(greentest.TestCase): + + def test_minimal_id(self): + g = gevent.spawn(lambda: 1) + self.assertGreaterEqual(g.minimal_ident, 0) + self.assertGreaterEqual(g.parent.minimal_ident, 0) + g.join() # don't leave dangling, breaks the leak checks + + def test_wait_noerrors(self): + x = gevent.spawn(lambda: 1) + y = gevent.spawn(lambda: 2) + z = gevent.spawn(lambda: 3) + gevent.joinall([x, y, z], raise_error=True) + self.assertEqual([x.value, y.value, z.value], [1, 2, 3]) + e = AsyncResult() + x.link(e) + self.assertEqual(e.get(), 1) + x.unlink(e) + e = AsyncResult() + x.link(e) + self.assertEqual(e.get(), 1) + + def test_wait_error(self): + + def x(): + sleep(DELAY) + return 1 + x = gevent.spawn(x) + y = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_wait_error'))) + self.assertRaises(ExpectedError, gevent.joinall, [x, y], raise_error=True) + self.assertRaises(ExpectedError, gevent.joinall, [y], raise_error=True) + x.join() + test_wait_error.ignore_leakcheck = True + + def test_joinall_exception_order(self): + # if there're several exceptions raised, the earliest one must be raised by joinall + def first(): + sleep(0.1) + raise ExpectedError('first') + a = gevent.spawn(first) + b = gevent.spawn(lambda: getcurrent().throw(ExpectedError('second'))) + try: + gevent.joinall([a, b], raise_error=True) + except ExpectedError as ex: + assert 'second' in str(ex), repr(str(ex)) + gevent.joinall([a, b]) + test_joinall_exception_order.ignore_leakcheck = True + + def test_joinall_count_raise_error(self): + # When joinall is asked not to raise an error, the 'count' param still + # works. + def raises_but_ignored(): + raise ExpectedError("count") + + def sleep_forever(): + while True: + sleep(0.1) + + sleeper = gevent.spawn(sleep_forever) + raiser = gevent.spawn(raises_but_ignored) + + gevent.joinall([sleeper, raiser], raise_error=False, count=1) + assert_ready(raiser) + assert_not_ready(sleeper) + + # Clean up our mess + sleeper.kill() + assert_ready(sleeper) + + def test_multiple_listeners_error(self): + # if there was an error while calling a callback + # it should not prevent the other listeners from being called + # also, all of the errors should be logged, check the output + # manually that they are + p = gevent.spawn(lambda: 5) + results = [] + + def listener1(*_args): + results.append(10) + raise ExpectedError('listener1') + + def listener2(*_args): + results.append(20) + raise ExpectedError('listener2') + + def listener3(*_args): + raise ExpectedError('listener3') + + p.link(listener1) + p.link(listener2) + p.link(listener3) + sleep(DELAY * 10) + self.assertIn(results, [[10, 20], [20, 10]]) + + p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_multiple_listeners_error'))) + results = [] + p.link(listener1) + p.link(listener2) + p.link(listener3) + sleep(DELAY * 10) + self.assertIn(results, [[10, 20], [20, 10]]) + + class Results(object): + + def __init__(self): + self.results = [] + + def listener1(self, p): + p.unlink(self.listener2) + self.results.append(5) + raise ExpectedError('listener1') + + def listener2(self, p): + p.unlink(self.listener1) + self.results.append(5) + raise ExpectedError('listener2') + + def listener3(self, _p): + raise ExpectedError('listener3') + + def _test_multiple_listeners_error_unlink(self, _p, link): + # notification must not happen after unlink even + # though notification process has been already started + results = self.Results() + + link(results.listener1) + link(results.listener2) + link(results.listener3) + sleep(DELAY * 10) + self.assertEqual([5], results.results) + + + def test_multiple_listeners_error_unlink_Greenlet_link(self): + p = gevent.spawn(lambda: 5) + self._test_multiple_listeners_error_unlink(p, p.link) + p.kill() + + def test_multiple_listeners_error_unlink_Greenlet_rawlink(self): + p = gevent.spawn(lambda: 5) + self._test_multiple_listeners_error_unlink(p, p.rawlink) + + def test_multiple_listeners_error_unlink_AsyncResult_rawlink(self): + e = AsyncResult() + gevent.spawn(e.set, 6) + self._test_multiple_listeners_error_unlink(e, e.rawlink) + + +def dummy_test_func(*_args): + pass + + +class A(object): + + def method(self): + pass + +hexobj = re.compile('-?0x[0123456789abcdef]+L?', re.I) + +class Subclass(gevent.Greenlet): + pass + +class TestStr(greentest.TestCase): + + def test_function(self): + g = gevent.Greenlet.spawn(dummy_test_func) + self.assertTrue(hexobj.sub('X', str(g)).endswith('at X: dummy_test_func>')) + assert_not_ready(g) + g.join() + assert_ready(g) + self.assertTrue(hexobj.sub('X', str(g)).endswith(' at X: dummy_test_func>'), str(g)) + + def test_method(self): + g = gevent.Greenlet.spawn(A().method) + str_g = hexobj.sub('X', str(g)) + str_g = str_g.replace(__name__, 'module') + self.assertTrue(str_g.startswith('>>')) + assert_not_ready(g) + g.join() + assert_ready(g) + str_g = hexobj.sub('X', str(g)) + str_g = str_g.replace(__name__, 'module') + self.assertTrue(str_g.endswith('at X: >>')) + + def test_subclass(self): + g = Subclass() + str_g = hexobj.sub('X', str(g)) + str_g = str_g.replace(__name__, 'module') + self.assertTrue(str_g.startswith('')) + + g = Subclass(None, 'question', answer=42) + str_g = hexobj.sub('X', str(g)) + str_g = str_g.replace(__name__, 'module') + self.assertTrue(str_g.endswith(" at X: _run('question', answer=42)>")) + + +class TestJoin(AbstractGenericWaitTestCase): + + def wait(self, timeout): + g = gevent.spawn(gevent.sleep, 10) + try: + return g.join(timeout=timeout) + finally: + g.kill() + + +class TestGet(AbstractGenericGetTestCase): + + def wait(self, timeout): + g = gevent.spawn(gevent.sleep, 10) + try: + return g.get(timeout=timeout) + finally: + g.kill() + + +class TestJoinAll0(AbstractGenericWaitTestCase): + + g = gevent.Greenlet() + + def wait(self, timeout): + gevent.joinall([self.g], timeout=timeout) + + +class TestJoinAll(AbstractGenericWaitTestCase): + + def wait(self, timeout): + g = gevent.spawn(gevent.sleep, 10) + try: + gevent.joinall([g], timeout=timeout) + finally: + g.kill() + + +class TestBasic(greentest.TestCase): + + def test_spawn_non_callable(self): + self.assertRaises(TypeError, gevent.spawn, 1) + self.assertRaises(TypeError, gevent.spawn_raw, 1) + + # Not passing the run argument, just the seconds argument + self.assertRaises(TypeError, gevent.spawn_later, 1) + # Passing both, but not implemented + self.assertRaises(TypeError, gevent.spawn_later, 1, 1) + + def test_spawn_raw_kwargs(self): + value = [] + + def f(*args, **kwargs): + value.append(args) + value.append(kwargs) + + g = gevent.spawn_raw(f, 1, name='value') + gevent.sleep(0.01) + assert not g + self.assertEqual(value[0], (1,)) + self.assertEqual(value[1], {'name': 'value'}) + + def test_simple_exit(self): + link_test = [] + + def func(delay, return_value=4): + gevent.sleep(delay) + return return_value + + g = gevent.Greenlet(func, 0.01, return_value=5) + g.rawlink(link_test.append) # use rawlink to avoid timing issues on Appveyor/Travis (not always successful) + assert not g, bool(g) + assert not g.dead + assert not g.started + assert not g.ready() + assert not g.successful() + assert g.value is None + assert g.exception is None + + g.start() + assert g # changed + assert not g.dead + assert g.started # changed + assert not g.ready() + assert not g.successful() + assert g.value is None + assert g.exception is None + + gevent.sleep(0.001) + self.assertTrue(g) + self.assertFalse(g.dead, g) + self.assertTrue(g.started, g) + self.assertFalse(g.ready(), g) + self.assertFalse(g.successful(), g) + self.assertIsNone(g.value, g) + self.assertIsNone(g.exception, g) + self.assertFalse(link_test) + + gevent.sleep(0.02) + assert not g + assert g.dead + assert not g.started + assert g.ready() + assert g.successful() + assert g.value == 5 + assert g.exception is None # not changed + assert link_test == [g] or greentest.RUNNING_ON_CI, link_test # changed + + def test_error_exit(self): + link_test = [] + + def func(delay, return_value=4): + gevent.sleep(delay) + error = ExpectedError('test_error_exit') + setattr(error, 'myattr', return_value) + raise error + + g = gevent.Greenlet(func, timing.SMALLEST_RELIABLE_DELAY, return_value=5) + # use rawlink to avoid timing issues on Appveyor (not always successful) + g.rawlink(link_test.append) + g.start() + gevent.sleep() + gevent.sleep(timing.LARGE_TICK) + self.assertFalse(g) + self.assertTrue(g.dead) + self.assertFalse(g.started) + self.assertTrue(g.ready()) + self.assertFalse(g.successful()) + self.assertIsNone(g.value) # not changed + self.assertEqual(g.exception.myattr, 5) + + assert link_test == [g] or greentest.RUNNING_ON_APPVEYOR, link_test + + def _assertKilled(self, g): + assert not g + assert g.dead + assert not g.started + assert g.ready() + assert g.successful(), (repr(g), g.value, g.exception) + assert isinstance(g.value, gevent.GreenletExit), (repr(g), g.value, g.exception) + assert g.exception is None + + def assertKilled(self, g): + self._assertKilled(g) + gevent.sleep(0.01) + self._assertKilled(g) + + def _test_kill(self, g, block): + g.kill(block=block) + if not block: + gevent.sleep(0.01) + self.assertKilled(g) + # kill second time must not hurt + g.kill(block=block) + self.assertKilled(g) + + def _test_kill_not_started(self, block): + link_test = [] + result = [] + g = gevent.Greenlet(lambda: result.append(1)) + g.link(link_test.append) + self._test_kill(g, block=block) + assert not result + assert link_test == [g] + + def test_kill_not_started_block(self): + self._test_kill_not_started(block=True) + + def test_kill_not_started_noblock(self): + self._test_kill_not_started(block=False) + + def _test_kill_just_started(self, block): + result = [] + link_test = [] + g = gevent.Greenlet(lambda: result.append(1)) + g.link(link_test.append) + g.start() + self._test_kill(g, block=block) + assert not result, result + assert link_test == [g] + + def test_kill_just_started_block(self): + self._test_kill_just_started(block=True) + + def test_kill_just_started_noblock(self): + self._test_kill_just_started(block=False) + + def _test_kill_just_started_later(self, block): + result = [] + link_test = [] + g = gevent.Greenlet(lambda: result.append(1)) + g.link(link_test.append) + g.start_later(1) + self._test_kill(g, block=block) + assert not result + + def test_kill_just_started_later_block(self): + self._test_kill_just_started_later(block=True) + + def test_kill_just_started_later_noblock(self): + self._test_kill_just_started_later(block=False) + + def _test_kill_running(self, block): + link_test = [] + g = gevent.spawn(gevent.sleep, 10) + g.link(link_test.append) + self._test_kill(g, block=block) + gevent.sleep(0.01) + assert link_test == [g] + + def test_kill_running_block(self): + self._test_kill_running(block=True) + + def test_kill_running_noblock(self): + self._test_kill_running(block=False) + + def test_exc_info_no_error(self): + # Before running + self.assertFalse(greenlet.Greenlet().exc_info) + g = greenlet.Greenlet(gevent.sleep) + g.start() + g.join() + self.assertFalse(g.exc_info) + + def test_tree_locals(self): + g = g2 = None + def func(): + child = greenlet.Greenlet() + self.assertIs(child.spawn_tree_locals, getcurrent().spawn_tree_locals) + self.assertIs(child.spawning_greenlet(), getcurrent()) + g = greenlet.Greenlet(func) + g2 = greenlet.Greenlet(func) + # Creating those greenlets did not give the main greenlet + # a locals dict. + self.assertFalse(hasattr(getcurrent(), 'spawn_tree_locals'), + getcurrent()) + self.assertIsNot(g.spawn_tree_locals, g2.spawn_tree_locals) + g.start() + g.join() + + raw = gevent.spawn_raw(func) + self.assertIsNotNone(raw.spawn_tree_locals) + self.assertIsNot(raw.spawn_tree_locals, g.spawn_tree_locals) + self.assertIs(raw.spawning_greenlet(), getcurrent()) + while not raw.dead: + gevent.sleep(0.01) + + def test_add_spawn_callback(self): + called = {'#': 0} + + def cb(gr): + called['#'] += 1 + gr._called_test = True + + gevent.Greenlet.add_spawn_callback(cb) + try: + g = gevent.spawn(lambda: None) + self.assertTrue(hasattr(g, '_called_test')) + g.join() + self.assertEqual(called['#'], 1) + + g = gevent.spawn_later(1e-5, lambda: None) + self.assertTrue(hasattr(g, '_called_test')) + g.join() + self.assertEqual(called['#'], 2) + + g = gevent.Greenlet(lambda: None) + g.start() + self.assertTrue(hasattr(g, '_called_test')) + g.join() + self.assertEqual(called['#'], 3) + + gevent.Greenlet.remove_spawn_callback(cb) + g = gevent.spawn(lambda: None) + self.assertFalse(hasattr(g, '_called_test')) + g.join() + self.assertEqual(called['#'], 3) + finally: + gevent.Greenlet.remove_spawn_callback(cb) + + def test_getframe_value_error(self): + def get(): + raise ValueError("call stack is not deep enough") + try: + ogf = greenlet.sys_getframe + except AttributeError: # pragma: no cover + # Must be running cython compiled + raise unittest.SkipTest("Cannot mock when Cython compiled") + greenlet.sys_getframe = get + try: + child = greenlet.Greenlet() + self.assertIsNone(child.spawning_stack) + finally: + greenlet.sys_getframe = ogf + + +class TestStart(greentest.TestCase): + + def test(self): + g = gevent.spawn(gevent.sleep, 0.01) + assert g.started + assert not g.dead + g.start() + assert g.started + assert not g.dead + g.join() + assert not g.started + assert g.dead + g.start() + assert not g.started + assert g.dead + + +def assert_ready(g): + assert g.dead, g + assert g.ready(), g + assert not bool(g), g + + +def assert_not_ready(g): + assert not g.dead, g + assert not g.ready(), g + + +class TestRef(greentest.TestCase): + + def test_init(self): + self.switch_expected = False + # in python-dbg mode this will check that Greenlet() does not create any circular refs + gevent.Greenlet() + + def test_kill_scheduled(self): + gevent.spawn(gevent.sleep, 10).kill() + + def test_kill_started(self): + g = gevent.spawn(gevent.sleep, 10) + try: + gevent.sleep(0.001) + finally: + g.kill() + + +@greentest.skipOnPurePython("Needs C extension") +class TestCExt(greentest.TestCase): # pragma: no cover (we only do coverage on pure-Python) + + def test_c_extension(self): + self.assertEqual(greenlet.Greenlet.__module__, + 'gevent._greenlet') + self.assertEqual(greenlet.SpawnedLink.__module__, + 'gevent._greenlet') + +@greentest.skipWithCExtensions("Needs pure python") +class TestPure(greentest.TestCase): + + def test_pure(self): + self.assertEqual(greenlet.Greenlet.__module__, + 'gevent.greenlet') + self.assertEqual(greenlet.SpawnedLink.__module__, + 'gevent.greenlet') + + +X = object() + +del AbstractGenericGetTestCase +del AbstractGenericWaitTestCase + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__greenletset.py b/src/gevent/tests/test__greenletset.py new file mode 100644 index 0000000..6122857 --- /dev/null +++ b/src/gevent/tests/test__greenletset.py @@ -0,0 +1,164 @@ +from __future__ import print_function, division, absolute_import +import time +import gevent.testing as greentest + +from gevent.testing import timing +import gevent +from gevent import pool +from gevent.timeout import Timeout + +DELAY = timing.LARGE_TICK + + +class SpecialError(Exception): + pass + + +class Undead(object): + + def __init__(self): + self.shot_count = 0 + + def __call__(self): + while True: + try: + gevent.sleep(1) + except SpecialError: + break + except: # pylint:disable=bare-except + self.shot_count += 1 + + +class Test(greentest.TestCase): + + __timeout__ = greentest.LARGE_TIMEOUT + + def test_basic(self): + s = pool.Group() + s.spawn(gevent.sleep, timing.LARGE_TICK) + self.assertEqual(len(s), 1, s) + s.spawn(gevent.sleep, timing.LARGE_TICK * 5) + self.assertEqual(len(s), 2, s) + gevent.sleep() + gevent.sleep(timing.LARGE_TICK * 2 + timing.LARGE_TICK_MIN_ADJ) + self.assertEqual(len(s), 1, s) + gevent.sleep(timing.LARGE_TICK * 5 + timing.LARGE_TICK_MIN_ADJ) + self.assertFalse(s) + + def test_waitall(self): + s = pool.Group() + s.spawn(gevent.sleep, DELAY) + s.spawn(gevent.sleep, DELAY * 2) + assert len(s) == 2, s + start = time.time() + s.join(raise_error=True) + delta = time.time() - start + self.assertFalse(s) + self.assertEqual(len(s), 0) + self.assertTimeWithinRange(delta, DELAY * 1.9, DELAY * 2.5) + + def test_kill_block(self): + s = pool.Group() + s.spawn(gevent.sleep, DELAY) + s.spawn(gevent.sleep, DELAY * 2) + assert len(s) == 2, s + start = time.time() + s.kill() + self.assertFalse(s) + self.assertEqual(len(s), 0) + delta = time.time() - start + assert delta < DELAY * 0.8, delta + + def test_kill_noblock(self): + s = pool.Group() + s.spawn(gevent.sleep, DELAY) + s.spawn(gevent.sleep, DELAY * 2) + assert len(s) == 2, s + s.kill(block=False) + assert len(s) == 2, s + gevent.sleep(0.0001) + self.assertFalse(s) + self.assertEqual(len(s), 0) + + def test_kill_fires_once(self): + u1 = Undead() + u2 = Undead() + p1 = gevent.spawn(u1) + p2 = gevent.spawn(u2) + + def check(count1, count2): + self.assertTrue(p1) + self.assertTrue(p2) + self.assertFalse(p1.dead, p1) + self.assertFalse(p2.dead, p2) + self.assertEqual(u1.shot_count, count1) + self.assertEqual(u2.shot_count, count2) + + gevent.sleep(0.01) + s = pool.Group([p1, p2]) + self.assertEqual(len(s), 2, s) + check(0, 0) + s.killone(p1, block=False) + check(0, 0) + gevent.sleep(0) + check(1, 0) + s.killone(p1) + check(1, 0) + s.killone(p1) + check(1, 0) + s.kill(block=False) + s.kill(block=False) + s.kill(block=False) + check(1, 0) + gevent.sleep(DELAY) + check(1, 1) + X = object() + kill_result = gevent.with_timeout(DELAY, s.kill, block=True, timeout_value=X) + assert kill_result is X, repr(kill_result) + assert len(s) == 2, s + check(1, 1) + + p1.kill(SpecialError) + p2.kill(SpecialError) + + def test_killall_subclass(self): + p1 = GreenletSubclass.spawn(lambda: 1 / 0) + p2 = GreenletSubclass.spawn(lambda: gevent.sleep(10)) + s = pool.Group([p1, p2]) + s.kill() + + def test_killall_iterable_argument_non_block(self): + p1 = GreenletSubclass.spawn(lambda: gevent.sleep(0.5)) + p2 = GreenletSubclass.spawn(lambda: gevent.sleep(0.5)) + s = set() + s.add(p1) + s.add(p2) + gevent.killall(s, block=False) + gevent.sleep(0.5) + for g in s: + assert g.dead + + def test_killall_iterable_argument_timeout(self): + def f(): + try: + gevent.sleep(1.5) + except: # pylint:disable=bare-except + gevent.sleep(1) + p1 = GreenletSubclass.spawn(f) + p2 = GreenletSubclass.spawn(f) + s = set() + s.add(p1) + s.add(p2) + with self.assertRaises(Timeout): + gevent.killall(s, timeout=0.5) + + for g in s: + self.assertFalse(g.dead, g) + + +class GreenletSubclass(gevent.Greenlet): + pass + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__greenness.py b/src/gevent/tests/test__greenness.py new file mode 100644 index 0000000..5da5a4c --- /dev/null +++ b/src/gevent/tests/test__greenness.py @@ -0,0 +1,73 @@ +# Copyright (c) 2008 AG Projects +# Author: Denis Bilenko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""Test than modules in gevent.green package are indeed green. +To do that spawn a green server and then access it using a green socket. +If either operation blocked the whole script would block and timeout. +""" +from gevent import monkey +monkey.patch_all() + +import gevent.testing as greentest + +try: + import urllib2 +except ImportError: + from urllib import request as urllib2 +try: + import BaseHTTPServer +except ImportError: + from http import server as BaseHTTPServer + +import gevent +from gevent.testing import params + + +class TestGreenness(greentest.TestCase): + check_totalrefcount = False + + def setUp(self): + server_address = params.DEFAULT_BIND_ADDR_TUPLE + BaseHTTPServer.BaseHTTPRequestHandler.protocol_version = "HTTP/1.0" + self.httpd = BaseHTTPServer.HTTPServer(server_address, BaseHTTPServer.BaseHTTPRequestHandler) + self.httpd.request_count = 0 + + def tearDown(self): + self.httpd.server_close() + self.httpd = None + + def serve(self): + self.httpd.handle_request() + self.httpd.request_count += 1 + + def test_urllib2(self): + server = gevent.spawn(self.serve) + + port = self.httpd.socket.getsockname()[1] + with self.assertRaises(urllib2.HTTPError) as exc: + urllib2.urlopen('http://127.0.0.1:%s' % port) + self.assertEqual(exc.exception.code, 501) + server.get(0.01) + self.assertEqual(self.httpd.request_count, 1) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__hub.py b/src/gevent/tests/test__hub.py new file mode 100644 index 0000000..4496202 --- /dev/null +++ b/src/gevent/tests/test__hub.py @@ -0,0 +1,332 @@ +# Copyright (c) 2009 AG Projects +# Author: Denis Bilenko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import re +import time +import unittest + +import gevent.testing as greentest +import gevent.testing.timing + +import gevent +from gevent import socket +from gevent.hub import Waiter, get_hub +from gevent._compat import NativeStrIO + +DELAY = 0.1 + + +class TestCloseSocketWhilePolling(greentest.TestCase): + + def test(self): + sock = socket.socket() + self._close_on_teardown(sock) + t = get_hub().loop.timer(0) + t.start(sock.close) + with self.assertRaises(socket.error): + try: + sock.connect(('python.org', 81)) + finally: + t.close() + + gevent.sleep(0) + + +class TestExceptionInMainloop(greentest.TestCase): + + def test_sleep(self): + # even if there was an error in the mainloop, the hub should continue to work + start = time.time() + gevent.sleep(DELAY) + delay = time.time() - start + + delay_range = DELAY * 0.9 + self.assertTimeWithinRange(delay, DELAY - delay_range, DELAY + delay_range) + + error = greentest.ExpectedException('TestExceptionInMainloop.test_sleep/fail') + + def fail(): + raise error + + with get_hub().loop.timer(0.001) as t: + t.start(fail) + + self.expect_one_error() + + start = time.time() + gevent.sleep(DELAY) + delay = time.time() - start + + self.assert_error(value=error) + self.assertTimeWithinRange(delay, DELAY - delay_range, DELAY + delay_range) + + + +class TestSleep(gevent.testing.timing.AbstractGenericWaitTestCase): + + def wait(self, timeout): + gevent.sleep(timeout) + + def test_simple(self): + gevent.sleep(0) + + +class TestWaiterGet(gevent.testing.timing.AbstractGenericWaitTestCase): + + def setUp(self): + super(TestWaiterGet, self).setUp() + self.waiter = Waiter() + + def wait(self, timeout): + with get_hub().loop.timer(timeout) as evt: + evt.start(self.waiter.switch, None) + return self.waiter.get() + + +class TestWaiter(greentest.TestCase): + + def test(self): + waiter = Waiter() + self.assertEqual(str(waiter), '') + waiter.switch(25) + self.assertEqual(str(waiter), '') + self.assertEqual(waiter.get(), 25) + + waiter = Waiter() + waiter.throw(ZeroDivisionError) + assert re.match('^ midtime: + p.send_signal(signal_to_send) + midtime = endtime + 1 # only once + time.sleep(0.1) + else: + # Kill unresponsive child and exit with error 1 + p.terminate() + p.wait() + raise AssertionError("Failed to wait for child") + + # If we get here, it's because we caused the process to exit; it + # didn't hang. Under Windows, however, we have to use CTRL_BREAK_EVENT, + # which has an arbitrary returncode depending on versions (so does CTRL_C_EVENT + # on Python 2). We still + # count this as success. + self.assertEqual(p.returncode if not WIN else 0, 0) + p.stdout.close() + + if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__iwait.py b/src/gevent/tests/test__iwait.py new file mode 100644 index 0000000..0976e40 --- /dev/null +++ b/src/gevent/tests/test__iwait.py @@ -0,0 +1,42 @@ +import gevent +import gevent.testing as greentest +from gevent.lock import Semaphore + + +class Testiwait(greentest.TestCase): + + def test_noiter(self): + # Test that gevent.iwait returns objects which can be iterated upon + # without additional calls to iter() + + sem1 = Semaphore() + sem2 = Semaphore() + + gevent.spawn(sem1.release) + ready = next(gevent.iwait((sem1, sem2))) + self.assertEqual(sem1, ready) + + def test_iwait_partial(self): + # Test that the iwait context manager allows the iterator to be + # consumed partially without a memory leak. + + sem = Semaphore() + let = gevent.spawn(sem.release) + with gevent.iwait((sem,), timeout=0.01) as iterator: + self.assertEqual(sem, next(iterator)) + let.get() + + def test_iwait_nogarbage(self): + sem1 = Semaphore() + sem2 = Semaphore() + let = gevent.spawn(sem1.release) + with gevent.iwait((sem1, sem2)) as iterator: + self.assertEqual(sem1, next(iterator)) + self.assertEqual(sem2.linkcount(), 1) + + self.assertEqual(sem2.linkcount(), 0) + let.get() + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__joinall.py b/src/gevent/tests/test__joinall.py new file mode 100644 index 0000000..5d31281 --- /dev/null +++ b/src/gevent/tests/test__joinall.py @@ -0,0 +1,10 @@ +import gevent + + +def func(): + pass + + +a = gevent.spawn(func) +b = gevent.spawn(func) +gevent.joinall([a, b, a]) diff --git a/src/gevent/tests/test__local.py b/src/gevent/tests/test__local.py new file mode 100644 index 0000000..823b10e --- /dev/null +++ b/src/gevent/tests/test__local.py @@ -0,0 +1,428 @@ +import gevent.testing as greentest +from copy import copy +# Comment the line below to see that the standard thread.local is working correct +from gevent import monkey; monkey.patch_all() + + +from threading import local +from threading import Thread + +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping + +class ReadProperty(object): + """A property that can be overridden""" + + # A non-data descriptor + + def __get__(self, inst, klass): + return 42 if inst is not None else self + + +class A(local): + __slots__ = ['initialized', 'obj'] + + path = '' + + type_path = 'MyPath' + + read_property = ReadProperty() + + def __init__(self, obj): + super(A, self).__init__() + if not hasattr(self, 'initialized'): + self.obj = obj + self.path = '' + + +class Obj(object): + pass + +# These next two classes have to be global to avoid the leakchecks +deleted_sentinels = [] +created_sentinels = [] + +class Sentinel(object): + def __del__(self): + deleted_sentinels.append(id(self)) + + +class MyLocal(local): + + CLASS_PROP = 42 + + def __init__(self): + local.__init__(self) + self.sentinel = Sentinel() + created_sentinels.append(id(self.sentinel)) + + @property + def desc(self): + return self + +class MyLocalSubclass(MyLocal): + pass + +class WithGetattr(local): + + def __getattr__(self, name): + if name == 'foo': + return 42 + return super(WithGetattr, self).__getattr__(name) + +class LocalWithABC(local, Mapping): + + def __getitem__(self, name): + return self.d[name] + + def __iter__(self): + return iter(self.d) + + def __len__(self): + return len(self.d) + +class LocalWithStaticMethod(local): + + @staticmethod + def a_staticmethod(): + return 42 + +class LocalWithClassMethod(local): + + @classmethod + def a_classmethod(cls): + return cls + + + + +class TestGeventLocal(greentest.TestCase): + # pylint:disable=attribute-defined-outside-init,blacklisted-name + + def setUp(self): + del deleted_sentinels[:] + del created_sentinels[:] + + tearDown = setUp + + def test_create_local_subclass_init_args(self): + with self.assertRaisesRegex(TypeError, + "Initialization arguments are not supported"): + local("foo") + + with self.assertRaisesRegex(TypeError, + "Initialization arguments are not supported"): + local(kw="foo") + + + def test_local_opts_not_subclassed(self): + l = local() + l.attr = 1 + self.assertEqual(l.attr, 1) + + def test_cannot_set_delete_dict(self): + l = local() + with self.assertRaises(AttributeError): + l.__dict__ = 1 + + with self.assertRaises(AttributeError): + del l.__dict__ + + def test_delete_with_no_dict(self): + l = local() + with self.assertRaises(AttributeError): + delattr(l, 'thing') + + def del_local(): + with self.assertRaises(AttributeError): + delattr(l, 'thing') + + t = Thread(target=del_local) + t.start() + t.join() + + def test_slot_and_type_attributes(self): + a = A(Obj()) + a.initialized = 1 + self.assertEqual(a.initialized, 1) + + # The slot is shared + def demonstrate_slots_shared(): + self.assertEqual(a.initialized, 1) + a.initialized = 2 + + greenlet = Thread(target=demonstrate_slots_shared) + greenlet.start() + greenlet.join() + + self.assertEqual(a.initialized, 2) + + # The slot overrides dict values + a.__dict__['initialized'] = 42 # pylint:disable=unsupported-assignment-operation + self.assertEqual(a.initialized, 2) + + # Deleting the slot deletes the slot, but not the dict + del a.initialized + self.assertFalse(hasattr(a, 'initialized')) + self.assertIn('initialized', a.__dict__) + + # We can delete the 'path' ivar + # and fall back to the type + del a.path + self.assertEqual(a.path, '') + + with self.assertRaises(AttributeError): + del a.path + + # A read property calls get + self.assertEqual(a.read_property, 42) + a.read_property = 1 + self.assertEqual(a.read_property, 1) + self.assertIsInstance(A.read_property, ReadProperty) + + # Type attributes can be read + self.assertEqual(a.type_path, 'MyPath') + self.assertNotIn('type_path', a.__dict__) + + # and replaced in the dict + a.type_path = 'Local' + self.assertEqual(a.type_path, 'Local') + self.assertIn('type_path', a.__dict__) + + def test_attribute_error(self): + # pylint:disable=attribute-defined-outside-init + a = A(Obj()) + with self.assertRaises(AttributeError): + getattr(a, 'fizz_buzz') + + def set_fizz_buzz(): + a.fizz_buzz = 1 + + greenlet = Thread(target=set_fizz_buzz) + greenlet.start() + greenlet.join() + + with self.assertRaises(AttributeError): + getattr(a, 'fizz_buzz') + + def test_getattr_called(self): + getter = WithGetattr() + self.assertEqual(42, getter.foo) + getter.foo = 'baz' + self.assertEqual('baz', getter.foo) + + + def test_copy(self): + a = A(Obj()) + a.path = '123' + a.obj.echo = 'test' + b = copy(a) + + # Copy makes a shallow copy. Meaning that the attribute path + # has to be independent in the original and the copied object because the + # value is a string, but the attribute obj should be just reference to + # the instance of the class Obj + + self.assertEqual(a.path, b.path, 'The values in the two objects must be equal') + self.assertEqual(a.obj, b.obj, 'The values must be equal') + + b.path = '321' + self.assertNotEqual(a.path, b.path, 'The values in the two objects must be different') + + a.obj.echo = "works" + self.assertEqual(a.obj, b.obj, 'The values must be equal') + + def test_copy_no_subclass(self): + + a = local() + setattr(a, 'thing', 42) + b = copy(a) + self.assertEqual(b.thing, 42) + self.assertIsNot(a.__dict__, b.__dict__) + + def test_objects(self): + # Test which failed in the eventlet?! + + a = A({}) + a.path = '123' + b = A({'one': 2}) + b.path = '123' + self.assertEqual(a.path, b.path, 'The values in the two objects must be equal') + + b.path = '321' + + self.assertNotEqual(a.path, b.path, 'The values in the two objects must be different') + + def test_class_attr(self, kind=MyLocal): + mylocal = kind() + self.assertEqual(42, mylocal.CLASS_PROP) + + mylocal.CLASS_PROP = 1 + self.assertEqual(1, mylocal.CLASS_PROP) + self.assertEqual(mylocal.__dict__['CLASS_PROP'], 1) + + del mylocal.CLASS_PROP + self.assertEqual(42, mylocal.CLASS_PROP) + + self.assertIs(mylocal, mylocal.desc) + + def test_class_attr_subclass(self): + self.test_class_attr(kind=MyLocalSubclass) + + def test_locals_collected_when_greenlet_dead_but_still_referenced(self): + # https://github.com/gevent/gevent/issues/387 + import gevent + + my_local = MyLocal() + my_local.sentinel = None + greentest.gc_collect_if_needed() + + del created_sentinels[:] + del deleted_sentinels[:] + + def demonstrate_my_local(): + # Get the important parts + getattr(my_local, 'sentinel') + + # Create and reference greenlets + greenlets = [Thread(target=demonstrate_my_local) for _ in range(5)] + for t in greenlets: + t.start() + gevent.sleep() + + self.assertEqual(len(created_sentinels), len(greenlets)) + + for g in greenlets: + assert not g.is_alive() + gevent.sleep() # let the callbacks run + greentest.gc_collect_if_needed() + + # The sentinels should be gone too + self.assertEqual(len(deleted_sentinels), len(greenlets)) + + @greentest.skipOnLibuvOnPyPyOnWin("GC makes this non-deterministic, especially on Windows") + def test_locals_collected_when_unreferenced_even_in_running_greenlet(self): + # In fact only on Windows do we see GC being an issue; + # pypy2 5.0 on macos and travis don't have a problem. + # https://github.com/gevent/gevent/issues/981 + import gevent + import gc + gc.collect() + + count = 1000 + + running_greenlet = None + + def demonstrate_my_local(): + for _ in range(1000): + x = MyLocal() + self.assertIsNotNone(x.sentinel) + x = None + + gc.collect() + gc.collect() + + self.assertEqual(count, len(created_sentinels)) + # They're all dead, even though this greenlet is + # still running + self.assertEqual(count, len(deleted_sentinels)) + + # The links were removed as well. + self.assertFalse(running_greenlet.has_links()) + + + running_greenlet = gevent.spawn(demonstrate_my_local) + gevent.sleep() + running_greenlet.join() + + self.assertEqual(count, len(deleted_sentinels)) + + @greentest.ignores_leakcheck + def test_local_dicts_for_greenlet(self): + import gevent + from gevent.local import all_local_dicts_for_greenlet + + class MyGreenlet(gevent.Greenlet): + results = None + id_x = None + def _run(self): # pylint:disable=method-hidden + x = local() + x.foo = 42 + self.id_x = id(x) + self.results = all_local_dicts_for_greenlet(self) + + g = MyGreenlet() + g.start() + g.join() + self.assertTrue(g.successful, g) + self.assertEqual(g.results, + [((local, g.id_x), {'foo': 42})]) + + def test_local_with_abc(self): + # an ABC (or generally any non-exact-type) in the MRO doesn't + # break things. See https://github.com/gevent/gevent/issues/1201 + + x = LocalWithABC() + x.d = {'a': 1} + self.assertEqual({'a': 1}, x.d) + # The ABC part works + self.assertIn('a', x.d) + self.assertEqual(['a'], list(x.keys())) + + def test_local_with_staticmethod(self): + x = LocalWithStaticMethod() + self.assertEqual(42, x.a_staticmethod()) + + def test_local_with_classmethod(self): + x = LocalWithClassMethod() + self.assertIs(LocalWithClassMethod, x.a_classmethod()) + +try: + from zope import interface +except ImportError: + interface = None + +@greentest.skipIf(interface is None, "Needs zope.interface") +class TestLocalInterface(greentest.TestCase): + __timeout__ = None + + @greentest.ignores_leakcheck + def test_provides(self): + # https://github.com/gevent/gevent/issues/1122 + + # pylint:disable=inherit-non-class + class IFoo(interface.Interface): + pass + + @interface.implementer(IFoo) + class Base(object): + pass + + class Derived(Base, local): + pass + + d = Derived() + p = list(interface.providedBy(d)) + self.assertEqual([IFoo], p) + + + +@greentest.skipOnPurePython("Needs C extension") +class TestCExt(greentest.TestCase): # pragma: no cover + + def test_c_extension(self): + self.assertEqual(local.__module__, + 'gevent._local') + +@greentest.skipWithCExtensions("Needs pure-python") +class TestPure(greentest.TestCase): + + def test_extension(self): + self.assertEqual(local.__module__, + 'gevent.local') + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__loop_callback.py b/src/gevent/tests/test__loop_callback.py new file mode 100644 index 0000000..39a2f13 --- /dev/null +++ b/src/gevent/tests/test__loop_callback.py @@ -0,0 +1,13 @@ +from gevent.core import loop + +count = 0 + + +def incr(): + global count + count += 1 + +loop = loop() +loop.run_callback(incr) +loop.run() +assert count == 1, count diff --git a/src/gevent/tests/test__makefile_ref.py b/src/gevent/tests/test__makefile_ref.py new file mode 100644 index 0000000..36ac4e8 --- /dev/null +++ b/src/gevent/tests/test__makefile_ref.py @@ -0,0 +1,516 @@ +from __future__ import print_function +import os +from gevent import monkey; monkey.patch_all() +import socket +import ssl +import threading +import unittest +import errno +import weakref + + +import gevent.testing as greentest + + +dirname = os.path.dirname(os.path.abspath(__file__)) +certfile = os.path.join(dirname, '2_7_keycert.pem') +pid = os.getpid() + +PY3 = greentest.PY3 +PYPY = greentest.PYPY +CPYTHON = not PYPY +PY2 = not PY3 +fd_types = int +if PY3: + long = int +fd_types = (int, long) +WIN = greentest.WIN + +from gevent.testing import get_open_files +try: + import psutil +except ImportError: + psutil = None + + +class Test(greentest.TestCase): + + extra_allowed_open_states = () + + def tearDown(self): + self.extra_allowed_open_states = () + super(Test, self).tearDown() + + def assert_raises_EBADF(self, func): + try: + result = func() + except (socket.error, OSError) as ex: + # Windows/Py3 raises "OSError: [WinError 10038]" + if ex.args[0] == errno.EBADF: + return + if WIN and ex.args[0] == 10038: + return + raise + raise AssertionError('NOT RAISED EBADF: %r() returned %r' % (func, result)) + + def assert_fd_open(self, fileno): + assert isinstance(fileno, fd_types) + open_files = get_open_files() + if fileno not in open_files: + raise AssertionError('%r is not open:\n%s' % (fileno, open_files['data'])) + + def assert_fd_closed(self, fileno): + assert isinstance(fileno, fd_types), repr(fileno) + assert fileno > 0, fileno + open_files = get_open_files() + if fileno in open_files: + raise AssertionError('%r is not closed:\n%s' % (fileno, open_files['data'])) + + def _assert_sock_open(self, sock): + # requires the psutil output + open_files = get_open_files() + sockname = sock.getsockname() + for x in open_files['data']: + if getattr(x, 'laddr', None) == sockname: + assert x.status in (psutil.CONN_LISTEN, psutil.CONN_ESTABLISHED) + self.extra_allowed_open_states, x.status + return + raise AssertionError("%r is not open:\n%s" % (sock, open_files['data'])) + + def assert_open(self, sock, *rest): + if isinstance(sock, fd_types): + if not WIN: + self.assert_fd_open(sock) + else: + fileno = sock.fileno() + assert isinstance(fileno, fd_types), fileno + sockname = sock.getsockname() + assert isinstance(sockname, tuple), sockname + if not WIN: + self.assert_fd_open(fileno) + else: + self._assert_sock_open(sock) + if rest: + self.assert_open(rest[0], *rest[1:]) + + def assert_closed(self, sock, *rest): + if isinstance(sock, fd_types): + self.assert_fd_closed(sock) + else: + # Under Python3, the socket module returns -1 for a fileno + # of a closed socket; under Py2 it raises + if PY3: + self.assertEqual(sock.fileno(), -1) + else: + self.assert_raises_EBADF(sock.fileno) + self.assert_raises_EBADF(sock.getsockname) + self.assert_raises_EBADF(sock.accept) + if rest: + self.assert_closed(rest[0], *rest[1:]) + + def make_open_socket(self): + s = socket.socket() + s.bind(('127.0.0.1', 0)) + self._close_on_teardown(s) + if WIN or greentest.LINUX: + # Windows and linux (with psutil) doesn't show as open until + # we call listen (linux with lsof accepts either) + s.listen(1) + self.assert_open(s, s.fileno()) + return s + + if CPYTHON and PY2: + # Keeping raw sockets alive keeps SSL sockets + # from being closed too, at least on CPython2, so we + # need to use weakrefs. + + # In contrast, on PyPy, *only* having a weakref lets the + # original socket die and leak + + def _close_on_teardown(self, resource): + self.close_on_teardown.append(weakref.ref(resource)) + return resource + + def _tearDownCloseOnTearDown(self): + self.close_on_teardown = [r() for r in self.close_on_teardown if r() is not None] + super(Test, self)._tearDownCloseOnTearDown() + +# Sometimes its this one, sometimes it's test_ssl. No clue why or how. +@greentest.skipOnAppVeyor("This sometimes times out for no apparent reason.") +class TestSocket(Test): + + def test_simple_close(self): + s = self.make_open_socket() + fileno = s.fileno() + s.close() + self.assert_closed(s, fileno) + + def test_makefile1(self): + s = self.make_open_socket() + fileno = s.fileno() + f = s.makefile() + self.assert_open(s, fileno) + s.close() + # Under python 2, this closes socket wrapper object but not the file descriptor; + # under python 3, both stay open + if PY3: + self.assert_open(s, fileno) + else: + self.assert_closed(s) + self.assert_open(fileno) + f.close() + self.assert_closed(s) + self.assert_closed(fileno) + + def test_makefile2(self): + s = self.make_open_socket() + fileno = s.fileno() + self.assert_open(s, fileno) + f = s.makefile() + self.assert_open(s) + self.assert_open(s, fileno) + f.close() + # closing fileobject does not close the socket + self.assert_open(s, fileno) + s.close() + self.assert_closed(s, fileno) + + def test_server_simple(self): + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen(1) + + connector = socket.socket() + self._close_on_teardown(connector) + + def connect(): + connector.connect(('127.0.0.1', port)) + + t = threading.Thread(target=connect) + t.start() + + try: + client_socket, _addr = listener.accept() + fileno = client_socket.fileno() + self.assert_open(client_socket, fileno) + client_socket.close() + self.assert_closed(client_socket) + finally: + t.join() + listener.close() + connector.close() + + def test_server_makefile1(self): + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen(1) + + connector = socket.socket() + self._close_on_teardown(connector) + + def connect(): + connector.connect(('127.0.0.1', port)) + + t = threading.Thread(target=connect) + t.start() + + try: + client_socket, _addr = listener.accept() + fileno = client_socket.fileno() + f = client_socket.makefile() + self.assert_open(client_socket, fileno) + client_socket.close() + # Under python 2, this closes socket wrapper object but not the file descriptor; + # under python 3, both stay open + if PY3: + self.assert_open(client_socket, fileno) + else: + self.assert_closed(client_socket) + self.assert_open(fileno) + f.close() + self.assert_closed(client_socket, fileno) + finally: + t.join() + listener.close() + connector.close() + + def test_server_makefile2(self): + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen(1) + + connector = socket.socket() + self._close_on_teardown(connector) + + def connect(): + connector.connect(('127.0.0.1', port)) + + t = threading.Thread(target=connect) + t.start() + + try: + client_socket, _addr = listener.accept() + fileno = client_socket.fileno() + f = client_socket.makefile() + self.assert_open(client_socket, fileno) + # closing fileobject does not close the socket + f.close() + self.assert_open(client_socket, fileno) + client_socket.close() + self.assert_closed(client_socket, fileno) + finally: + t.join() + listener.close() + connector.close() + + +@greentest.skipOnAppVeyor("This sometimes times out for no apparent reason.") +class TestSSL(Test): + + def _ssl_connect_task(self, connector, port): + connector.connect(('127.0.0.1', port)) + try: + # Note: We get ResourceWarning about 'x' + # on Python 3 if we don't join the spawned thread + x = ssl.wrap_socket(connector) + except socket.error: + # Observed on Windows with PyPy2 5.9.0 and libuv: + # if we don't switch in a timely enough fashion, + # the server side runs ahead of us and closes + # our socket first, so this fails. + pass + else: + #self._close_on_teardown(x) + x.close() + + def _make_ssl_connect_task(self, connector, port): + t = threading.Thread(target=self._ssl_connect_task, args=(connector, port)) + t.daemon = True + return t + + def __cleanup(self, task, *sockets): + # workaround for test_server_makefile1, test_server_makefile2, + # test_server_simple, test_serverssl_makefile1. + + # On PyPy on Linux, it is important to join the SSL Connect + # Task FIRST, before closing the sockets. If we do it after + # (which makes more sense) we hang. It's not clear why, except + # that it has something to do with context switches. Inserting a call to + # gevent.sleep(0.1) instead of joining the task has the same + # effect. If the previous tests hang, then later tests can fail with + # SSLError: unknown alert type. + + # XXX: Why do those two things happen? + + # On PyPy on macOS, we don't have that problem and can use the + # more logical order. + + task.join() + for s in sockets: + s.close() + + del sockets + del task + + def test_simple_close(self): + s = self.make_open_socket() + fileno = s.fileno() + s = ssl.wrap_socket(s) + self._close_on_teardown(s) + fileno = s.fileno() + self.assert_open(s, fileno) + s.close() + self.assert_closed(s, fileno) + + def test_makefile1(self): + raw_s = self.make_open_socket() + s = ssl.wrap_socket(raw_s) + + self._close_on_teardown(s) + fileno = s.fileno() + self.assert_open(s, fileno) + f = s.makefile() + self.assert_open(s, fileno) + s.close() + self.assert_open(s, fileno) + f.close() + raw_s.close() + self.assert_closed(s, fileno) + + + def test_makefile2(self): + s = self.make_open_socket() + fileno = s.fileno() + + s = ssl.wrap_socket(s) + self._close_on_teardown(s) + fileno = s.fileno() + self.assert_open(s, fileno) + f = s.makefile() + self.assert_open(s, fileno) + f.close() + # closing fileobject does not close the socket + self.assert_open(s, fileno) + s.close() + self.assert_closed(s, fileno) + + def test_server_simple(self): + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen(1) + + connector = socket.socket() + self._close_on_teardown(connector) + + t = self._make_ssl_connect_task(connector, port) + t.start() + + try: + client_socket, _addr = listener.accept() + self._close_on_teardown(client_socket.close) + client_socket = ssl.wrap_socket(client_socket, keyfile=certfile, certfile=certfile, server_side=True) + self._close_on_teardown(client_socket) + fileno = client_socket.fileno() + self.assert_open(client_socket, fileno) + client_socket.close() + self.assert_closed(client_socket, fileno) + finally: + self.__cleanup(t, listener, connector) + + def test_server_makefile1(self): + listener = socket.socket() + self._close_on_teardown(listener) + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen(1) + + + connector = socket.socket() + self._close_on_teardown(connector) + + t = self._make_ssl_connect_task(connector, port) + t.start() + + try: + client_socket, _addr = listener.accept() + self._close_on_teardown(client_socket.close) # hard ref + client_socket = ssl.wrap_socket(client_socket, keyfile=certfile, certfile=certfile, server_side=True) + self._close_on_teardown(client_socket) + fileno = client_socket.fileno() + self.assert_open(client_socket, fileno) + f = client_socket.makefile() + self.assert_open(client_socket, fileno) + client_socket.close() + self.assert_open(client_socket, fileno) + f.close() + self.assert_closed(client_socket, fileno) + finally: + self.__cleanup(t, listener, connector) + + + def test_server_makefile2(self): + listener = socket.socket() + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen(1) + + connector = socket.socket() + self._close_on_teardown(connector) + + t = self._make_ssl_connect_task(connector, port) + t.start() + + try: + client_socket, _addr = listener.accept() + self._close_on_teardown(client_socket) + client_socket = ssl.wrap_socket(client_socket, keyfile=certfile, certfile=certfile, server_side=True) + self._close_on_teardown(client_socket) + fileno = client_socket.fileno() + self.assert_open(client_socket, fileno) + f = client_socket.makefile() + self.assert_open(client_socket, fileno) + # Closing fileobject does not close SSLObject + f.close() + self.assert_open(client_socket, fileno) + client_socket.close() + self.assert_closed(client_socket, fileno) + finally: + self.__cleanup(t, connector, listener, client_socket) + + def test_serverssl_makefile1(self): + listener = socket.socket() + fileno = listener.fileno() + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen(1) + self._close_on_teardown(listener) + listener = ssl.wrap_socket(listener, keyfile=certfile, certfile=certfile) + + connector = socket.socket() + self._close_on_teardown(connector) + + t = self._make_ssl_connect_task(connector, port) + t.start() + + try: + client_socket, _addr = listener.accept() + fileno = client_socket.fileno() + self.assert_open(client_socket, fileno) + f = client_socket.makefile() + self.assert_open(client_socket, fileno) + client_socket.close() + self.assert_open(client_socket, fileno) + f.close() + self.assert_closed(client_socket, fileno) + finally: + self.__cleanup(t, listener, connector) + + @greentest.skipIf(greentest.RUNNING_ON_TRAVIS and greentest.PY37 and greentest.LIBUV, + "Often segfaults, cannot reproduce locally. " + "Not too worried about this before Python 3.7rc1. " + "https://travis-ci.org/gevent/gevent/jobs/327357684") + def test_serverssl_makefile2(self): + listener = socket.socket() + self._close_on_teardown(listener) + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen(1) + listener = ssl.wrap_socket(listener, keyfile=certfile, certfile=certfile) + + connector = socket.socket() + + def connect(): + connector.connect(('127.0.0.1', port)) + s = ssl.wrap_socket(connector) + s.sendall(b'test_serverssl_makefile2') + s.close() + connector.close() + + t = threading.Thread(target=connect) + t.daemon = True + t.start() + + try: + client_socket, _addr = listener.accept() + fileno = client_socket.fileno() + self.assert_open(client_socket, fileno) + f = client_socket.makefile() + self.assert_open(client_socket, fileno) + self.assertEqual(f.read(), 'test_serverssl_makefile2') + self.assertEqual(f.read(), '') + f.close() + if WIN and psutil: + # Hmm? + self.extra_allowed_open_states = (psutil.CONN_CLOSE_WAIT,) + self.assert_open(client_socket, fileno) + client_socket.close() + self.assert_closed(client_socket, fileno) + finally: + self.__cleanup(t, listener) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__memleak.py b/src/gevent/tests/test__memleak.py new file mode 100644 index 0000000..7a191a4 --- /dev/null +++ b/src/gevent/tests/test__memleak.py @@ -0,0 +1,56 @@ +import sys +import unittest + +from gevent.testing import TestCase, main +import gevent +from gevent.timeout import Timeout + +@unittest.skipUnless( + hasattr(sys, 'gettotalrefcount'), + "Needs debug build" +) +class TestQueue(TestCase): # pragma: no cover + # pylint:disable=bare-except,no-member + + def test(self): + result = '' + try: + Timeout.start_new(0.01) + gevent.sleep(1) + raise AssertionError('must raise Timeout') + except KeyboardInterrupt: + raise + except: + pass + + result += '%s ' % sys.gettotalrefcount() + + try: + Timeout.start_new(0.01) + gevent.sleep(1) + raise AssertionError('must raise Timeout') + except KeyboardInterrupt: + raise + except: + pass + + result += '%s ' % sys.gettotalrefcount() + + try: + Timeout.start_new(0.01) + gevent.sleep(1) + raise AssertionError('must raise Timeout') + except KeyboardInterrupt: + raise + except: + pass + + result += '%s' % sys.gettotalrefcount() + + _, b, c = result.split() + assert b == c, 'total refcount mismatch: %s' % result + + + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__monkey.py b/src/gevent/tests/test__monkey.py new file mode 100644 index 0000000..9c33d7f --- /dev/null +++ b/src/gevent/tests/test__monkey.py @@ -0,0 +1,151 @@ +from subprocess import Popen + +from gevent import monkey +monkey.patch_all() + +import sys +import unittest +from gevent.testing.testcase import SubscriberCleanupMixin + +class TestMonkey(SubscriberCleanupMixin, unittest.TestCase): + + maxDiff = None + + def test_time(self): + import time + from gevent import time as gtime + self.assertIs(time.sleep, gtime.sleep) + + def test_thread(self): + try: + import thread + except ImportError: + import _thread as thread + import threading + + from gevent import thread as gthread + self.assertIs(thread.start_new_thread, gthread.start_new_thread) + self.assertIs(threading._start_new_thread, gthread.start_new_thread) + + # Event patched by default + self.assertTrue(monkey.is_object_patched('threading', 'Event')) + + if sys.version_info[0] == 2: + from gevent import threading as gthreading + from gevent.event import Event as GEvent + self.assertIs(threading._sleep, gthreading._sleep) + self.assertTrue(monkey.is_object_patched('threading', '_Event')) + self.assertIs(threading._Event, GEvent) + + def test_socket(self): + import socket + from gevent import socket as gevent_socket + self.assertIs(socket.create_connection, gevent_socket.create_connection) + + def test_os(self): + import os + import types + from gevent import os as gos + for name in ('fork', 'forkpty'): + if hasattr(os, name): + attr = getattr(os, name) + self.assertNotIn('built-in', repr(attr)) + self.assertNotIsInstance(attr, types.BuiltinFunctionType) + self.assertIsInstance(attr, types.FunctionType) + self.assertIs(attr, getattr(gos, name)) + + def test_saved(self): + self.assertTrue(monkey.saved) + for modname in monkey.saved: + self.assertTrue(monkey.is_module_patched(modname)) + + for objname in monkey.saved[modname]: + self.assertTrue(monkey.is_object_patched(modname, objname)) + + def test_patch_subprocess_twice(self): + self.assertNotIn('gevent', repr(Popen)) + self.assertIs(Popen, monkey.get_original('subprocess', 'Popen')) + monkey.patch_subprocess() + self.assertIs(Popen, monkey.get_original('subprocess', 'Popen')) + + def test_patch_twice_warnings_events(self): + import warnings + from zope.interface import verify + + orig_saved = {} + for k, v in monkey.saved.items(): + orig_saved[k] = v.copy() + + from gevent import events + all_events = [] + events.subscribers.append(all_events.append) + + def veto(event): + if isinstance(event, events.GeventWillPatchModuleEvent) and event.module_name == 'ssl': + raise events.DoNotPatch + + events.subscribers.append(veto) + + with warnings.catch_warnings(record=True) as issued_warnings: + # Patch again, triggering three warnings, one for os=False/signal=True, + # one for repeated monkey-patching, one for patching after ssl (on python >= 2.7.9) + monkey.patch_all(os=False, extra_kwarg=42) + self.assertGreaterEqual(len(issued_warnings), 2) + self.assertIn('SIGCHLD', str(issued_warnings[-1].message)) + self.assertIn('more than once', str(issued_warnings[0].message)) + + # Patching with the exact same argument doesn't issue a second warning. + # in fact, it doesn't do anything + del issued_warnings[:] + monkey.patch_all(os=False) + orig_saved['_gevent_saved_patch_all'] = monkey.saved['_gevent_saved_patch_all'] + + self.assertFalse(issued_warnings) + + # Make sure that re-patching did not change the monkey.saved + # attribute, overwriting the original functions. + if 'logging' in monkey.saved and 'logging' not in orig_saved: + # some part of the warning or unittest machinery imports logging + orig_saved['logging'] = monkey.saved['logging'] + self.assertEqual(orig_saved, monkey.saved) + + # Make sure some problematic attributes stayed correct. + # NOTE: This was only a problem if threading was not previously imported. + for k, v in monkey.saved['threading'].items(): + self.assertNotIn('gevent', str(v)) + + self.assertIsInstance(all_events[0], events.GeventWillPatchAllEvent) + self.assertEqual({'extra_kwarg': 42}, all_events[0].patch_all_kwargs) + verify.verifyObject(events.IGeventWillPatchAllEvent, all_events[0]) + + self.assertIsInstance(all_events[1], events.GeventWillPatchModuleEvent) + verify.verifyObject(events.IGeventWillPatchModuleEvent, all_events[1]) + + self.assertIsInstance(all_events[2], events.GeventDidPatchModuleEvent) + verify.verifyObject(events.IGeventWillPatchModuleEvent, all_events[1]) + + self.assertIsInstance(all_events[-2], events.GeventDidPatchBuiltinModulesEvent) + verify.verifyObject(events.IGeventDidPatchBuiltinModulesEvent, all_events[-2]) + + self.assertIsInstance(all_events[-1], events.GeventDidPatchAllEvent) + verify.verifyObject(events.IGeventDidPatchAllEvent, all_events[-1]) + + for e in all_events: + self.assertFalse(isinstance(e, events.GeventDidPatchModuleEvent) + and e.module_name == 'ssl') + + def test_patch_queue(self): + try: + import queue + except ImportError: + # Python 2 called this Queue. Note that having + # python-future installed gives us a queue module on + # Python 2 as well. + queue = None + if not hasattr(queue, 'SimpleQueue'): + raise unittest.SkipTest("Needs SimpleQueue") + # pylint:disable=no-member + self.assertIs(queue.SimpleQueue, queue._PySimpleQueue) + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__monkey_builtins_future.py b/src/gevent/tests/test__monkey_builtins_future.py new file mode 100644 index 0000000..599253d --- /dev/null +++ b/src/gevent/tests/test__monkey_builtins_future.py @@ -0,0 +1,16 @@ +# Under Python 2, if the `future` module is installed, we get +# a `builtins` module, which mimics the `builtins` module from +# Python 3, but does not have the __import__ and some other functions. +# Make sure we can still run in that case. +import sys +try: + # fake out a "broken" builtins module + import builtins +except ImportError: + class builtins(object): + pass + sys.modules['builtins'] = builtins() + +if not hasattr(builtins, '__import__'): + import gevent.monkey + gevent.monkey.patch_builtins() diff --git a/src/gevent/tests/test__monkey_hub_in_thread.py b/src/gevent/tests/test__monkey_hub_in_thread.py new file mode 100644 index 0000000..981ca6c --- /dev/null +++ b/src/gevent/tests/test__monkey_hub_in_thread.py @@ -0,0 +1,28 @@ +from gevent.monkey import patch_all +patch_all(thread=False) +from threading import Thread +import time + +# The first time we init the hub is in the native +# thread with time.sleep(), needing multiple +# threads at the same time. Note: this is very timing +# dependent. +# See #687 + + +def func(): + time.sleep() + + +def main(): + threads = [] + for _ in range(3): + th = Thread(target=func) + th.start() + threads.append(th) + for th in threads: + th.join() + + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__monkey_logging.py b/src/gevent/tests/test__monkey_logging.py new file mode 100644 index 0000000..9244ec6 --- /dev/null +++ b/src/gevent/tests/test__monkey_logging.py @@ -0,0 +1,41 @@ +# If the logging module is imported *before* monkey patching, +# the existing handlers are correctly monkey patched to use gevent locks +import logging +logging.basicConfig() + +import threading +import sys +PY2 = sys.version_info[0] == 2 + + +def _inner_lock(lock): + # The inner attribute changed between 2 and 3 + attr = getattr(lock, '_block' if not PY2 else '_RLock__block', None) + return attr + + +def checkLocks(kind, ignore_none=True): + handlers = logging._handlerList + assert handlers + + for weakref in handlers: + # In py26, these are actual handlers, not weakrefs + handler = weakref() if callable(weakref) else weakref + attr = _inner_lock(handler.lock) + if attr is None and ignore_none: + continue + assert isinstance(attr, kind), (handler.lock, attr, kind) + + attr = _inner_lock(logging._lock) + if attr is None and ignore_none: + return + assert isinstance(attr, kind) + +checkLocks(type(threading._allocate_lock())) + +import gevent.monkey +gevent.monkey.patch_all() + +import gevent.lock + +checkLocks(type(gevent.thread.allocate_lock()), ignore_none=False) diff --git a/src/gevent/tests/test__monkey_multiple_imports.py b/src/gevent/tests/test__monkey_multiple_imports.py new file mode 100644 index 0000000..576062e --- /dev/null +++ b/src/gevent/tests/test__monkey_multiple_imports.py @@ -0,0 +1,6 @@ +# https://github.com/gevent/gevent/issues/615 +# Under Python 3, with its use of importlib, +# if the monkey patch is done when the importlib import lock is held +# (e.g., during recursive imports) we could fail to release the lock. +# This is surprisingly common. +__import__('_import_import_patch') diff --git a/src/gevent/tests/test__monkey_queue.py b/src/gevent/tests/test__monkey_queue.py new file mode 100644 index 0000000..c07a256 --- /dev/null +++ b/src/gevent/tests/test__monkey_queue.py @@ -0,0 +1,331 @@ +# Some simple queue module tests, plus some failure conditions +# to ensure the Queue locks remain stable. +from gevent import monkey +monkey.patch_all() + +from gevent import queue as Queue +import threading +import time +import unittest + + +QUEUE_SIZE = 5 + +# A thread to run a function that unclogs a blocked Queue. +class _TriggerThread(threading.Thread): + def __init__(self, fn, args): + self.fn = fn + self.args = args + #self.startedEvent = threading.Event() + from gevent.event import Event + self.startedEvent = Event() + threading.Thread.__init__(self) + + def run(self): + # The sleep isn't necessary, but is intended to give the blocking + # function in the main thread a chance at actually blocking before + # we unclog it. But if the sleep is longer than the timeout-based + # tests wait in their blocking functions, those tests will fail. + # So we give them much longer timeout values compared to the + # sleep here (I aimed at 10 seconds for blocking functions -- + # they should never actually wait that long - they should make + # progress as soon as we call self.fn()). + time.sleep(0.01) + self.startedEvent.set() + self.fn(*self.args) + + +# Execute a function that blocks, and in a separate thread, a function that +# triggers the release. Returns the result of the blocking function. Caution: +# block_func must guarantee to block until trigger_func is called, and +# trigger_func must guarantee to change queue state so that block_func can make +# enough progress to return. In particular, a block_func that just raises an +# exception regardless of whether trigger_func is called will lead to +# timing-dependent sporadic failures, and one of those went rarely seen but +# undiagnosed for years. Now block_func must be unexceptional. If block_func +# is supposed to raise an exception, call do_exceptional_blocking_test() +# instead. + +class BlockingTestMixin(object): + + def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): + self.t = _TriggerThread(trigger_func, trigger_args) + self.t.start() + self.result = block_func(*block_args) + # If block_func returned before our thread made the call, we failed! + if not self.t.startedEvent.isSet(): + self.fail("blocking function '%r' appeared not to block" % + block_func) + self.t.join(10) # make sure the thread terminates + if self.t.isAlive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) + return self.result + + # Call this instead if block_func is supposed to raise an exception. + def do_exceptional_blocking_test(self, block_func, block_args, trigger_func, + trigger_args, expected_exception_class): + self.t = _TriggerThread(trigger_func, trigger_args) + self.t.start() + try: + with self.assertRaises(expected_exception_class): + block_func(*block_args) + finally: + self.t.join(10) # make sure the thread terminates + if self.t.isAlive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) + if not self.t.startedEvent.isSet(): + self.fail("trigger thread ended but event never set") + + +class BaseQueueTest(unittest.TestCase, BlockingTestMixin): + type2test = Queue.Queue + + def setUp(self): + self.cum = 0 + self.cumlock = threading.Lock() + + def simple_queue_test(self, q): + if not q.empty(): + raise RuntimeError("Call this function with an empty queue") + # I guess we better check things actually queue correctly a little :) + q.put(111) + q.put(333) + q.put(222) + q.put(444) + target_first_items = dict( + Queue=111, + LifoQueue=444, + PriorityQueue=111) + actual_first_item = (q.peek(), q.get()) + self.assertEqual(actual_first_item, + (target_first_items[q.__class__.__name__], + target_first_items[q.__class__.__name__]), + "q.peek() and q.get() are not equal!") + target_order = dict(Queue=[333, 222, 444], + LifoQueue=[222, 333, 111], + PriorityQueue=[222, 333, 444]) + actual_order = [q.get(), q.get(), q.get()] + self.assertEqual(actual_order, target_order[q.__class__.__name__], + "Didn't seem to queue the correct data!") + for i in range(QUEUE_SIZE-1): + q.put(i) + self.assertFalse(q.empty(), "Queue should not be empty") + self.assertFalse(q.full(), "Queue should not be full") + q.put(999) + self.assertTrue(q.full(), "Queue should be full") + try: + q.put(888, block=0) + self.fail("Didn't appear to block with a full queue") + except Queue.Full: + pass + try: + q.put(888, timeout=0.01) + self.fail("Didn't appear to time-out with a full queue") + except Queue.Full: + pass + self.assertEqual(q.qsize(), QUEUE_SIZE) + # Test a blocking put + self.do_blocking_test(q.put, (888,), q.get, ()) + self.do_blocking_test(q.put, (888, True, 10), q.get, ()) + # Empty it + for i in range(QUEUE_SIZE): + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + try: + q.get(block=0) + self.fail("Didn't appear to block with an empty queue") + except Queue.Empty: + pass + try: + q.get(timeout=0.01) + self.fail("Didn't appear to time-out with an empty queue") + except Queue.Empty: + pass + # Test a blocking get + self.do_blocking_test(q.get, (), q.put, ('empty',)) + self.do_blocking_test(q.get, (True, 10), q.put, ('empty',)) + + def worker(self, q): + while True: + x = q.get() + if x is None: + q.task_done() + return + #with self.cumlock: + self.cum += x + q.task_done() + + def queue_join_test(self, q): + self.cum = 0 + for i in (0, 1): + threading.Thread(target=self.worker, args=(q,)).start() + for i in range(100): + q.put(i) + q.join() + self.assertEqual(self.cum, sum(range(100)), + "q.join() did not block until all tasks were done") + for i in (0, 1): + q.put(None) # instruct the threads to close + q.join() # verify that you can join twice + + def test_queue_task_done(self): + # Test to make sure a queue task completed successfully. + q = Queue.JoinableQueue() # self.type2test() + # XXX the same test in subclasses + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + + def test_queue_join(self): + # Test that a queue join()s successfully, and before anything else + # (done twice for insurance). + q = Queue.JoinableQueue() # self.type2test() + # XXX the same test in subclass + self.queue_join_test(q) + self.queue_join_test(q) + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + + def test_queue_task_done_with_items(self): + # Passing items to the constructor allows for as + # many task_done calls. Joining before all the task done + # are called returns false + # XXX the same test in subclass + l = [1, 2, 3] + q = Queue.JoinableQueue(items=l) + for i in l: + self.assertFalse(q.join(timeout=0.001)) + self.assertEqual(i, q.get()) + q.task_done() + + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + self.assertTrue(q.join(timeout=0.001)) + + def test_simple_queue(self): + # Do it a couple of times on the same queue. + # Done twice to make sure works with same instance reused. + q = self.type2test(QUEUE_SIZE) + self.simple_queue_test(q) + self.simple_queue_test(q) + +class LifoQueueTest(BaseQueueTest): + type2test = Queue.LifoQueue + +class PriorityQueueTest(BaseQueueTest): + type2test = Queue.PriorityQueue + + def test__init(self): + item1 = (2, 'b') + item2 = (1, 'a') + q = self.type2test(items=[item1, item2]) + self.assertTupleEqual(item2, q.get_nowait()) + self.assertTupleEqual(item1, q.get_nowait()) + + +# A Queue subclass that can provoke failure at a moment's notice :) +class FailingQueueException(Exception): + pass + +class FailingQueue(Queue.Queue): + def __init__(self, *args): + self.fail_next_put = False + self.fail_next_get = False + Queue.Queue.__init__(self, *args) + def _put(self, item): + if self.fail_next_put: + self.fail_next_put = False + raise FailingQueueException("You Lose") + return Queue.Queue._put(self, item) + def _get(self): + if self.fail_next_get: + self.fail_next_get = False + raise FailingQueueException("You Lose") + return Queue.Queue._get(self) + +class FailingQueueTest(unittest.TestCase, BlockingTestMixin): + + def failing_queue_test(self, q): + if not q.empty(): + raise RuntimeError("Call this function with an empty queue") + for i in range(QUEUE_SIZE-1): + q.put(i) + # Test a failing non-blocking put. + q.fail_next_put = True + with self.assertRaises(FailingQueueException): + q.put("oops", block=0) + + q.fail_next_put = True + with self.assertRaises(FailingQueueException): + q.put("oops", timeout=0.1) + q.put(999) + self.assertTrue(q.full(), "Queue should be full") + # Test a failing blocking put + q.fail_next_put = True + with self.assertRaises(FailingQueueException): + self.do_blocking_test(q.put, (888,), q.get, ()) + + # Check the Queue isn't damaged. + # put failed, but get succeeded - re-add + q.put(999) + # Test a failing timeout put + q.fail_next_put = True + self.do_exceptional_blocking_test(q.put, (888, True, 10), q.get, (), + FailingQueueException) + # Check the Queue isn't damaged. + # put failed, but get succeeded - re-add + q.put(999) + self.assertTrue(q.full(), "Queue should be full") + q.get() + self.assertFalse(q.full(), "Queue should not be full") + q.put(999) + self.assertTrue(q.full(), "Queue should be full") + # Test a blocking put + self.do_blocking_test(q.put, (888,), q.get, ()) + # Empty it + for i in range(QUEUE_SIZE): + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + q.put("first") + q.fail_next_get = True + with self.assertRaises(FailingQueueException): + q.get() + + self.assertFalse(q.empty(), "Queue should not be empty") + q.fail_next_get = True + with self.assertRaises(FailingQueueException): + q.get(timeout=0.1) + self.assertFalse(q.empty(), "Queue should not be empty") + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + q.fail_next_get = True + self.do_exceptional_blocking_test(q.get, (), q.put, ('empty',), + FailingQueueException) + # put succeeded, but get failed. + self.assertFalse(q.empty(), "Queue should not be empty") + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + + def test_failing_queue(self): + # Test to make sure a queue is functioning correctly. + # Done twice to the same instance. + q = FailingQueue(QUEUE_SIZE) + self.failing_queue_test(q) + self.failing_queue_test(q) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/gevent/tests/test__monkey_scope.py b/src/gevent/tests/test__monkey_scope.py new file mode 100644 index 0000000..1a3cc67 --- /dev/null +++ b/src/gevent/tests/test__monkey_scope.py @@ -0,0 +1,61 @@ +import os +import os.path +import sys + +import unittest + +from subprocess import Popen +from subprocess import PIPE + +class TestRun(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.cwd = os.getcwd() + os.chdir(os.path.dirname(__file__)) + + def tearDown(self): + os.chdir(self.cwd) + + def _run(self, script): + env = os.environ.copy() + env['PYTHONWARNINGS'] = 'ignore' + args = [sys.executable, '-m', 'gevent.monkey', script, 'patched'] + p = Popen(args, stdout=PIPE, stderr=PIPE, env=env) + gout, gerr = p.communicate() + self.assertEqual(0, p.returncode, (gout, gerr)) + + args = [sys.executable, script, 'stdlib'] + p = Popen(args, stdout=PIPE, stderr=PIPE) + + pout, perr = p.communicate() + self.assertEqual(0, p.returncode, (pout, perr)) + + glines = gout.decode("utf-8").splitlines() + plines = pout.decode('utf-8').splitlines() + self.assertEqual(glines, plines) + self.assertEqual(gerr, perr) + + return glines, gerr + + def test_run_simple(self): + self._run(os.path.join('monkey_package', 'script.py')) + + def test_run_package(self): + # Run a __main__ inside a package. + lines, _ = self._run('monkey_package') + + self.assertTrue(lines[0].endswith('__main__.py'), lines[0]) + self.assertEqual(lines[1], '__main__') + + def test_issue_302(self): + lines, _ = self._run(os.path.join('monkey_package', 'issue302monkey.py')) + + self.assertEqual(lines[0], 'True') + lines[1] = lines[1].replace('\\', '/') # windows path + self.assertEqual(lines[1], 'monkey_package/issue302monkey.py') + self.assertEqual(lines[2], 'True', lines) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__monkey_selectors.py b/src/gevent/tests/test__monkey_selectors.py new file mode 100644 index 0000000..08babb2 --- /dev/null +++ b/src/gevent/tests/test__monkey_selectors.py @@ -0,0 +1,22 @@ + +import sys +import gevent.testing as greentest +try: + import selectors # Do this before the patch, just to force it +except ImportError: + pass +from gevent.monkey import patch_all +patch_all() + +if sys.platform != 'win32' and sys.version_info[:2] >= (3, 4): + + class TestSelectors(greentest.TestCase): + + def test_selectors_select_is_patched(self): + # https://github.com/gevent/gevent/issues/835 + _select = selectors.SelectSelector._select + self.assertTrue(hasattr(_select, '_gevent_monkey'), dir(_select)) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__monkey_sigchld.py b/src/gevent/tests/test__monkey_sigchld.py new file mode 100644 index 0000000..0385fd8 --- /dev/null +++ b/src/gevent/tests/test__monkey_sigchld.py @@ -0,0 +1,69 @@ +import errno +import os +import sys +#os.environ['GEVENT_NOWAITPID'] = 'True' + +import gevent +import gevent.monkey +gevent.monkey.patch_all() + +pid = None +awaiting_child = [] + + +def handle_sigchld(*_args): + # Make sure we can do a blocking operation + gevent.sleep() + # Signal completion + awaiting_child.pop() + # Raise an ignored error + raise TypeError("This should be ignored but printed") + +import signal +if hasattr(signal, 'SIGCHLD'): + assert signal.getsignal(signal.SIGCHLD) == signal.SIG_DFL + signal.signal(signal.SIGCHLD, handle_sigchld) + handler = signal.getsignal(signal.SIGCHLD) + assert signal.getsignal(signal.SIGCHLD) is handle_sigchld, handler + + if hasattr(os, 'forkpty'): + def forkpty(): + # For printing in errors + return os.forkpty()[0] + funcs = (os.fork, forkpty) + else: + funcs = (os.fork,) + + for func in funcs: + awaiting_child = [True] + pid = func() + if not pid: + # child + gevent.sleep(0.3) + sys.exit(0) + else: + timeout = gevent.Timeout(1) + try: + while awaiting_child: + gevent.sleep(0.01) + # We should now be able to waitpid() for an arbitrary child + wpid, status = os.waitpid(-1, os.WNOHANG) + if wpid != pid: + raise AssertionError("Failed to wait on a child pid forked with a function", + wpid, pid, func) + + # And a second call should raise ECHILD + try: + wpid, status = os.waitpid(-1, os.WNOHANG) + raise AssertionError("Should not be able to wait again") + except OSError as e: + assert e.errno == errno.ECHILD + except gevent.Timeout as t: + if timeout is not t: + raise + raise AssertionError("Failed to wait using", func) + finally: + timeout.close() + sys.exit(0) +else: + print("No SIGCHLD, not testing") diff --git a/src/gevent/tests/test__monkey_sigchld_2.py b/src/gevent/tests/test__monkey_sigchld_2.py new file mode 100644 index 0000000..e1cab8c --- /dev/null +++ b/src/gevent/tests/test__monkey_sigchld_2.py @@ -0,0 +1,51 @@ +# Mimics what gunicorn workers do: monkey patch in the child process +# and try to reset signal handlers to SIG_DFL. +# NOTE: This breaks again when gevent.subprocess is used, or any child +# watcher. +import os +import sys + +import signal + + +def handle(*_args): + if not pid: + # We only do this is the child so our + # parent's waitpid can get the status. + # This is the opposite of gunicorn. + os.waitpid(-1, os.WNOHANG) +# The signal watcher must be installed *before* monkey patching +if hasattr(signal, 'SIGCHLD'): + # On Python 2, the signal handler breaks the platform + # module, because it uses os.popen. pkg_resources uses the platform + # module. + # Cache that info. + import platform + platform.uname() + signal.signal(signal.SIGCHLD, handle) + + pid = os.fork() + + if pid: # parent + try: + _, stat = os.waitpid(pid, 0) + except OSError: + # Interrupted system call + _, stat = os.waitpid(pid, 0) + assert stat == 0, stat + else: + # Under Python 2, os.popen() directly uses the popen call, and + # popen's file uses the pclose() system call to + # wait for the child. If it's already waited on, + # it raises the same exception. + # Python 3 uses the subprocess module directly which doesn't + # have this problem. + import gevent.monkey + gevent.monkey.patch_all() + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + f = os.popen('true') + f.close() + + sys.exit(0) +else: + print("No SIGCHLD, not testing") diff --git a/src/gevent/tests/test__monkey_sigchld_3.py b/src/gevent/tests/test__monkey_sigchld_3.py new file mode 100644 index 0000000..100e6eb --- /dev/null +++ b/src/gevent/tests/test__monkey_sigchld_3.py @@ -0,0 +1,52 @@ +# Mimics what gunicorn workers do *if* the arbiter is also monkey-patched: +# After forking from the master monkey-patched process, the child +# resets signal handlers to SIG_DFL. If we then fork and watch *again*, +# we shouldn't hang. (Note that we carefully handle this so as not to break +# os.popen) +from __future__ import print_function +# Patch in the parent process. +import gevent.monkey +gevent.monkey.patch_all() + +from gevent import get_hub + +import os +import sys + +import signal +import subprocess + +def _waitpid(p): + try: + _, stat = os.waitpid(p, 0) + except OSError: + # Interrupted system call + _, stat = os.waitpid(p, 0) + assert stat == 0, stat + +if hasattr(signal, 'SIGCHLD'): + # Do what subprocess does and make sure we have the watcher + # in the parent + get_hub().loop.install_sigchld() + + + pid = os.fork() + + if pid: # parent + _waitpid(pid) + else: + # Child resets. + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + + # Go through subprocess because we expect it to automatically + # set up the waiting for us. + popen = subprocess.Popen([sys.executable, '-c', 'import sys'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen.stderr.read() + popen.stdout.read() + popen.wait() # This hangs if it doesn't. + popen.stderr.close() + popen.stdout.close() + sys.exit(0) +else: + print("No SIGCHLD, not testing") diff --git a/src/gevent/tests/test__monkey_ssl_warning.py b/src/gevent/tests/test__monkey_ssl_warning.py new file mode 100644 index 0000000..1e94667 --- /dev/null +++ b/src/gevent/tests/test__monkey_ssl_warning.py @@ -0,0 +1,34 @@ +import unittest +import warnings + +# This file should only have this one test in it +# because we have to be careful about our imports +# and because we need to be careful about our patching. + +class Test(unittest.TestCase): + + def test_with_pkg_resources(self): + # Issue 1108: Python 2, importing pkg_resources, + # as is done for namespace packages, imports ssl, + # leading to an unwanted SSL warning. + __import__('pkg_resources') + + from gevent import monkey + + self.assertFalse(monkey.saved) + + with warnings.catch_warnings(record=True) as issued_warnings: + warnings.simplefilter('always') + + monkey.patch_all() + monkey.patch_all() + + issued_warnings = [x for x in issued_warnings + if isinstance(x.message, monkey.MonkeyPatchWarning)] + + self.assertFalse(issued_warnings, [str(i) for i in issued_warnings]) + self.assertEqual(0, len(issued_warnings)) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__monkey_ssl_warning2.py b/src/gevent/tests/test__monkey_ssl_warning2.py new file mode 100644 index 0000000..c7c1239 --- /dev/null +++ b/src/gevent/tests/test__monkey_ssl_warning2.py @@ -0,0 +1,44 @@ +import unittest +import warnings +import sys + +# All supported python versions now provide SSLContext. +# We import it by name and subclass it here by name. +# compare with warning3.py +from ssl import SSLContext + +class MySubclass(SSLContext): + pass + +# This file should only have this one test in it +# because we have to be careful about our imports +# and because we need to be careful about our patching. + +class Test(unittest.TestCase): + + @unittest.skipIf(sys.version_info[:2] < (3, 6), + "Only on Python 3.6+") + def test_ssl_subclass_and_module_reference(self): + + from gevent import monkey + + self.assertFalse(monkey.saved) + + with warnings.catch_warnings(record=True) as issued_warnings: + warnings.simplefilter('always') + + monkey.patch_all() + monkey.patch_all() + + issued_warnings = [x for x in issued_warnings + if isinstance(x.message, monkey.MonkeyPatchWarning)] + + self.assertEqual(1, len(issued_warnings)) + message = issued_warnings[0].message + self.assertIn("Modules that had direct imports", str(message)) + self.assertIn("Subclasses (NOT patched)", str(message)) + + + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__monkey_ssl_warning3.py b/src/gevent/tests/test__monkey_ssl_warning3.py new file mode 100644 index 0000000..76b2a79 --- /dev/null +++ b/src/gevent/tests/test__monkey_ssl_warning3.py @@ -0,0 +1,47 @@ +import unittest +import warnings +import sys + +# All supported python versions now provide SSLContext. +# We subclass without importing by name. Compare with +# warning2.py +import ssl + +class MySubclass(ssl.SSLContext): + pass + +# This file should only have this one test in it +# because we have to be careful about our imports +# and because we need to be careful about our patching. + +class Test(unittest.TestCase): + + @unittest.skipIf(sys.version_info[:2] < (3, 6), + "Only on Python 3.6+") + def test_ssl_subclass_and_module_reference(self): + + from gevent import monkey + + self.assertFalse(monkey.saved) + + with warnings.catch_warnings(record=True) as issued_warnings: + warnings.simplefilter('always') + + monkey.patch_all() + monkey.patch_all() + + issued_warnings = [x for x in issued_warnings + if isinstance(x.message, monkey.MonkeyPatchWarning)] + + self.assertEqual(1, len(issued_warnings)) + message = str(issued_warnings[0].message) + + self.assertNotIn("Modules that had direct imports", message) + self.assertIn("Subclasses (NOT patched)", message) + # the gevent subclasses should not be in here. + self.assertNotIn('gevent.', message) + + + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__nondefaultloop.py b/src/gevent/tests/test__nondefaultloop.py new file mode 100644 index 0000000..489ff52 --- /dev/null +++ b/src/gevent/tests/test__nondefaultloop.py @@ -0,0 +1,12 @@ +# test for issue #210 +from gevent import core +from gevent.testing.util import alarm + + +alarm(1) + +log = [] +loop = core.loop(default=False) +loop.run_callback(log.append, 1) +loop.run() +assert log == [1], log diff --git a/src/gevent/tests/test__order.py b/src/gevent/tests/test__order.py new file mode 100644 index 0000000..83aa1c9 --- /dev/null +++ b/src/gevent/tests/test__order.py @@ -0,0 +1,61 @@ +import gevent +import gevent.testing as greentest +from gevent.testing.six import xrange + + +class appender(object): + + def __init__(self, lst, item): + self.lst = lst + self.item = item + + def __call__(self, *args): + self.lst.append(self.item) + + +class Test(greentest.TestCase): + + count = 2 + + def test_greenlet_link(self): + lst = [] + + # test that links are executed in the same order as they were added + g = gevent.spawn(lst.append, 0) + + for i in xrange(1, self.count): + g.link(appender(lst, i)) + g.join() + self.assertEqual(lst, list(range(self.count))) + + +class Test3(Test): + count = 3 + + +class Test4(Test): + count = 4 + + +class TestM(Test): + count = 1000 + + +class TestSleep0(greentest.TestCase): + + def test(self): + lst = [] + gevent.spawn(sleep0, lst, '1') + gevent.spawn(sleep0, lst, '2') + gevent.wait() + self.assertEqual(' '.join(lst), '1A 2A 1B 2B') + + +def sleep0(lst, param): + lst.append(param + 'A') + gevent.sleep(0) + lst.append(param + 'B') + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__os.py b/src/gevent/tests/test__os.py new file mode 100644 index 0000000..dacf2c4 --- /dev/null +++ b/src/gevent/tests/test__os.py @@ -0,0 +1,178 @@ +from __future__ import print_function, absolute_import, division + +import sys +from os import pipe + + +import gevent +from gevent import os +from gevent import Greenlet, joinall + +from gevent import testing as greentest +from gevent.testing import mock +from gevent.testing import six +from gevent.testing.skipping import skipOnLibuvOnPyPyOnWin + + +class TestOS_tp(greentest.TestCase): + + __timeout__ = greentest.LARGE_TIMEOUT + + def pipe(self): + return pipe() + + read = staticmethod(os.tp_read) + write = staticmethod(os.tp_write) + + @skipOnLibuvOnPyPyOnWin("Sometimes times out") + def _test_if_pipe_blocks(self, buffer_class): + r, w = self.pipe() + # set nbytes such that for sure it is > maximum pipe buffer + nbytes = 1000000 + block = b'x' * 4096 + buf = buffer_class(block) + # Lack of "nonlocal" keyword in Python 2.x: + bytesread = [0] + byteswritten = [0] + + def produce(): + while byteswritten[0] != nbytes: + bytesleft = nbytes - byteswritten[0] + byteswritten[0] += self.write(w, buf[:min(bytesleft, 4096)]) + + def consume(): + while bytesread[0] != nbytes: + bytesleft = nbytes - bytesread[0] + bytesread[0] += len(self.read(r, min(bytesleft, 4096))) + + producer = Greenlet(produce) + producer.start() + consumer = Greenlet(consume) + consumer.start_later(1) + # If patching was not succesful, the producer will have filled + # the pipe before the consumer starts, and would block the entire + # process. Therefore the next line would never finish. + joinall([producer, consumer]) + self.assertEqual(bytesread[0], nbytes) + self.assertEqual(bytesread[0], byteswritten[0]) + + if sys.version_info[0] < 3: + + def test_if_pipe_blocks_buffer(self): + self._test_if_pipe_blocks(six.builtins.buffer) + + if sys.version_info[:2] >= (2, 7): + + def test_if_pipe_blocks_memoryview(self): + self._test_if_pipe_blocks(six.builtins.memoryview) + + +@greentest.skipUnless(hasattr(os, 'make_nonblocking'), + "Only on POSIX") +class TestOS_nb(TestOS_tp): + + def read(self, fd, count): + return os.nb_read(fd, count) + + def write(self, fd, count): + return os.nb_write(fd, count) + + def pipe(self): + r, w = super(TestOS_nb, self).pipe() + os.make_nonblocking(r) + os.make_nonblocking(w) + return r, w + + def _make_ignored_oserror(self): + import errno + ignored_oserror = OSError() + ignored_oserror.errno = errno.EINTR + return ignored_oserror + + + def _check_hub_event_closed(self, mock_get_hub, fd, event): + mock_get_hub.assert_called_once_with() + hub = mock_get_hub.return_value + io = hub.loop.io + io.assert_called_once_with(fd, event) + + event = io.return_value + event.close.assert_called_once_with() + + def _test_event_closed_on_normal_io(self, nb_func, nb_arg, + mock_io, mock_get_hub, event): + mock_io.side_effect = [self._make_ignored_oserror(), 42] + + fd = 100 + result = nb_func(fd, nb_arg) + self.assertEqual(result, 42) + + self._check_hub_event_closed(mock_get_hub, fd, event) + + def _test_event_closed_on_io_error(self, nb_func, nb_arg, + mock_io, mock_get_hub, event): + mock_io.side_effect = [self._make_ignored_oserror(), ValueError()] + + fd = 100 + + with self.assertRaises(ValueError): + nb_func(fd, nb_arg) + + self._check_hub_event_closed(mock_get_hub, fd, event) + + @mock.patch('gevent.os.get_hub') + @mock.patch('gevent.os._write') + def test_event_closed_on_write(self, mock_write, mock_get_hub): + self._test_event_closed_on_normal_io(os.nb_write, b'buf', + mock_write, mock_get_hub, + 2) + + @mock.patch('gevent.os.get_hub') + @mock.patch('gevent.os._write') + def test_event_closed_on_write_error(self, mock_write, mock_get_hub): + self._test_event_closed_on_io_error(os.nb_write, b'buf', + mock_write, mock_get_hub, + 2) + + @mock.patch('gevent.os.get_hub') + @mock.patch('gevent.os._read') + def test_event_closed_on_read(self, mock_read, mock_get_hub): + self._test_event_closed_on_normal_io(os.nb_read, b'buf', + mock_read, mock_get_hub, + 1) + + @mock.patch('gevent.os.get_hub') + @mock.patch('gevent.os._read') + def test_event_closed_on_read_error(self, mock_read, mock_get_hub): + self._test_event_closed_on_io_error(os.nb_read, b'buf', + mock_read, mock_get_hub, + 1) + + +@greentest.skipUnless(hasattr(os, 'fork_and_watch'), + "Only on POSIX") +class TestForkAndWatch(greentest.TestCase): + + __timeout__ = greentest.LARGE_TIMEOUT + + def test_waitpid_all(self): + # Cover this specific case. + pid = os.fork_and_watch() + if pid: + os.waitpid(-1, 0) + # Can't assert on what the pid actually was, + # our testrunner may have spawned multiple children. + os._reap_children(0) # make the leakchecker happy + else: # pragma: no cover + gevent.sleep(2) + os._exit(0) + + def test_waitpid_wrong_neg(self): + self.assertRaises(OSError, os.waitpid, -2, 0) + + def test_waitpid_wrong_pos(self): + self.assertRaises(OSError, os.waitpid, 1, 0) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__pool.py b/src/gevent/tests/test__pool.py new file mode 100644 index 0000000..989ee9c --- /dev/null +++ b/src/gevent/tests/test__pool.py @@ -0,0 +1,603 @@ +from time import time +import gevent +import gevent.pool +from gevent.event import Event +from gevent.queue import Queue + +import gevent.testing as greentest +import gevent.testing.timing +import random +from gevent.testing import ExpectedException + +import unittest + + +class TestCoroutinePool(unittest.TestCase): + klass = gevent.pool.Pool + + def test_apply_async(self): + done = Event() + + def some_work(_): + done.set() + + pool = self.klass(2) + pool.apply_async(some_work, ('x', )) + done.wait() + + def test_apply(self): + value = 'return value' + + def some_work(): + return value + + pool = self.klass(2) + result = pool.apply(some_work) + self.assertEqual(value, result) + + def test_apply_raises(self): + pool = self.klass(1) + + def raiser(): + raise ExpectedException() + try: + pool.apply(raiser) + except ExpectedException: + pass + else: + self.fail("Should have raised ExpectedException") + # Don't let the metaclass automatically force any error + # that reaches the hub from a spawned greenlet to become + # fatal; that defeats the point of the test. + test_apply_raises.error_fatal = False + + def test_multiple_coros(self): + evt = Event() + results = [] + + def producer(): + gevent.sleep(0.001) + results.append('prod') + evt.set() + + def consumer(): + results.append('cons1') + evt.wait() + results.append('cons2') + + pool = self.klass(2) + done = pool.spawn(consumer) + pool.apply_async(producer) + done.get() + self.assertEqual(['cons1', 'prod', 'cons2'], results) + + def dont_test_timer_cancel(self): + timer_fired = [] + + def fire_timer(): + timer_fired.append(True) + + def some_work(): + gevent.timer(0, fire_timer) # pylint:disable=no-member + + pool = self.klass(2) + pool.apply(some_work) + gevent.sleep(0) + self.assertEqual(timer_fired, []) + + def test_reentrant(self): + pool = self.klass(1) + result = pool.apply(pool.apply, (lambda a: a + 1, (5, ))) + self.assertEqual(result, 6) + evt = Event() + pool.apply_async(evt.set) + evt.wait() + + @greentest.skipOnPyPy("Does not work on PyPy") # Why? + def test_stderr_raising(self): + # testing that really egregious errors in the error handling code + # (that prints tracebacks to stderr) don't cause the pool to lose + # any members + import sys + pool = self.klass(size=1) + + # we're going to do this by causing the traceback.print_exc in + # safe_apply to raise an exception and thus exit _main_loop + normal_err = sys.stderr + try: + sys.stderr = FakeFile() + waiter = pool.spawn(crash) + with gevent.Timeout(2): + self.assertRaises(RuntimeError, waiter.get) + # the pool should have something free at this point since the + # waiter returned + # pool.Pool change: if an exception is raised during execution of a link, + # the rest of the links are scheduled to be executed on the next hub iteration + # this introduces a delay in updating pool.sem which makes pool.free_count() report 0 + # therefore, sleep: + gevent.sleep(0) + self.assertEqual(pool.free_count(), 1) + # shouldn't block when trying to get + with gevent.Timeout.start_new(0.1): + pool.apply(gevent.sleep, (0, )) + finally: + sys.stderr = normal_err + pool.join() + + +def crash(*_args, **_kw): + raise RuntimeError("Whoa") + + +class FakeFile(object): + + def write(self, *_args): + raise RuntimeError('Whaaa') + + +class PoolBasicTests(greentest.TestCase): + klass = gevent.pool.Pool + + def test_execute_async(self): + p = self.klass(size=2) + self.assertEqual(p.free_count(), 2) + r = [] + + first = p.spawn(r.append, 1) + self.assertEqual(p.free_count(), 1) + first.get() + self.assertEqual(r, [1]) + gevent.sleep(0) + self.assertEqual(p.free_count(), 2) + + #Once the pool is exhausted, calling an execute forces a yield. + + p.apply_async(r.append, (2, )) + self.assertEqual(1, p.free_count()) + self.assertEqual(r, [1]) + + p.apply_async(r.append, (3, )) + self.assertEqual(0, p.free_count()) + self.assertEqual(r, [1]) + + p.apply_async(r.append, (4, )) + self.assertEqual(r, [1]) + gevent.sleep(0.01) + self.assertEqual(sorted(r), [1, 2, 3, 4]) + + def test_discard(self): + p = self.klass(size=1) + first = p.spawn(gevent.sleep, 1000) + p.discard(first) + first.kill() + self.assertFalse(first) + self.assertEqual(len(p), 0) + self.assertEqual(p._semaphore.counter, 1) + + def test_add_method(self): + p = self.klass(size=1) + first = gevent.spawn(gevent.sleep, 1000) + try: + second = gevent.spawn(gevent.sleep, 1000) + try: + self.assertEqual(p.free_count(), 1) + self.assertEqual(len(p), 0) + p.add(first) + self.assertEqual(p.free_count(), 0) + self.assertEqual(len(p), 1) + + with self.assertRaises(gevent.Timeout): + with gevent.Timeout(0.1): + p.add(second) + + self.assertEqual(p.free_count(), 0) + self.assertEqual(len(p), 1) + finally: + second.kill() + finally: + first.kill() + + @greentest.ignores_leakcheck + def test_add_method_non_blocking(self): + p = self.klass(size=1) + first = gevent.spawn(gevent.sleep, 1000) + try: + second = gevent.spawn(gevent.sleep, 1000) + try: + p.add(first) + with self.assertRaises(gevent.pool.PoolFull): + p.add(second, blocking=False) + finally: + second.kill() + finally: + first.kill() + + @greentest.ignores_leakcheck + def test_add_method_timeout(self): + p = self.klass(size=1) + first = gevent.spawn(gevent.sleep, 1000) + try: + second = gevent.spawn(gevent.sleep, 1000) + try: + p.add(first) + with self.assertRaises(gevent.pool.PoolFull): + p.add(second, timeout=0.100) + finally: + second.kill() + finally: + first.kill() + + @greentest.ignores_leakcheck + def test_start_method_timeout(self): + p = self.klass(size=1) + first = gevent.spawn(gevent.sleep, 1000) + try: + second = gevent.Greenlet(gevent.sleep, 1000) + try: + p.add(first) + with self.assertRaises(gevent.pool.PoolFull): + p.start(second, timeout=0.100) + finally: + second.kill() + finally: + first.kill() + + def test_apply(self): + p = self.klass() + result = p.apply(lambda a: ('foo', a), (1, )) + self.assertEqual(result, ('foo', 1)) + + def test_init_error(self): + self.switch_expected = False + self.assertRaises(ValueError, self.klass, -1) + +# +# tests from standard library test/test_multiprocessing.py + + +class TimingWrapper(object): + + def __init__(self, func): + self.func = func + self.elapsed = None + + def __call__(self, *args, **kwds): + t = time() + try: + return self.func(*args, **kwds) + finally: + self.elapsed = time() - t + + +def sqr(x, wait=0.0): + gevent.sleep(wait) + return x * x + + +def squared(x): + return x * x + + +def sqr_random_sleep(x): + gevent.sleep(random.random() * 0.1) + return x * x + + +def final_sleep(): + for i in range(3): + yield i + gevent.sleep(0.2) + + +TIMEOUT1, TIMEOUT2, TIMEOUT3 = 0.082, 0.035, 0.14 + + +SMALL_RANGE = 10 +LARGE_RANGE = 1000 + +if (greentest.PYPY and greentest.WIN) or greentest.RUN_LEAKCHECKS or greentest.RUN_COVERAGE: + # See comments in test__threadpool.py. + LARGE_RANGE = 25 +elif greentest.RUNNING_ON_CI or greentest.EXPECT_POOR_TIMER_RESOLUTION: + LARGE_RANGE = 100 + +class TestPool(greentest.TestCase): # pylint:disable=too-many-public-methods + __timeout__ = greentest.LARGE_TIMEOUT + size = 1 + + def setUp(self): + greentest.TestCase.setUp(self) + self.pool = gevent.pool.Pool(self.size) + + def cleanup(self): + self.pool.join() + + def test_apply(self): + papply = self.pool.apply + self.assertEqual(papply(sqr, (5,)), 25) + self.assertEqual(papply(sqr, (), {'x': 3}), 9) + + def test_map(self): + pmap = self.pool.map + self.assertEqual(pmap(sqr, range(SMALL_RANGE)), list(map(squared, range(SMALL_RANGE)))) + self.assertEqual(pmap(sqr, range(100)), list(map(squared, range(100)))) + + def test_async(self): + res = self.pool.apply_async(sqr, (7, TIMEOUT1,)) + get = TimingWrapper(res.get) + self.assertEqual(get(), 49) + self.assertTimeoutAlmostEqual(get.elapsed, TIMEOUT1, 1) + + def test_async_callback(self): + result = [] + res = self.pool.apply_async(sqr, (7, TIMEOUT1,), callback=result.append) + get = TimingWrapper(res.get) + self.assertEqual(get(), 49) + self.assertTimeoutAlmostEqual(get.elapsed, TIMEOUT1, 1) + gevent.sleep(0) # lets the callback run + self.assertEqual(result, [49]) + + def test_async_timeout(self): + res = self.pool.apply_async(sqr, (6, TIMEOUT2 + 0.2)) + get = TimingWrapper(res.get) + self.assertRaises(gevent.Timeout, get, timeout=TIMEOUT2) + self.assertTimeoutAlmostEqual(get.elapsed, TIMEOUT2, 1) + self.pool.join() + + def test_imap_list_small(self): + it = self.pool.imap(sqr, range(SMALL_RANGE)) + self.assertEqual(list(it), list(map(sqr, range(SMALL_RANGE)))) + + def test_imap_it_small(self): + it = self.pool.imap(sqr, range(SMALL_RANGE)) + for i in range(SMALL_RANGE): + self.assertEqual(next(it), i * i) + self.assertRaises(StopIteration, next, it) + + def test_imap_it_large(self): + it = self.pool.imap(sqr, range(LARGE_RANGE)) + for i in range(LARGE_RANGE): + self.assertEqual(next(it), i * i) + self.assertRaises(StopIteration, next, it) + + def test_imap_random(self): + it = self.pool.imap(sqr_random_sleep, range(SMALL_RANGE)) + self.assertEqual(list(it), list(map(squared, range(SMALL_RANGE)))) + + def test_imap_unordered(self): + it = self.pool.imap_unordered(sqr, range(LARGE_RANGE)) + self.assertEqual(sorted(it), list(map(squared, range(LARGE_RANGE)))) + + it = self.pool.imap_unordered(sqr, range(LARGE_RANGE)) + self.assertEqual(sorted(it), list(map(squared, range(LARGE_RANGE)))) + + def test_imap_unordered_random(self): + it = self.pool.imap_unordered(sqr_random_sleep, range(SMALL_RANGE)) + self.assertEqual(sorted(it), list(map(squared, range(SMALL_RANGE)))) + + def test_empty_imap_unordered(self): + it = self.pool.imap_unordered(sqr, []) + self.assertEqual(list(it), []) + + def test_empty_imap(self): + it = self.pool.imap(sqr, []) + self.assertEqual(list(it), []) + + def test_empty_map(self): + self.assertEqual(self.pool.map(sqr, []), []) + + def test_terminate(self): + result = self.pool.map_async(gevent.sleep, [0.1] * ((self.size or 10) * 2)) + gevent.sleep(0.1) + kill = TimingWrapper(self.pool.kill) + kill() + self.assertTimeWithinRange(kill.elapsed, 0.0, 0.5) + result.join() + + def sleep(self, x): + gevent.sleep(float(x) / 10.) + return str(x) + + def test_imap_unordered_sleep(self): + # testing that imap_unordered returns items in competion order + result = list(self.pool.imap_unordered(self.sleep, [10, 1, 2])) + if self.pool.size == 1: + expected = ['10', '1', '2'] + else: + expected = ['1', '2', '10'] + self.assertEqual(result, expected) + + # https://github.com/gevent/gevent/issues/423 + def test_imap_no_stop(self): + q = Queue() + q.put(123) + gevent.spawn_later(0.1, q.put, StopIteration) + result = list(self.pool.imap(lambda _: _, q)) + self.assertEqual(result, [123]) + + def test_imap_unordered_no_stop(self): + q = Queue() + q.put(1234) + gevent.spawn_later(0.1, q.put, StopIteration) + result = list(self.pool.imap_unordered(lambda _: _, q)) + self.assertEqual(result, [1234]) + + # same issue, but different test: https://github.com/gevent/gevent/issues/311 + def test_imap_final_sleep(self): + result = list(self.pool.imap(sqr, final_sleep())) + self.assertEqual(result, [0, 1, 4]) + + def test_imap_unordered_final_sleep(self): + result = list(self.pool.imap_unordered(sqr, final_sleep())) + self.assertEqual(result, [0, 1, 4]) + + # Issue 638 + def test_imap_unordered_bounded_queue(self): + iterable = list(range(100)) + + running = [0] + + def short_running_func(i, _j): + running[0] += 1 + return i + + def make_reader(mapping): + # Simulate a long running reader. No matter how many workers + # we have, we will never have a queue more than size 1 + def reader(): + result = [] + for i, x in enumerate(mapping): + self.assertTrue(running[0] <= i + 2, running[0]) + result.append(x) + gevent.sleep(0.01) + self.assertTrue(len(mapping.queue) <= 2, len(mapping.queue)) + return result + return reader + + # Send two iterables to make sure varargs and kwargs are handled + # correctly + for meth in self.pool.imap_unordered, self.pool.imap: + running[0] = 0 + mapping = meth(short_running_func, iterable, iterable, + maxsize=1) + + reader = make_reader(mapping) + l = reader() + self.assertEqual(sorted(l), iterable) + +@greentest.ignores_leakcheck +class TestPool2(TestPool): + size = 2 + +@greentest.ignores_leakcheck +class TestPool3(TestPool): + size = 3 + +@greentest.ignores_leakcheck +class TestPool10(TestPool): + size = 10 + + +class TestPoolUnlimit(TestPool): + size = None + + +class TestPool0(greentest.TestCase): + size = 0 + + def test_wait_full(self): + p = gevent.pool.Pool(size=0) + self.assertEqual(0, p.free_count()) + self.assertTrue(p.full()) + self.assertEqual(0, p.wait_available(timeout=0.01)) + + +class TestJoinSleep(gevent.testing.timing.AbstractGenericWaitTestCase): + + def wait(self, timeout): + p = gevent.pool.Pool() + g = p.spawn(gevent.sleep, 10) + try: + p.join(timeout=timeout) + finally: + g.kill() + + +class TestJoinSleep_raise_error(gevent.testing.timing.AbstractGenericWaitTestCase): + + def wait(self, timeout): + p = gevent.pool.Pool() + g = p.spawn(gevent.sleep, 10) + try: + p.join(timeout=timeout, raise_error=True) + finally: + g.kill() + + +class TestJoinEmpty(greentest.TestCase): + switch_expected = False + + def test(self): + p = gevent.pool.Pool() + res = p.join() + self.assertTrue(res, "empty should return true") + + +class TestSpawn(greentest.TestCase): + switch_expected = True + + def test(self): + p = gevent.pool.Pool(1) + self.assertEqual(len(p), 0) + p.spawn(gevent.sleep, 0.1) + self.assertEqual(len(p), 1) + p.spawn(gevent.sleep, 0.1) # this spawn blocks until the old one finishes + self.assertEqual(len(p), 1) + gevent.sleep(0.19 if not greentest.RUNNING_ON_APPVEYOR else 0.5) + self.assertEqual(len(p), 0) + + def testSpawnAndWait(self): + p = gevent.pool.Pool(1) + self.assertEqual(len(p), 0) + p.spawn(gevent.sleep, 0.1) + self.assertEqual(len(p), 1) + res = p.join(0.01) + self.assertFalse(res, "waiting on a full pool should return false") + res = p.join() + self.assertTrue(res, "waiting to finish should be true") + self.assertEqual(len(p), 0) + +def error_iter(): + yield 1 + yield 2 + raise ExpectedException + + +class TestErrorInIterator(greentest.TestCase): + error_fatal = False + + def test(self): + p = gevent.pool.Pool(3) + self.assertRaises(ExpectedException, p.map, lambda x: None, error_iter()) + gevent.sleep(0.001) + + def test_unordered(self): + p = gevent.pool.Pool(3) + + def unordered(): + return list(p.imap_unordered(lambda x: None, error_iter())) + + self.assertRaises(ExpectedException, unordered) + gevent.sleep(0.001) + + +def divide_by(x): + return 1.0 / x + + +class TestErrorInHandler(greentest.TestCase): + error_fatal = False + + def test_map(self): + p = gevent.pool.Pool(3) + self.assertRaises(ZeroDivisionError, p.map, divide_by, [1, 0, 2]) + + def test_imap(self): + p = gevent.pool.Pool(1) + it = p.imap(divide_by, [1, 0, 2]) + self.assertEqual(next(it), 1.0) + self.assertRaises(ZeroDivisionError, next, it) + self.assertEqual(next(it), 0.5) + self.assertRaises(StopIteration, next, it) + + def test_imap_unordered(self): + p = gevent.pool.Pool(1) + it = p.imap_unordered(divide_by, [1, 0, 2]) + self.assertEqual(next(it), 1.0) + self.assertRaises(ZeroDivisionError, next, it) + self.assertEqual(next(it), 0.5) + self.assertRaises(StopIteration, next, it) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__pywsgi.py b/src/gevent/tests/test__pywsgi.py new file mode 100644 index 0000000..9fe2261 --- /dev/null +++ b/src/gevent/tests/test__pywsgi.py @@ -0,0 +1,1811 @@ +# Copyright (c) 2007, Linden Research, Inc. +# Copyright (c) 2009-2010 gevent contributors +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# pylint: disable=too-many-lines,unused-argument +from __future__ import print_function + +from gevent import monkey + +monkey.patch_all(thread=False) + +try: + from urllib.parse import parse_qs +except ImportError: + # Python 2 + from urlparse import parse_qs +import os +import sys +try: + # On Python 2, we want the C-optimized version if + # available; it has different corner-case behaviour than + # the Python implementation, and it used by socket.makefile + # by default. + from cStringIO import StringIO +except ImportError: + from io import BytesIO as StringIO +import weakref + +from wsgiref.validate import validator + +import gevent.testing as greentest +import gevent +from gevent.testing import PY3, PYPY +from gevent import socket +from gevent import pywsgi +from gevent.pywsgi import Input + + +CONTENT_LENGTH = 'Content-Length' +CONN_ABORTED_ERRORS = greentest.CONN_ABORTED_ERRORS +server_implements_chunked = True +server_implements_pipeline = True +server_implements_100continue = True +DEBUG = '-v' in sys.argv + +REASONS = {200: 'OK', + 500: 'Internal Server Error'} + + +class ConnectionClosed(Exception): + pass + + +def read_headers(fd): + response_line = fd.readline() + if not response_line: + raise ConnectionClosed + response_line = response_line.decode('latin-1') + headers = {} + while True: + line = fd.readline().strip() + if not line: + break + line = line.decode('latin-1') + try: + key, value = line.split(': ', 1) + except: + print('Failed to split: %r' % (line, )) + raise + assert key.lower() not in {x.lower() for x in headers}, 'Header %r:%r sent more than once: %r' % (key, value, headers) + headers[key] = value + return response_line, headers + + +def iread_chunks(fd): + while True: + line = fd.readline() + chunk_size = line.strip() + try: + chunk_size = int(chunk_size, 16) + except: + print('Failed to parse chunk size: %r' % line) + raise + if chunk_size == 0: + crlf = fd.read(2) + assert crlf == b'\r\n', repr(crlf) + break + data = fd.read(chunk_size) + yield data + crlf = fd.read(2) + assert crlf == b'\r\n', repr(crlf) + + +class Response(object): + + def __init__(self, status_line, headers): + self.status_line = status_line + self.headers = headers + self.body = None + self.chunks = False + try: + version, code, self.reason = status_line[:-2].split(' ', 2) + self.code = int(code) + HTTP, self.version = version.split('/') + assert HTTP == 'HTTP', repr(HTTP) + assert self.version in ('1.0', '1.1'), repr(self.version) + except Exception: + print('Error: %r' % status_line) + raise + + def __iter__(self): + yield self.status_line + yield self.headers + yield self.body + + def __str__(self): + args = (self.__class__.__name__, self.status_line, self.headers, self.body, self.chunks) + return '<%s status_line=%r headers=%r body=%r chunks=%r>' % args + + def assertCode(self, code): + if hasattr(code, '__contains__'): + assert self.code in code, 'Unexpected code: %r (expected %r)\n%s' % (self.code, code, self) + else: + assert self.code == code, 'Unexpected code: %r (expected %r)\n%s' % (self.code, code, self) + + def assertReason(self, reason): + assert self.reason == reason, 'Unexpected reason: %r (expected %r)\n%s' % (self.reason, reason, self) + + def assertVersion(self, version): + assert self.version == version, 'Unexpected version: %r (expected %r)\n%s' % (self.version, version, self) + + def assertHeader(self, header, value): + real_value = self.headers.get(header, False) + assert real_value == value, \ + 'Unexpected header %r: %r (expected %r)\n%s' % (header, real_value, value, self) + + def assertBody(self, body): + if isinstance(body, str) and PY3: + body = body.encode("ascii") + assert self.body == body, 'Unexpected body: %r (expected %r)\n%s' % (self.body, body, self) + + @classmethod + def read(cls, fd, code=200, reason='default', version='1.1', + body=None, chunks=None, content_length=None): + # pylint:disable=too-many-branches + _status_line, headers = read_headers(fd) + self = cls(_status_line, headers) + if code is not None: + self.assertCode(code) + if reason == 'default': + reason = REASONS.get(code) + if reason is not None: + self.assertReason(reason) + if version is not None: + self.assertVersion(version) + if self.code == 100: + return self + if content_length is not None: + if isinstance(content_length, int): + content_length = str(content_length) + self.assertHeader('Content-Length', content_length) + try: + if 'chunked' in headers.get('Transfer-Encoding', ''): + if CONTENT_LENGTH in headers: + print("WARNING: server used chunked transfer-encoding despite having Content-Length header (libevent 1.x's bug)") + self.chunks = list(iread_chunks(fd)) + self.body = b''.join(self.chunks) + elif CONTENT_LENGTH in headers: + num = int(headers[CONTENT_LENGTH]) + self.body = fd.read(num) + else: + self.body = fd.read() + except: + print('Response.read failed to read the body:\n%s' % self) + import traceback; traceback.print_exc() + raise + if body is not None: + self.assertBody(body) + if chunks is not None: + assert chunks == self.chunks, (chunks, self.chunks) + return self + +read_http = Response.read + + +class TestCase(greentest.TestCase): + server = None + validator = staticmethod(validator) + application = None + + # Bind to default address, which should give us ipv6 (when available) + # and ipv4. (see self.connect()) + listen_addr = '' + # connect on ipv4, even though we bound to ipv6 too + # to prove ipv4 works...except on Windows, it apparently doesn't. + # So use the hostname. + connect_addr = 'localhost' + + def init_logger(self): + import logging + logger = logging.getLogger('gevent.pywsgi') + return logger + + def init_server(self, application): + logger = self.logger = self.init_logger() + self.server = pywsgi.WSGIServer((self.listen_addr, 0), application, + log=logger, error_log=logger) + + def setUp(self): + application = self.application + if self.validator is not None: + application = self.validator(application) + self.init_server(application) + self.server.start() + self.port = self.server.server_port + greentest.TestCase.setUp(self) + + if greentest.CPYTHON and greentest.PY2: + # Keeping raw sockets alive keeps SSL sockets + # from being closed too, at least on CPython2, so we + # need to use weakrefs. + + # In contrast, on PyPy, *only* having a weakref lets the + # original socket die and leak + + def _close_on_teardown(self, resource): + self.close_on_teardown.append(weakref.ref(resource)) + return resource + + def _tearDownCloseOnTearDown(self): + self.close_on_teardown = [r() for r in self.close_on_teardown if r() is not None] + super(TestCase, self)._tearDownCloseOnTearDown() + + def tearDown(self): + greentest.TestCase.tearDown(self) + if self.server is not None: + with gevent.Timeout.start_new(0.5): + self.server.stop() + self.server = None + + + def connect(self): + conn = socket.create_connection((self.connect_addr, self.port)) + self._close_on_teardown(conn) + result = conn + if PY3: + conn_makefile = conn.makefile + + def makefile(*args, **kwargs): + if 'bufsize' in kwargs: + kwargs['buffering'] = kwargs.pop('bufsize') + + if 'mode' in kwargs: + return conn_makefile(*args, **kwargs) + + # Under Python3, you can't read and write to the same + # makefile() opened in (default) r, and r+ is not allowed + kwargs['mode'] = 'rwb' + rconn = conn_makefile(*args, **kwargs) + _rconn_write = rconn.write + + def write(data): + if isinstance(data, str): + data = data.encode('ascii') + return _rconn_write(data) + rconn.write = write + self._close_on_teardown(rconn) + return rconn + + class proxy(object): + def __getattribute__(self, name): + if name == 'makefile': + return makefile + return getattr(conn, name) + result = proxy() + return result + + def makefile(self): + return self.connect().makefile(bufsize=1) + + def urlopen(self, *args, **kwargs): + fd = self.connect().makefile(bufsize=1) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + return read_http(fd, *args, **kwargs) + + +class CommonTests(TestCase): + + def test_basic(self): + fd = self.makefile() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + response = read_http(fd, body='hello world') + if response.headers.get('Connection') == 'close' and not server_implements_pipeline: + return + fd.write('GET /notexist HTTP/1.1\r\nHost: localhost\r\n\r\n') + read_http(fd, code=404, reason='Not Found', body='not found') + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + read_http(fd, body='hello world') + fd.close() + + def test_pipeline(self): + if not server_implements_pipeline: + return + fd = self.makefile() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n' + 'GET /notexist HTTP/1.1\r\nHost: localhost\r\n\r\n') + read_http(fd, body='hello world') + exception = AssertionError('HTTP pipelining not supported; the second request is thrown away') + try: + timeout = gevent.Timeout.start_new(0.5, exception=exception) + try: + read_http(fd, code=404, reason='Not Found', body='not found') + fd.close() + finally: + timeout.close() + except AssertionError as ex: + if ex is not exception: + raise + + def test_connection_close(self): + fd = self.makefile() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + response = read_http(fd) + if response.headers.get('Connection') == 'close' and not server_implements_pipeline: + return + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + read_http(fd) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + # This may either raise, or it may return an empty response, + # depend on timing and the Python version. + try: + result = fd.readline() + except socket.error as ex: + if ex.args[0] not in CONN_ABORTED_ERRORS: + raise + else: + self.assertFalse( + result, + 'The remote side is expected to close the connection, but it sent %r' % (result,)) + + def SKIP_test_006_reject_long_urls(self): + fd = self.makefile() + path_parts = [] + for _ in range(3000): + path_parts.append('path') + path = '/'.join(path_parts) + request = 'GET /%s HTTP/1.0\r\nHost: localhost\r\n\r\n' % path + fd.write(request) + result = fd.readline() + status = result.split(' ')[1] + self.assertEqual(status, '414') + fd.close() + + +class TestNoChunks(CommonTests): + # when returning a list of strings a shortcut is employed by the server: + # it calculates the content-length and joins all the chunks before sending + validator = None + + def application(self, env, start_response): + self.assertTrue(env.get('wsgi.input_terminated')) + path = env['PATH_INFO'] + if path == '/': + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [b'hello ', b'world'] + start_response('404 Not Found', [('Content-Type', 'text/plain')]) + return [b'not ', b'found'] + + def test(self): + fd = self.makefile() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + response = read_http(fd, body='hello world') + self.assertFalse(response.chunks) + response.assertHeader('Content-Length', '11') + + if not server_implements_pipeline: + fd = self.makefile() + + fd.write('GET /not-found HTTP/1.1\r\nHost: localhost\r\n\r\n') + response = read_http(fd, code=404, reason='Not Found', body='not found') + self.assertFalse(response.chunks) + response.assertHeader('Content-Length', '9') + + +class TestExplicitContentLength(TestNoChunks): # pylint:disable=too-many-ancestors + # when returning a list of strings a shortcut is empoyed by the + # server - it caculates the content-length + + def application(self, env, start_response): + self.assertTrue(env.get('wsgi.input_terminated')) + path = env['PATH_INFO'] + if path == '/': + start_response('200 OK', [('Content-Type', 'text/plain'), ('Content-Length', '11')]) + return [b'hello ', b'world'] + + start_response('404 Not Found', [('Content-Type', 'text/plain'), ('Content-Length', '9')]) + return [b'not ', b'found'] + + +class TestYield(CommonTests): + + @staticmethod + def application(env, start_response): + path = env['PATH_INFO'] + if path == '/': + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield b"hello world" + else: + start_response('404 Not Found', [('Content-Type', 'text/plain')]) + yield b"not found" + + +class TestBytearray(CommonTests): + + validator = None + + @staticmethod + def application(env, start_response): + path = env['PATH_INFO'] + if path == '/': + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [bytearray(b"hello "), bytearray(b"world")] + start_response('404 Not Found', [('Content-Type', 'text/plain')]) + return [bytearray(b"not found")] + + +class MultiLineHeader(TestCase): + @staticmethod + def application(env, start_response): + assert "test.submit" in env["CONTENT_TYPE"] + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [b"ok"] + + def test_multiline_116(self): + """issue #116""" + request = '\r\n'.join(( + 'POST / HTTP/1.0', + 'Host: localhost', + 'Content-Type: multipart/related; boundary="====XXXX====";', + ' type="text/xml";start="test.submit"', + 'Content-Length: 0', + '', '')) + fd = self.makefile() + fd.write(request) + read_http(fd) + + +class TestGetArg(TestCase): + + @staticmethod + def application(env, start_response): + body = env['wsgi.input'].read(3) + if PY3: + body = body.decode('ascii') + a = parse_qs(body).get('a', [1])[0] + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [('a is %s, body is %s' % (a, body)).encode('ascii')] + + def test_007_get_arg(self): + # define a new handler that does a get_arg as well as a read_body + fd = self.makefile() + request = '\r\n'.join(( + 'POST / HTTP/1.0', + 'Host: localhost', + 'Content-Length: 3', + '', + 'a=a')) + fd.write(request) + + # send some junk after the actual request + fd.write('01234567890123456789') + read_http(fd, body='a is a, body is a=a') + fd.close() + +class TestCloseIter(TestCase): + + # The *Validator* closes the iterators! + validator = None + + def application(self, env, start_response): + start_response('200 OK', [('Content-Type', 'text/plain')]) + return self + + def __iter__(self): + yield bytearray(b"Hello World") + yield b"!" + + closed = False + + def close(self): + self.closed += 1 + + def test_close_is_called(self): + self.closed = False + fd = self.makefile() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + read_http(fd, body=b"Hello World!", chunks=[b'Hello World', b'!']) + # We got closed exactly once. + self.assertEqual(self.closed, 1) + +class TestChunkedApp(TestCase): + + chunks = [b'this', b'is', b'chunked'] + + def body(self): + return b''.join(self.chunks) + + def application(self, env, start_response): + self.assertTrue(env.get('wsgi.input_terminated')) + start_response('200 OK', [('Content-Type', 'text/plain')]) + for chunk in self.chunks: + yield chunk + + def test_chunked_response(self): + fd = self.makefile() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + response = read_http(fd, body=self.body(), chunks=None) + if server_implements_chunked: + response.assertHeader('Transfer-Encoding', 'chunked') + self.assertEqual(response.chunks, self.chunks) + else: + response.assertHeader('Transfer-Encoding', False) + response.assertHeader('Content-Length', str(len(self.body()))) + self.assertEqual(response.chunks, False) + + def test_no_chunked_http_1_0(self): + fd = self.makefile() + fd.write('GET / HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n') + response = read_http(fd) + self.assertEqual(response.body, self.body()) + self.assertEqual(response.headers.get('Transfer-Encoding'), None) + content_length = response.headers.get('Content-Length') + if content_length is not None: + self.assertEqual(content_length, str(len(self.body()))) + + +class TestBigChunks(TestChunkedApp): + chunks = [b'a' * 8192] * 3 + + +class TestNegativeRead(TestCase): + + def application(self, env, start_response): + self.assertTrue(env.get('wsgi.input_terminated')) + start_response('200 OK', [('Content-Type', 'text/plain')]) + if env['PATH_INFO'] == '/read': + data = env['wsgi.input'].read(-1) + return [data] + + def test_negative_chunked_read(self): + fd = self.makefile() + data = (b'POST /read HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'2\r\noh\r\n4\r\n hai\r\n0\r\n\r\n') + fd.write(data) + read_http(fd, body='oh hai') + + def test_negative_nonchunked_read(self): + fd = self.makefile() + data = (b'POST /read HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Content-Length: 6\r\n\r\n' + b'oh hai') + fd.write(data) + read_http(fd, body='oh hai') + + +class TestNegativeReadline(TestCase): + validator = None + + @staticmethod + def application(env, start_response): + start_response('200 OK', [('Content-Type', 'text/plain')]) + if env['PATH_INFO'] == '/readline': + data = env['wsgi.input'].readline(-1) + return [data] + + def test_negative_chunked_readline(self): + fd = self.makefile() + data = (b'POST /readline HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'2\r\noh\r\n4\r\n hai\r\n0\r\n\r\n') + fd.write(data) + read_http(fd, body='oh hai') + + def test_negative_nonchunked_readline(self): + fd = self.makefile() + data = (b'POST /readline HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Content-Length: 6\r\n\r\n' + b'oh hai') + fd.write(data) + read_http(fd, body='oh hai') + + +class TestChunkedPost(TestCase): + + + def application(self, env, start_response): + self.assertTrue(env.get('wsgi.input_terminated')) + start_response('200 OK', [('Content-Type', 'text/plain')]) + if env['PATH_INFO'] == '/a': + data = env['wsgi.input'].read(6) + return [data] + + if env['PATH_INFO'] == '/b': + lines = [x for x in iter(lambda: env['wsgi.input'].read(6), b'')] + return lines + + if env['PATH_INFO'] == '/c': + return [x for x in iter(lambda: env['wsgi.input'].read(1), b'')] + + def test_014_chunked_post(self): + fd = self.makefile() + data = (b'POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'2\r\noh\r\n4\r\n hai\r\n0\r\n\r\n') + fd.write(data) + read_http(fd, body='oh hai') + # self.close_opened() # XXX: Why? + + fd = self.makefile() + fd.write(data.replace(b'/a', b'/b')) + read_http(fd, body='oh hai') + + fd = self.makefile() + fd.write(data.replace(b'/a', b'/c')) + read_http(fd, body='oh hai') + + def test_229_incorrect_chunk_no_newline(self): + # Giving both a Content-Length and a Transfer-Encoding, + # TE is preferred. But if the chunking is bad from the client, + # missing its terminating newline, + # the server doesn't hang + data = (b'POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Content-Length: 12\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'{"hi": "ho"}') + fd = self.makefile() + fd.write(data) + read_http(fd, code=400) + + def test_229_incorrect_chunk_non_hex(self): + # Giving both a Content-Length and a Transfer-Encoding, + # TE is preferred. But if the chunking is bad from the client, + # the server doesn't hang + data = (b'POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Content-Length: 12\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'{"hi": "ho"}\r\n') + fd = self.makefile() + fd.write(data) + read_http(fd, code=400) + + def test_229_correct_chunk_quoted_ext(self): + data = (b'POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'2;token="oh hi"\r\noh\r\n4\r\n hai\r\n0\r\n\r\n') + fd = self.makefile() + fd.write(data) + read_http(fd, body='oh hai') + + def test_229_correct_chunk_token_ext(self): + data = (b'POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'2;token=oh_hi\r\noh\r\n4\r\n hai\r\n0\r\n\r\n') + fd = self.makefile() + fd.write(data) + read_http(fd, body='oh hai') + + def test_229_incorrect_chunk_token_ext_too_long(self): + data = (b'POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + b'Transfer-Encoding: chunked\r\n\r\n' + b'2;token=oh_hi\r\noh\r\n4\r\n hai\r\n0\r\n\r\n') + data = data.replace(b'oh_hi', b'_oh_hi' * 4000) + fd = self.makefile() + fd.write(data) + read_http(fd, code=400) + + +class TestUseWrite(TestCase): + + body = b'abcde' + end = b'end' + content_length = str(len(body + end)) + + def application(self, env, start_response): + if env['PATH_INFO'] == '/explicit-content-length': + write = start_response('200 OK', [('Content-Type', 'text/plain'), + ('Content-Length', self.content_length)]) + write(self.body) + elif env['PATH_INFO'] == '/no-content-length': + write = start_response('200 OK', [('Content-Type', 'text/plain')]) + write(self.body) + elif env['PATH_INFO'] == '/no-content-length-twice': + write = start_response('200 OK', [('Content-Type', 'text/plain')]) + write(self.body) + write(self.body) + else: + raise Exception('Invalid url') + return [self.end] + + def test_explicit_content_length(self): + fd = self.makefile() + fd.write('GET /explicit-content-length HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + response = read_http(fd, body=self.body + self.end) + response.assertHeader('Content-Length', self.content_length) + response.assertHeader('Transfer-Encoding', False) + + def test_no_content_length(self): + fd = self.makefile() + fd.write('GET /no-content-length HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + response = read_http(fd, body=self.body + self.end) + if server_implements_chunked: + response.assertHeader('Content-Length', False) + response.assertHeader('Transfer-Encoding', 'chunked') + else: + response.assertHeader('Content-Length', self.content_length) + + def test_no_content_length_twice(self): + fd = self.makefile() + fd.write('GET /no-content-length-twice HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + response = read_http(fd, body=self.body + self.body + self.end) + if server_implements_chunked: + response.assertHeader('Content-Length', False) + response.assertHeader('Transfer-Encoding', 'chunked') + assert response.chunks == [self.body, self.body, self.end], response.chunks + else: + response.assertHeader('Content-Length', str(5 + 5 + 3)) + + +class HttpsTestCase(TestCase): + + certfile = os.path.join(os.path.dirname(__file__), 'test_server.crt') + keyfile = os.path.join(os.path.dirname(__file__), 'test_server.key') + + def init_server(self, application): + self.server = pywsgi.WSGIServer((self.listen_addr, 0), application, + certfile=self.certfile, keyfile=self.keyfile) + + def urlopen(self, method='GET', post_body=None, **kwargs): # pylint:disable=arguments-differ + import ssl + raw_sock = self.connect() + sock = ssl.wrap_socket(raw_sock) + fd = sock.makefile(bufsize=1) # pylint:disable=unexpected-keyword-arg + fd.write('%s / HTTP/1.1\r\nHost: localhost\r\n' % method) + if post_body is not None: + fd.write('Content-Length: %s\r\n\r\n' % len(post_body)) + fd.write(post_body) + if kwargs.get('body') is None: + kwargs['body'] = post_body + else: + fd.write('\r\n') + fd.flush() + try: + return read_http(fd, **kwargs) + finally: + fd.close() + sock.close() + raw_sock.close() + + def application(self, environ, start_response): + assert environ['wsgi.url_scheme'] == 'https', environ['wsgi.url_scheme'] + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [environ['wsgi.input'].read(10)] + + +import gevent.ssl +HAVE_SSLCONTEXT = getattr(gevent.ssl, 'create_default_context') +if HAVE_SSLCONTEXT: + + class HttpsSslContextTestCase(HttpsTestCase): + def init_server(self, application): + # On 2.7, our certs don't line up with hostname. + # If we just use create_default_context as-is, we get + # `ValueError: check_hostname requires server_hostname`. + # If we set check_hostname to False, we get + # `SSLError: [SSL: PEER_DID_NOT_RETURN_A_CERTIFICATE] peer did not return a certificate` + # (Neither of which happens in Python 3.) But the unverified context + # works both places. See also test___example_servers.py + from gevent.ssl import _create_unverified_context # pylint:disable=no-name-in-module + context = _create_unverified_context() + context.load_cert_chain(certfile=self.certfile, keyfile=self.keyfile) + self.server = pywsgi.WSGIServer((self.listen_addr, 0), + application, ssl_context=context) + +class TestHttps(HttpsTestCase): + + if hasattr(socket, 'ssl'): + + def test_012_ssl_server(self): + result = self.urlopen(method="POST", post_body='abc') + self.assertEqual(result.body, 'abc') + + def test_013_empty_return(self): + result = self.urlopen() + self.assertEqual(result.body, '') + +if HAVE_SSLCONTEXT: + class TestHttpsWithContext(HttpsSslContextTestCase, TestHttps): # pylint:disable=too-many-ancestors + pass + +class TestInternational(TestCase): + validator = None # wsgiref.validate.IteratorWrapper([]) does not have __len__ + + def application(self, environ, start_response): + path_bytes = b'/\xd0\xbf\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82' + if PY3: + # Under PY3, the escapes were decoded as latin-1 + path_bytes = path_bytes.decode('latin-1') + + self.assertEqual(environ['PATH_INFO'], path_bytes) + self.assertEqual(environ['QUERY_STRING'], '%D0%B2%D0%BE%D0%BF%D1%80%D0%BE%D1%81=%D0%BE%D1%82%D0%B2%D0%B5%D1%82') + start_response("200 PASSED", [('Content-Type', 'text/plain')]) + return [] + + def test(self): + sock = self.connect() + sock.sendall(b'''GET /%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82?%D0%B2%D0%BE%D0%BF%D1%80%D0%BE%D1%81=%D0%BE%D1%82%D0%B2%D0%B5%D1%82 HTTP/1.1 +Host: localhost +Connection: close + +'''.replace(b'\n', b'\r\n')) + read_http(sock.makefile(), reason='PASSED', chunks=False, body='', content_length=0) + + +class TestNonLatin1HeaderFromApplication(TestCase): + error_fatal = False # Allow sending the exception response, don't kill the greenlet + + validator = None # Don't validate the application, it's deliberately bad + header = b'\xe1\xbd\x8a3' # bomb in utf-8 bytes + should_error = PY3 # non-native string under Py3 + + def setUp(self): + super(TestNonLatin1HeaderFromApplication, self).setUp() + self.errors = [] + + def tearDown(self): + self.errors = [] + super(TestNonLatin1HeaderFromApplication, self).tearDown() + + def application(self, environ, start_response): + # We return a header that cannot be encoded in latin-1 + try: + start_response("200 PASSED", + [('Content-Type', 'text/plain'), + ('Custom-Header', self.header)]) + except: + self.errors.append(sys.exc_info()[:2]) + raise + return [] + + def test(self): + sock = self.connect() + self.expect_one_error() + sock.sendall(b'''GET / HTTP/1.1\r\n\r\n''') + if self.should_error: + read_http(sock.makefile(), code=500, reason='Internal Server Error') + self.assert_error(where_type=pywsgi.SecureEnviron) + self.assertEqual(len(self.errors), 1) + _, v = self.errors[0] + self.assertIsInstance(v, UnicodeError) + else: + read_http(sock.makefile(), code=200, reason='PASSED') + self.assertEqual(len(self.errors), 0) + + +class TestNonLatin1UnicodeHeaderFromApplication(TestNonLatin1HeaderFromApplication): + # Flip-flop of the superclass: Python 3 native string, Python 2 unicode object + header = u"\u1f4a3" # bomb in unicode + # Error both on py3 and py2. On py2, non-native string. On py3, native string + # that cannot be encoded to latin-1 + should_error = True + + +class TestInputReadline(TestCase): + # this test relies on the fact that readline() returns '' after it reached EOF + # this behaviour is not mandated by WSGI spec, it's just happens that gevent.wsgi behaves like that + # as such, this may change in the future + + validator = None + + def application(self, environ, start_response): + input = environ['wsgi.input'] + lines = [] + while True: + line = input.readline() + if not line: + break + line = line.decode('ascii') if PY3 else line + lines.append(repr(line) + ' ') + start_response('200 hello', []) + return [l.encode('ascii') for l in lines] if PY3 else lines + + def test(self): + fd = self.makefile() + content = 'hello\n\nworld\n123' + fd.write('POST / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n' + 'Content-Length: %s\r\n\r\n%s' % (len(content), content)) + fd.flush() + read_http(fd, reason='hello', body="'hello\\n' '\\n' 'world\\n' '123' ") + + +class TestInputIter(TestInputReadline): + + def application(self, environ, start_response): + input = environ['wsgi.input'] + lines = [] + for line in input: + if not line: + break + line = line.decode('ascii') if PY3 else line + lines.append(repr(line) + ' ') + start_response('200 hello', []) + return [l.encode('ascii') for l in lines] if PY3 else lines + + +class TestInputReadlines(TestInputReadline): + + def application(self, environ, start_response): + input = environ['wsgi.input'] + lines = [l.decode('ascii') if PY3 else l for l in input.readlines()] + lines = [repr(line) + ' ' for line in lines] + start_response('200 hello', []) + return [l.encode('ascii') for l in lines] if PY3 else lines + + +class TestInputN(TestCase): + # testing for this: + # File "/home/denis/work/gevent/gevent/pywsgi.py", line 70, in _do_read + # if length and length > self.content_length - self.position: + # TypeError: unsupported operand type(s) for -: 'NoneType' and 'int' + + validator = None + + def application(self, environ, start_response): + environ['wsgi.input'].read(5) + start_response('200 OK', []) + return [] + + def test(self): + self.urlopen() + + +class TestError(TestCase): + + error = object() + error_fatal = False + + def application(self, env, start_response): + self.error = greentest.ExpectedException('TestError.application') + raise self.error + + def test(self): + self.expect_one_error() + self.urlopen(code=500) + self.assert_error(greentest.ExpectedException, self.error) + + +class TestError_after_start_response(TestError): + + def application(self, env, start_response): + self.error = greentest.ExpectedException('TestError_after_start_response.application') + start_response('200 OK', [('Content-Type', 'text/plain')]) + raise self.error + + +class TestEmptyYield(TestCase): + + @staticmethod + def application(env, start_response): + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield b"" + yield b"" + + def test_err(self): + fd = self.connect().makefile(bufsize=1) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + + if server_implements_chunked: + chunks = [] + else: + chunks = False + + read_http(fd, body='', chunks=chunks) + + garbage = fd.read() + self.assertEqual(garbage, b"", "got garbage: %r" % garbage) + + +class TestFirstEmptyYield(TestCase): + + @staticmethod + def application(env, start_response): + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield b"" + yield b"hello" + + def test_err(self): + fd = self.connect().makefile(bufsize=1) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + + if server_implements_chunked: + chunks = [b'hello'] + else: + chunks = False + + read_http(fd, body='hello', chunks=chunks) + + garbage = fd.read() + self.assertTrue(garbage == b"", "got garbage: %r" % garbage) + + +class TestEmptyYield304(TestCase): + + @staticmethod + def application(env, start_response): + start_response('304 Not modified', []) + yield b"" + yield b"" + + def test_err(self): + fd = self.connect().makefile(bufsize=1) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + read_http(fd, code=304, body='', chunks=False) + garbage = fd.read() + self.assertEqual(garbage, b"", "got garbage: %r" % garbage) + + +class TestContentLength304(TestCase): + validator = None + + def application(self, env, start_response): + try: + start_response('304 Not modified', [('Content-Length', '100')]) + except AssertionError as ex: + start_response('200 Raised', []) + return ex.args + else: + raise AssertionError('start_response did not fail but it should') + + def test_err(self): + fd = self.connect().makefile(bufsize=1) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + body = "Invalid Content-Length for 304 response: '100' (must be absent or zero)" + read_http(fd, code=200, reason='Raised', body=body, chunks=False) + garbage = fd.read() + self.assertEqual(garbage, b"", "got garbage: %r" % garbage) + + +class TestBody304(TestCase): + validator = None + + def application(self, env, start_response): + start_response('304 Not modified', []) + return [b'body'] + + def test_err(self): + fd = self.connect().makefile(bufsize=1) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + try: + read_http(fd) + except AssertionError as ex: + self.assertEqual(str(ex), 'The 304 response must have no body') + else: + raise AssertionError('AssertionError must be raised') + + +class TestWrite304(TestCase): + validator = None + error_raised = None + + def application(self, env, start_response): + write = start_response('304 Not modified', []) + self.error_raised = False + try: + write('body') + except AssertionError: + self.error_raised = True + raise + + def test_err(self): + fd = self.connect().makefile(bufsize=1) + fd.write(b'GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + with self.assertRaises(AssertionError) as exc: + read_http(fd) + ex = exc.exception + self.assertEqual(str(ex), 'The 304 response must have no body') + self.assertTrue(self.error_raised, 'write() must raise') + + +class TestEmptyWrite(TestEmptyYield): + + @staticmethod + def application(env, start_response): + write = start_response('200 OK', [('Content-Type', 'text/plain')]) + write(b"") + write(b"") + return [] + + +class BadRequestTests(TestCase): + validator = None + # pywsgi checks content-length, but wsgi does not + content_length = None + + def application(self, env, start_response): + self.assertEqual(env['CONTENT_LENGTH'], self.content_length) + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [b'hello'] + + def test_negative_content_length(self): + self.content_length = '-100' + fd = self.connect().makefile(bufsize=1) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: %s\r\n\r\n' % self.content_length) + read_http(fd, code=(200, 400)) + + def test_illegal_content_length(self): + self.content_length = 'abc' + fd = self.connect().makefile(bufsize=1) + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nContent-Length: %s\r\n\r\n' % self.content_length) + read_http(fd, code=(200, 400)) + + +class ChunkedInputTests(TestCase): + dirt = "" + validator = None + + def application(self, env, start_response): + input = env['wsgi.input'] + response = [] + + pi = env["PATH_INFO"] + + if pi == "/short-read": + d = input.read(10) + response = [d] + elif pi == "/lines": + for x in input: + response.append(x) + elif pi == "/ping": + input.read(1) + response.append(b"pong") + else: + raise RuntimeError("bad path") + + start_response('200 OK', [('Content-Type', 'text/plain')]) + return response + + def chunk_encode(self, chunks, dirt=None): + if dirt is None: + dirt = self.dirt + + return chunk_encode(chunks, dirt=dirt) + + def body(self, dirt=None): + return self.chunk_encode(["this", " is ", "chunked", "\nline", " 2", "\n", "line3", ""], dirt=dirt) + + def ping(self, fd): + fd.write("GET /ping HTTP/1.1\r\n\r\n") + read_http(fd, body="pong") + + def ping_if_possible(self, fd): + try: + self.ping(fd) + except ConnectionClosed: + if server_implements_pipeline: + raise + fd = self.connect().makefile(bufsize=1) + self.ping(fd) + + def test_short_read_with_content_length(self): + body = self.body() + req = b"POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\nContent-Length:1000\r\n\r\n" + body + conn = self.connect() + fd = conn.makefile(bufsize=1) + fd.write(req) + read_http(fd, body="this is ch") + + self.ping_if_possible(fd) + fd.close() + conn.close() + + def test_short_read_with_zero_content_length(self): + body = self.body() + req = b"POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\nContent-Length:0\r\n\r\n" + body + #print("REQUEST:", repr(req)) + + fd = self.connect().makefile(bufsize=1) + fd.write(req) + read_http(fd, body="this is ch") + + self.ping_if_possible(fd) + + def test_short_read(self): + body = self.body() + req = b"POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\n\r\n" + body + + fd = self.connect().makefile(bufsize=1) + fd.write(req) + read_http(fd, body="this is ch") + + self.ping_if_possible(fd) + + def test_dirt(self): + body = self.body(dirt="; here is dirt\0bla") + req = b"POST /ping HTTP/1.1\r\ntransfer-encoding: Chunked\r\n\r\n" + body + + fd = self.connect().makefile(bufsize=1) + fd.write(req) + try: + read_http(fd, body="pong") + except AssertionError as ex: + if str(ex).startswith('Unexpected code: 400'): + if not server_implements_chunked: + print('ChunkedNotImplementedWarning') + return + raise + + self.ping_if_possible(fd) + + def test_chunked_readline(self): + body = self.body() + req = "POST /lines HTTP/1.1\r\nContent-Length: %s\r\ntransfer-encoding: Chunked\r\n\r\n" % (len(body)) + req = req.encode('latin-1') + req += body + + fd = self.connect().makefile(bufsize=1) + fd.write(req) + read_http(fd, body='this is chunked\nline 2\nline3') + + def test_close_before_finished(self): + self.expect_one_error() + body = b'4\r\nthi' + req = b"POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\n\r\n" + body + sock = self.connect() + fd = sock.makefile(bufsize=1, mode='wb') + fd.write(req) + fd.close() + + # Python 3 keeps the socket open even though the only + # makefile is gone; python 2 closed them both (because there were + # no outstanding references to the socket). Closing is essential for the server + # to get the message that the read will fail. It's better to be explicit + # to avoid a ResourceWarning + sock.close() + # Under Py2 it still needs to go away, which was implicit before + del fd + del sock + + gevent.get_hub().loop.update_now() + gevent.sleep(0.01) # timing needed for cpython + + if greentest.PYPY: + # XXX: Something is keeping the socket alive, + # by which I mean, the close event is not propagating to the server + # and waking up its recv() loop...we are stuck with the three bytes of + # 'thi' in the buffer and trying to read the forth. No amount of tinkering + # with the timing changes this...the only thing that does is running a + # GC and letting some object get collected. Might this be a problem in real life? + + import gc + gc.collect() + gevent.sleep(0.01) + gevent.get_hub().loop.update_now() + gc.collect() + gevent.sleep(0.01) + + # XXX2: Sometimes windows and PyPy/Travis fail to get this error, leading to a test failure. + # This would have to be due to the socket being kept around and open, + # not closed at the low levels. I haven't seen this locally. + # In the PyPy case, I've seen the IOError reported on the console, but not + # captured in the variables. + # https://travis-ci.org/gevent/gevent/jobs/329232976#L1374 + self.assert_error(IOError, 'unexpected end of file while parsing chunked data') + + +class Expect100ContinueTests(TestCase): + validator = None + + def application(self, environ, start_response): + content_length = int(environ['CONTENT_LENGTH']) + if content_length > 1024: + start_response('417 Expectation Failed', [('Content-Length', '7'), ('Content-Type', 'text/plain')]) + return [b'failure'] + + # pywsgi did sent a "100 continue" for each read + # see http://code.google.com/p/gevent/issues/detail?id=93 + text = environ['wsgi.input'].read(1) + text += environ['wsgi.input'].read(content_length - 1) + start_response('200 OK', [('Content-Length', str(len(text))), ('Content-Type', 'text/plain')]) + return [text] + + def test_continue(self): + fd = self.connect().makefile(bufsize=1) + + fd.write('PUT / HTTP/1.1\r\nHost: localhost\r\nContent-length: 1025\r\nExpect: 100-continue\r\n\r\n') + try: + read_http(fd, code=417, body="failure") + except AssertionError as ex: + if str(ex).startswith('Unexpected code: 400'): + if not server_implements_100continue: + print('100ContinueNotImplementedWarning') + return + raise + + fd.write('PUT / HTTP/1.1\r\nHost: localhost\r\nContent-length: 7\r\nExpect: 100-continue\r\n\r\ntesting') + read_http(fd, code=100) + read_http(fd, body="testing") + + +class MultipleCookieHeadersTest(TestCase): + + validator = None + + def application(self, environ, start_response): + self.assertEqual(environ['HTTP_COOKIE'], 'name1="value1"; name2="value2"') + self.assertEqual(environ['HTTP_COOKIE2'], 'nameA="valueA"; nameB="valueB"') + start_response('200 OK', []) + return [] + + def test(self): + fd = self.connect().makefile(bufsize=1) + fd.write('''GET / HTTP/1.1 +Host: localhost +Cookie: name1="value1" +Cookie2: nameA="valueA" +Cookie2: nameB="valueB" +Cookie: name2="value2"\n\n'''.replace('\n', '\r\n')) + read_http(fd) + + +class TestLeakInput(TestCase): + + _leak_wsgi_input = None + _leak_environ = None + + def tearDown(self): + TestCase.tearDown(self) + self._leak_wsgi_input = None + self._leak_environ = None + + def application(self, environ, start_response): + pi = environ["PATH_INFO"] + self._leak_wsgi_input = environ["wsgi.input"] + self._leak_environ = environ + if pi == "/leak-frame": + environ["_leak"] = sys._getframe(0) + + text = b"foobar" + start_response('200 OK', [('Content-Length', str(len(text))), ('Content-Type', 'text/plain')]) + return [text] + + def test_connection_close_leak_simple(self): + fd = self.connect().makefile(bufsize=1) + fd.write(b"GET / HTTP/1.0\r\nConnection: close\r\n\r\n") + d = fd.read() + self.assertTrue(d.startswith(b"HTTP/1.1 200 OK"), d) + + def test_connection_close_leak_frame(self): + fd = self.connect().makefile(bufsize=1) + fd.write(b"GET /leak-frame HTTP/1.0\r\nConnection: close\r\n\r\n") + d = fd.read() + self.assertTrue(d.startswith(b"HTTP/1.1 200 OK"), d) + self._leak_environ.pop('_leak') + +class TestHTTPResponseSplitting(TestCase): + # The validator would prevent the app from doing the + # bad things it needs to do + validator = None + + status = '200 OK' + headers = () + start_exc = None + + def setUp(self): + TestCase.setUp(self) + self.start_exc = None + self.status = TestHTTPResponseSplitting.status + self.headers = TestHTTPResponseSplitting.headers + + def tearDown(self): + TestCase.tearDown(self) + self.start_exc = None + + def application(self, environ, start_response): + try: + start_response(self.status, self.headers) + except Exception as e: # pylint: disable=broad-except + self.start_exc = e + else: + self.start_exc = None + return () + + def _assert_failure(self, message): + fd = self.makefile() + fd.write('GET / HTTP/1.0\r\nHost: localhost\r\n\r\n') + fd.read() + self.assertIsInstance(self.start_exc, ValueError) + self.assertEqual(self.start_exc.args[0], message) + + def test_newline_in_status(self): + self.status = '200 OK\r\nConnection: close\r\nContent-Length: 0\r\n\r\n' + self._assert_failure('carriage return or newline in status') + + def test_newline_in_header_value(self): + self.headers = [('Test', 'Hi\r\nConnection: close')] + self._assert_failure('carriage return or newline in header value') + + def test_newline_in_header_name(self): + self.headers = [('Test\r\n', 'Hi')] + self._assert_failure('carriage return or newline in header name') + + +class TestInvalidEnviron(TestCase): + validator = None + # check that WSGIServer does not insert any default values for CONTENT_LENGTH + + def application(self, environ, start_response): + for key, value in environ.items(): + if key in ('CONTENT_LENGTH', 'CONTENT_TYPE') or key.startswith('HTTP_'): + if key != 'HTTP_HOST': + raise AssertionError('Unexpected environment variable: %s=%r' % (key, value)) + start_response('200 OK', []) + return [] + + def test(self): + fd = self.makefile() + fd.write('GET / HTTP/1.0\r\nHost: localhost\r\n\r\n') + read_http(fd) + fd = self.makefile() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n') + read_http(fd) + + +class TestInvalidHeadersDropped(TestCase): + validator = None + # check that invalid headers with a _ are dropped + + def application(self, environ, start_response): + self.assertNotIn('HTTP_X_AUTH_USER', environ) + start_response('200 OK', []) + return [] + + def test(self): + fd = self.makefile() + fd.write('GET / HTTP/1.0\r\nx-auth_user: bob\r\n\r\n') + read_http(fd) + + +class Handler(pywsgi.WSGIHandler): + + def read_requestline(self): + data = self.rfile.read(7) + if data[0] == b'<'[0]: + # Returning nothing stops handle_one_request() + # Note that closing or even deleting self.socket() here + # can lead to the read side throwing Connection Reset By Peer, + # depending on the Python version and OS + data += self.rfile.read(15) + if data.lower() == b'': + self.socket.sendall(b'HELLO') + else: + self.log_error('Invalid request: %r', data) + return None + else: + return data + self.rfile.readline() + + +class TestHandlerSubclass(TestCase): + + validator = None + + def application(self, environ, start_response): + start_response('200 OK', []) + return [] + + def init_server(self, application): + self.server = pywsgi.WSGIServer((self.listen_addr, 0), + application, + handler_class=Handler) + + def test(self): + fd = self.makefile() + fd.write(b'\x00') + fd.flush() # flush() is needed on PyPy, apparently it buffers slightly differently + self.assertEqual(fd.read(), b'HELLO') + + fd = self.makefile() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + fd.flush() + read_http(fd) + + fd = self.makefile() + # Trigger an error + fd.write('\x00') + fd.flush() + self.assertEqual(fd.read(), b'') + + +class TestErrorAfterChunk(TestCase): + validator = None + + @staticmethod + def application(env, start_response): + start_response('200 OK', [('Content-Type', 'text/plain')]) + yield b"hello" + raise greentest.ExpectedException('TestErrorAfterChunk') + + def test(self): + fd = self.connect().makefile(bufsize=1) + self.expect_one_error() + fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n') + self.assertRaises(ValueError, read_http, fd) + self.assert_error(greentest.ExpectedException) + + +def chunk_encode(chunks, dirt=None): + if dirt is None: + dirt = "" + + b = b"" + for c in chunks: + x = "%x%s\r\n%s\r\n" % (len(c), dirt, c) + b += x.encode('ascii') + return b + + +class TestInputRaw(greentest.BaseTestCase): + def make_input(self, data, content_length=None, chunked_input=False): + if isinstance(data, list): + data = chunk_encode(data) + chunked_input = True + elif isinstance(data, str) and PY3: + data = data.encode("ascii") + return Input(StringIO(data), content_length=content_length, chunked_input=chunked_input) + + if PY3: + def assertEqual(self, data, expected, *args): # pylint:disable=arguments-differ + if isinstance(expected, str): + expected = expected.encode('ascii') + super(TestInputRaw, self).assertEqual(data, expected, *args) + + def test_short_post(self): + i = self.make_input("1", content_length=2) + self.assertRaises(IOError, i.read) + + def test_short_post_read_with_length(self): + i = self.make_input("1", content_length=2) + self.assertRaises(IOError, i.read, 2) + + def test_short_post_readline(self): + i = self.make_input("1", content_length=2) + self.assertRaises(IOError, i.readline) + + def test_post(self): + i = self.make_input("12", content_length=2) + data = i.read() + self.assertEqual(data, "12") + + def test_post_read_with_length(self): + i = self.make_input("12", content_length=2) + data = i.read(10) + self.assertEqual(data, "12") + + def test_chunked(self): + i = self.make_input(["1", "2", ""]) + data = i.read() + self.assertEqual(data, "12") + + def test_chunked_read_with_length(self): + i = self.make_input(["1", "2", ""]) + data = i.read(10) + self.assertEqual(data, "12") + + def test_chunked_missing_chunk(self): + i = self.make_input(["1", "2"]) + self.assertRaises(IOError, i.read) + + def test_chunked_missing_chunk_read_with_length(self): + i = self.make_input(["1", "2"]) + self.assertRaises(IOError, i.read, 10) + + def test_chunked_missing_chunk_readline(self): + i = self.make_input(["1", "2"]) + self.assertRaises(IOError, i.readline) + + def test_chunked_short_chunk(self): + i = self.make_input("2\r\n1", chunked_input=True) + self.assertRaises(IOError, i.read) + + def test_chunked_short_chunk_read_with_length(self): + i = self.make_input("2\r\n1", chunked_input=True) + self.assertRaises(IOError, i.read, 2) + + def test_chunked_short_chunk_readline(self): + i = self.make_input("2\r\n1", chunked_input=True) + self.assertRaises(IOError, i.readline) + + def test_32bit_overflow(self): + # https://github.com/gevent/gevent/issues/289 + # Should not raise an OverflowError on Python 2 + print("BEGIN 32bit") + data = b'asdf\nghij\n' + long_data = b'a' * (pywsgi.MAX_REQUEST_LINE + 10) + long_data += b'\n' + data = data + long_data + partial_data = b'qjk\n' # Note terminating \n + n = 25 * 1000000000 + print("N", n, "Data len", len(data)) + if hasattr(n, 'bit_length'): + self.assertEqual(n.bit_length(), 35) + if not PY3 and not PYPY: + # Make sure we have the impl we think we do + self.assertRaises(OverflowError, StringIO(data).readline, n) + + i = self.make_input(data, content_length=n) + # No size hint, but we have too large a content_length to fit + self.assertEqual(i.readline(), b'asdf\n') + # Large size hint + self.assertEqual(i.readline(n), b'ghij\n') + self.assertEqual(i.readline(n), long_data) + + # Now again with the real content length, assuring we can't read past it + i = self.make_input(data + partial_data, content_length=len(data) + 1) + self.assertEqual(i.readline(), b'asdf\n') + self.assertEqual(i.readline(n), b'ghij\n') + self.assertEqual(i.readline(n), long_data) + # Now we've reached content_length so we shouldn't be able to + # read anymore except the one byte remaining + self.assertEqual(i.readline(n), b'q') + + +class Test414(TestCase): + + @staticmethod + def application(env, start_response): + raise AssertionError('should not get there') + + def test(self): + fd = self.makefile() + longline = 'x' * 20000 + fd.write(('''GET /%s HTTP/1.0\r\nHello: world\r\n\r\n''' % longline).encode('latin-1')) + read_http(fd, code=414) + + +class TestLogging(TestCase): + + # Something that gets wrapped in a LoggingLogAdapter + class Logger(object): + accessed = None + logged = None + thing = None + + def log(self, level, msg): + self.logged = (level, msg) + + def access(self, msg): + self.accessed = msg + + def get_thing(self): + return self.thing + + def init_logger(self): + return self.Logger() + + @staticmethod + def application(env, start_response): + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [b'hello'] + + # Tests for issue #663 + + def test_proxy_methods_on_log(self): + # An object that looks like a logger gets wrapped + # with a proxy that + self.assertTrue(isinstance(self.server.log, pywsgi.LoggingLogAdapter)) + self.server.log.access("access") + self.server.log.write("write") + self.assertEqual(self.server.log.accessed, "access") + self.assertEqual(self.server.log.logged, (20, "write")) + + def test_set_attributes(self): + # Not defined by the wrapper, it goes to the logger + self.server.log.thing = 42 + self.assertEqual(self.server.log.get_thing(), 42) + + del self.server.log.thing + self.assertEqual(self.server.log.get_thing(), None) + + def test_status_log(self): + # Issue 664: Make sure we format the status line as a string + self.urlopen() + msg = self.server.log.logged[1] + self.assertTrue('"GET / HTTP/1.1" 200 ' in msg, msg) + + # Issue 756: Make sure we don't throw a newline on the end + self.assertTrue('\n' not in msg, msg) + +class TestEnviron(TestCase): + + # The wsgiref validator asserts type(environ) is dict. + # https://mail.python.org/pipermail/web-sig/2016-March/005455.html + validator = None + + def init_server(self, application): + super(TestEnviron, self).init_server(application) + self.server.environ_class = pywsgi.SecureEnviron + + def application(self, env, start_response): + self.assertIsInstance(env, pywsgi.SecureEnviron) + start_response('200 OK', [('Content-Type', 'text/plain')]) + return [] + + def test_environ_is_secure_by_default(self): + self.urlopen() + + def test_default_secure_repr(self): + environ = pywsgi.SecureEnviron() + self.assertIn('"}), str(environ)) + self.assertEqual(repr({'key': ""}), repr(environ)) + + environ.whitelist_keys = ('key',) + self.assertEqual(str({'key': 'value'}), str(environ)) + self.assertEqual(repr({'key': 'value'}), repr(environ)) + + del environ.whitelist_keys + + def test_override_class_defaults(self): + class EnvironClass(pywsgi.SecureEnviron): + __slots__ = () + + environ = EnvironClass() + + self.assertTrue(environ.secure_repr) + EnvironClass.default_secure_repr = False + self.assertFalse(environ.secure_repr) + + self.assertEqual(str({}), str(environ)) + self.assertEqual(repr({}), repr(environ)) + + EnvironClass.default_secure_repr = True + EnvironClass.default_whitelist_keys = ('key',) + + environ['key'] = 1 + self.assertEqual(str({'key': 1}), str(environ)) + self.assertEqual(repr({'key': 1}), repr(environ)) + + # Clean up for leaktests + del environ + del EnvironClass + import gc; gc.collect() + + + def test_copy_still_secure(self): + for cls in (pywsgi.Environ, pywsgi.SecureEnviron): + self.assertIsInstance(cls().copy(), cls) + + def test_pickle_copy_returns_dict(self): + # Anything going through copy.copy/pickle should + # return the same pickle that a dict would. + import pickle + import json + + for cls in (pywsgi.Environ, pywsgi.SecureEnviron): + bltin = {'key': 'value'} + env = cls(bltin) + self.assertIsInstance(env, cls) + self.assertEqual(bltin, env) + self.assertEqual(env, bltin) + + for protocol in range(0, pickle.HIGHEST_PROTOCOL + 1): + # It's impossible to get a subclass of dict to pickle + # identically, but it can restore identically + env_dump = pickle.dumps(env, protocol) + self.assertNotIn(b'Environ', env_dump) + loaded = pickle.loads(env_dump) + self.assertEqual(type(loaded), dict) + + self.assertEqual(json.dumps(bltin), json.dumps(env)) + +del CommonTests + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__queue.py b/src/gevent/tests/test__queue.py new file mode 100644 index 0000000..03c846f --- /dev/null +++ b/src/gevent/tests/test__queue.py @@ -0,0 +1,465 @@ +import unittest + +import gevent.testing as greentest +from gevent.testing import TestCase, main +import gevent +from gevent.hub import get_hub, LoopExit +from gevent import util +from gevent import queue +from gevent.queue import Empty, Full +from gevent.event import AsyncResult +from gevent.testing.timing import AbstractGenericGetTestCase + +# pylint:disable=too-many-ancestors + +class TestQueue(TestCase): + + def test_send_first(self): + self.switch_expected = False + q = queue.Queue() + q.put('hi') + self.assertEqual(q.peek(), 'hi') + self.assertEqual(q.get(), 'hi') + + def test_peek_empty(self): + q = queue.Queue() + # No putters waiting, in the main loop: LoopExit + self.assertRaises(LoopExit, q.peek) + + def waiter(q): + self.assertRaises(Empty, q.peek, timeout=0.01) + g = gevent.spawn(waiter, q) + gevent.sleep(0.1) + g.join() + + def test_peek_multi_greenlet(self): + q = queue.Queue() + g = gevent.spawn(q.peek) + g.start() + gevent.sleep(0) + q.put(1) + g.join() + self.assertTrue(g.exception is None) + self.assertEqual(q.peek(), 1) + + def test_send_last(self): + q = queue.Queue() + + def waiter(q): + with gevent.Timeout(0.1 if not greentest.RUNNING_ON_APPVEYOR else 0.5): + self.assertEqual(q.get(), 'hi2') + return "OK" + + p = gevent.spawn(waiter, q) + gevent.sleep(0.01) + q.put('hi2') + gevent.sleep(0.01) + assert p.get(timeout=0) == "OK" + + def test_max_size(self): + q = queue.Queue(2) + results = [] + + def putter(q): + q.put('a') + results.append('a') + q.put('b') + results.append('b') + q.put('c') + results.append('c') + return "OK" + + p = gevent.spawn(putter, q) + gevent.sleep(0) + self.assertEqual(results, ['a', 'b']) + self.assertEqual(q.get(), 'a') + gevent.sleep(0) + self.assertEqual(results, ['a', 'b', 'c']) + self.assertEqual(q.get(), 'b') + self.assertEqual(q.get(), 'c') + assert p.get(timeout=0) == "OK" + + def test_zero_max_size(self): + q = queue.Channel() + + def sender(evt, q): + q.put('hi') + evt.set('done') + + def receiver(evt, q): + x = q.get() + evt.set(x) + + e1 = AsyncResult() + e2 = AsyncResult() + + p1 = gevent.spawn(sender, e1, q) + gevent.sleep(0.001) + self.assertTrue(not e1.ready()) + p2 = gevent.spawn(receiver, e2, q) + self.assertEqual(e2.get(), 'hi') + self.assertEqual(e1.get(), 'done') + with gevent.Timeout(0): + gevent.joinall([p1, p2]) + + def test_multiple_waiters(self): + # tests that multiple waiters get their results back + q = queue.Queue() + + def waiter(q, evt): + evt.set(q.get()) + + sendings = ['1', '2', '3', '4'] + evts = [AsyncResult() for x in sendings] + for i, _ in enumerate(sendings): + gevent.spawn(waiter, q, evts[i]) # XXX use waitall for them + + gevent.sleep(0.01) # get 'em all waiting + + results = set() + + def collect_pending_results(): + for e in evts: + with gevent.Timeout(0.001, False): + x = e.get() + results.add(x) + return len(results) + + q.put(sendings[0]) + self.assertEqual(collect_pending_results(), 1) + q.put(sendings[1]) + self.assertEqual(collect_pending_results(), 2) + q.put(sendings[2]) + q.put(sendings[3]) + self.assertEqual(collect_pending_results(), 4) + + def test_waiters_that_cancel(self): + q = queue.Queue() + + def do_receive(q, evt): + with gevent.Timeout(0, RuntimeError()): + try: + result = q.get() + evt.set(result) # pragma: no cover (should have raised) + except RuntimeError: + evt.set('timed out') + + evt = AsyncResult() + gevent.spawn(do_receive, q, evt) + self.assertEqual(evt.get(), 'timed out') + + q.put('hi') + self.assertEqual(q.get(), 'hi') + + def test_senders_that_die(self): + q = queue.Queue() + + def do_send(q): + q.put('sent') + + gevent.spawn(do_send, q) + self.assertEqual(q.get(), 'sent') + + def test_two_waiters_one_dies(self): + + def waiter(q, evt): + evt.set(q.get()) + + def do_receive(q, evt): + with gevent.Timeout(0, RuntimeError()): + try: + result = q.get() + evt.set(result) # pragma: no cover (should have raised) + except RuntimeError: + evt.set('timed out') + + q = queue.Queue() + dying_evt = AsyncResult() + waiting_evt = AsyncResult() + gevent.spawn(do_receive, q, dying_evt) + gevent.spawn(waiter, q, waiting_evt) + gevent.sleep(0.1) + q.put('hi') + self.assertEqual(dying_evt.get(), 'timed out') + self.assertEqual(waiting_evt.get(), 'hi') + + def test_two_bogus_waiters(self): + def do_receive(q, evt): + with gevent.Timeout(0, RuntimeError()): + try: + result = q.get() + evt.set(result) # pragma: no cover (should have raised) + except RuntimeError: + evt.set('timed out') + + q = queue.Queue() + e1 = AsyncResult() + e2 = AsyncResult() + gevent.spawn(do_receive, q, e1) + gevent.spawn(do_receive, q, e2) + gevent.sleep(0.1) + q.put('sent') + self.assertEqual(e1.get(), 'timed out') + self.assertEqual(e2.get(), 'timed out') + self.assertEqual(q.get(), 'sent') + + +class TestChannel(TestCase): + + def test_send(self): + channel = queue.Channel() + + events = [] + + def another_greenlet(): + events.append(channel.get()) + events.append(channel.get()) + + g = gevent.spawn(another_greenlet) + + events.append('sending') + channel.put('hello') + events.append('sent hello') + channel.put('world') + events.append('sent world') + + self.assertEqual(['sending', 'hello', 'sent hello', 'world', 'sent world'], events) + g.get() + + def test_wait(self): + channel = queue.Channel() + events = [] + + def another_greenlet(): + events.append('sending hello') + channel.put('hello') + events.append('sending world') + channel.put('world') + events.append('sent world') + + g = gevent.spawn(another_greenlet) + + events.append('waiting') + events.append(channel.get()) + events.append(channel.get()) + + self.assertEqual(['waiting', 'sending hello', 'hello', 'sending world', 'world'], events) + gevent.sleep(0) + self.assertEqual(['waiting', 'sending hello', 'hello', 'sending world', 'world', 'sent world'], events) + g.get() + + def test_iterable(self): + channel = queue.Channel() + gevent.spawn(channel.put, StopIteration) + r = list(channel) + self.assertEqual(r, []) + +class TestJoinableQueue(TestCase): + + def test_task_done(self): + channel = queue.JoinableQueue() + X = object() + gevent.spawn(channel.put, X) + result = channel.get() + self.assertIs(result, X) + self.assertEqual(1, channel.unfinished_tasks) + channel.task_done() + self.assertEqual(0, channel.unfinished_tasks) + + +class TestNoWait(TestCase): + + def test_put_nowait_simple(self): + result = [] + q = queue.Queue(1) + + def store_result(func, *args): + result.append(func(*args)) + + run_callback = get_hub().loop.run_callback + + run_callback(store_result, util.wrap_errors(Full, q.put_nowait), 2) + run_callback(store_result, util.wrap_errors(Full, q.put_nowait), 3) + gevent.sleep(0) + assert len(result) == 2, result + assert result[0] is None, result + assert isinstance(result[1], queue.Full), result + + def test_get_nowait_simple(self): + result = [] + q = queue.Queue(1) + q.put(4) + + def store_result(func, *args): + result.append(func(*args)) + + run_callback = get_hub().loop.run_callback + + run_callback(store_result, util.wrap_errors(Empty, q.get_nowait)) + run_callback(store_result, util.wrap_errors(Empty, q.get_nowait)) + gevent.sleep(0) + assert len(result) == 2, result + assert result[0] == 4, result + assert isinstance(result[1], queue.Empty), result + + # get_nowait must work from the mainloop + def test_get_nowait_unlock(self): + result = [] + q = queue.Queue(1) + p = gevent.spawn(q.put, 5) + + def store_result(func, *args): + result.append(func(*args)) + + assert q.empty(), q + gevent.sleep(0) + assert q.full(), q + get_hub().loop.run_callback(store_result, q.get_nowait) + gevent.sleep(0) + assert q.empty(), q + assert result == [5], result + assert p.ready(), p + assert p.dead, p + assert q.empty(), q + + def test_get_nowait_unlock_channel(self): + result = [] + q = queue.Channel() + p = gevent.spawn(q.put, 5) + + def store_result(func, *args): + result.append(func(*args)) + + assert q.empty(), q + assert q.full(), q + gevent.sleep(0.001) + assert q.empty(), q + assert q.full(), q + get_hub().loop.run_callback(store_result, q.get_nowait) + gevent.sleep(0.001) + assert q.empty(), q + assert q.full(), q + assert result == [5], result + assert p.ready(), p + assert p.dead, p + assert q.empty(), q + + # put_nowait must work from the mainloop + def test_put_nowait_unlock(self): + result = [] + q = queue.Queue() + p = gevent.spawn(q.get) + + def store_result(func, *args): + result.append(func(*args)) + + self.assertTrue(q.empty(), q) + self.assertFalse(q.full(), q) + gevent.sleep(0.001) + + self.assertTrue(q.empty(), q) + self.assertFalse(q.full(), q) + + get_hub().loop.run_callback(store_result, q.put_nowait, 10) + + self.assertFalse(p.ready(), p) + gevent.sleep(0.001) + + self.assertEqual(result, [None]) + self.assertTrue(p.ready(), p) + self.assertFalse(q.full(), q) + self.assertTrue(q.empty(), q) + + +class TestJoinEmpty(TestCase): + + def test_issue_45(self): + """Test that join() exits immediately if not jobs were put into the queue""" + self.switch_expected = False + q = queue.JoinableQueue() + q.join() + +class AbstractTestWeakRefMixin(object): + + def test_weak_reference(self): + import weakref + one = self._makeOne() + ref = weakref.ref(one) + self.assertIs(one, ref()) + + +class TestGetInterrupt(AbstractTestWeakRefMixin, AbstractGenericGetTestCase): + + Timeout = Empty + + kind = queue.Queue + + def wait(self, timeout): + return self._makeOne().get(timeout=timeout) + + def _makeOne(self): + return self.kind() + +class TestGetInterruptJoinableQueue(TestGetInterrupt): + kind = queue.JoinableQueue + +class TestGetInterruptLifoQueue(TestGetInterrupt): + kind = queue.LifoQueue + +class TestGetInterruptPriorityQueue(TestGetInterrupt): + kind = queue.PriorityQueue + +class TestGetInterruptChannel(TestGetInterrupt): + kind = queue.Channel + + +class TestPutInterrupt(AbstractGenericGetTestCase): + kind = queue.Queue + Timeout = Full + + def setUp(self): + super(TestPutInterrupt, self).setUp() + self.queue = self._makeOne() + + def wait(self, timeout): + while not self.queue.full(): + self.queue.put(1) + return self.queue.put(2, timeout=timeout) + + def _makeOne(self): + return self.kind(1) + + +class TestPutInterruptJoinableQueue(TestPutInterrupt): + kind = queue.JoinableQueue + +class TestPutInterruptLifoQueue(TestPutInterrupt): + kind = queue.LifoQueue + +class TestPutInterruptPriorityQueue(TestPutInterrupt): + kind = queue.PriorityQueue + +class TestPutInterruptChannel(TestPutInterrupt): + kind = queue.Channel + + def _makeOne(self): + return self.kind() + + +if hasattr(queue, 'SimpleQueue'): + + class TestGetInterruptSimpleQueue(TestGetInterrupt): + kind = queue.SimpleQueue + + def test_raises_timeout_Timeout(self): + raise unittest.SkipTest("Not supported") + + test_raises_timeout_Timeout_exc_customized = test_raises_timeout_Timeout + test_outer_timeout_is_not_lost = test_raises_timeout_Timeout + + +del AbstractGenericGetTestCase + + +if __name__ == '__main__': + main() diff --git a/src/gevent/tests/test__real_greenlet.py b/src/gevent/tests/test__real_greenlet.py new file mode 100644 index 0000000..9a9e5b4 --- /dev/null +++ b/src/gevent/tests/test__real_greenlet.py @@ -0,0 +1,29 @@ +"""Testing that greenlet restores sys.exc_info. + +Passes with CPython + greenlet 0.4.0 + +Fails with PyPy 2.2.1 +""" +from __future__ import print_function +import sys +import greenlet + + +print('Your greenlet version: %s' % (getattr(greenlet, '__version__', None), )) + + +result = [] + + +def func(): + result.append(repr(sys.exc_info())) + + +g = greenlet.greenlet(func) +try: + 1 / 0 +except ZeroDivisionError: + g.switch() + + +assert result == ['(None, None, None)'], result diff --git a/src/gevent/tests/test__refcount.py b/src/gevent/tests/test__refcount.py new file mode 100644 index 0000000..e17013d --- /dev/null +++ b/src/gevent/tests/test__refcount.py @@ -0,0 +1,162 @@ +# Copyright (c) 2008 AG Projects +# Author: Denis Bilenko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +"""This test checks that underlying socket instances (gevent.socket.socket._sock) +are not leaked by the hub. +""" +from __future__ import print_function +from _socket import socket as c_socket +import sys +if sys.version_info[0] >= 3: + # Python3 enforces that __weakref__ appears only once, + # and not when a slotted class inherits from an unslotted class. + # We mess around with the class MRO below and violate that rule + # (because socket.socket defines __slots__ with __weakref__), + # so import socket.socket before that can happen. + __import__('socket') + Socket = c_socket +else: + class Socket(c_socket): + "Something we can have a weakref to" + +import _socket +_socket.socket = Socket + +import gevent.testing as greentest +from gevent import monkey; monkey.patch_all() +from gevent.testing import flaky + +from pprint import pformat +try: + from thread import start_new_thread +except ImportError: + from _thread import start_new_thread +from time import sleep +import weakref +import gc + +import socket +socket._realsocket = Socket + +SOCKET_TIMEOUT = 0.1 +if greentest.RUNNING_ON_CI: + SOCKET_TIMEOUT *= 2 + +def init_server(): + s = socket.socket() + s.settimeout(SOCKET_TIMEOUT) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(('127.0.0.1', 0)) + s.listen(5) + return s + + +def handle_request(s, raise_on_timeout): + try: + conn, _ = s.accept() + except socket.timeout: + if raise_on_timeout: + raise + else: + return + #print('handle_request - accepted') + res = conn.recv(100) + assert res == b'hello', repr(res) + #print('handle_request - recvd %r' % res) + res = conn.send(b'bye') + #print('handle_request - sent %r' % res) + #print('handle_request - conn refcount: %s' % sys.getrefcount(conn)) + conn.close() + + +def make_request(port): + #print('make_request') + s = socket.socket() + s.connect(('127.0.0.1', port)) + #print('make_request - connected') + res = s.send(b'hello') + #print('make_request - sent %s' % res) + res = s.recv(100) + assert res == b'bye', repr(res) + #print('make_request - recvd %r' % res) + s.close() + + +def run_interaction(run_client): + s = init_server() + start_new_thread(handle_request, (s, run_client)) + if run_client: + port = s.getsockname()[1] + start_new_thread(make_request, (port, )) + sleep(0.1 + SOCKET_TIMEOUT) + #print(sys.getrefcount(s._sock)) + #s.close() + w = weakref.ref(s._sock) + s.close() + if greentest.WIN: + # The background thread may not have even had a chance to run + # yet, sleep again to be sure it does. Otherwise there could be + # strong refs to the socket still around. + try: + sleep(0.1 + SOCKET_TIMEOUT) + except Exception: # pylint:disable=broad-except + pass + + return w + + +def run_and_check(run_client): + w = run_interaction(run_client=run_client) + if greentest.PYPY: + # PyPy doesn't use a strict ref counting and must + # run a gc, but the object should be gone + gc.collect() + if w(): + print(pformat(gc.get_referrers(w()))) + for x in gc.get_referrers(w()): + print(pformat(x)) + for y in gc.get_referrers(x): + print('-', pformat(y)) + raise AssertionError('server should be dead by now') + + +@greentest.skipOnCI("Often fail with timeouts or force closed connections; not sure why.") +@greentest.skipIf( + greentest.RUN_LEAKCHECKS and greentest.PY3, + "Often fail with force closed connections; not sure why. " +) +class Test(greentest.TestCase): + + __timeout__ = greentest.LARGE_TIMEOUT + + @flaky.reraises_flaky_timeout(socket.timeout) + def test_clean_exit(self): + run_and_check(True) + run_and_check(True) + + @flaky.reraises_flaky_timeout(socket.timeout) + def test_timeout_exit(self): + run_and_check(False) + run_and_check(False) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__refcount_core.py b/src/gevent/tests/test__refcount_core.py new file mode 100644 index 0000000..edf8f40 --- /dev/null +++ b/src/gevent/tests/test__refcount_core.py @@ -0,0 +1,24 @@ +import weakref + + +class Dummy: + def __init__(self): + __import__('gevent.core') + +try: + assert weakref.ref(Dummy())() is None + + from gevent import socket + s = socket.socket() + r = weakref.ref(s) + s.close() + del s + assert r() is None +except AssertionError: # pragma: no cover + import sys + if hasattr(sys, 'pypy_version_info'): + # PyPy uses a non refcounted GC which may defer + # the collection of the weakref, unlike CPython + pass + else: + raise diff --git a/src/gevent/tests/test__select.py b/src/gevent/tests/test__select.py new file mode 100644 index 0000000..d211023 --- /dev/null +++ b/src/gevent/tests/test__select.py @@ -0,0 +1,114 @@ +from gevent.testing import six +import sys +import os +import errno +from gevent import select, socket +import gevent.core +import gevent.testing as greentest +import gevent.testing.timing +import unittest + + +class TestSelect(gevent.testing.timing.AbstractGenericWaitTestCase): + + def wait(self, timeout): + select.select([], [], [], timeout) + + + +@greentest.skipOnWindows("Cant select on files") +class TestSelectRead(gevent.testing.timing.AbstractGenericWaitTestCase): + + def wait(self, timeout): + r, w = os.pipe() + try: + select.select([r], [], [], timeout) + finally: + os.close(r) + os.close(w) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + # Backported from test_select.py in 3.4 + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + try: + select.select([fd], [], [], 0) + except OSError as err: + # Python 3 + self.assertEqual(err.errno, errno.EBADF) + except select.error as err: # pylint:disable=duplicate-except + # Python 2 (select.error is OSError on py3) + self.assertEqual(err.args[0], errno.EBADF) + else: + self.fail("exception not raised") + + +@unittest.skipUnless(hasattr(select, 'poll'), "Needs poll") +@greentest.skipOnWindows("Cant poll on files") +class TestPollRead(gevent.testing.timing.AbstractGenericWaitTestCase): + def wait(self, timeout): + # On darwin, the read pipe is reported as writable + # immediately, for some reason. So we carefully register + # it only for read events (the default is read and write) + r, w = os.pipe() + try: + poll = select.poll() + poll.register(r, select.POLLIN) + poll.poll(timeout * 1000) + finally: + poll.unregister(r) + os.close(r) + os.close(w) + + def test_unregister_never_registered(self): + # "Attempting to remove a file descriptor that was + # never registered causes a KeyError exception to be + # raised." + poll = select.poll() + self.assertRaises(KeyError, poll.unregister, 5) + + @unittest.skipIf(hasattr(gevent.core, 'libuv'), + "Depending on whether the fileno is reused or not this either crashes or does nothing." + "libuv won't open a watcher for a closed file on linux.") + def test_poll_invalid(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + + poll = select.poll() + poll.register(fd, select.POLLIN) + # Close after registering; libuv refuses to even + # create a watcher if it would get EBADF (so this turns into + # a test of whether or not we successfully initted the watcher). + fp.close() + result = poll.poll(0) + self.assertEqual(result, [(fd, select.POLLNVAL)]) # pylint:disable=no-member + +class TestSelectTypes(greentest.TestCase): + + def test_int(self): + sock = socket.socket() + try: + select.select([int(sock.fileno())], [], [], 0.001) + finally: + sock.close() + + if hasattr(six.builtins, 'long'): + def test_long(self): + sock = socket.socket() + try: + select.select( + [six.builtins.long(sock.fileno())], [], [], 0.001) + finally: + sock.close() + + def test_string(self): + self.switch_expected = False + self.assertRaises(TypeError, select.select, ['hello'], [], [], 0.001) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__semaphore.py b/src/gevent/tests/test__semaphore.py new file mode 100644 index 0000000..83c8f94 --- /dev/null +++ b/src/gevent/tests/test__semaphore.py @@ -0,0 +1,91 @@ +import gevent.testing as greentest +import gevent +from gevent.lock import Semaphore +from gevent.thread import allocate_lock +import weakref +try: + from _thread import allocate_lock as std_allocate_lock +except ImportError: # Py2 + from thread import allocate_lock as std_allocate_lock + +# pylint:disable=broad-except + +class TestSemaphore(greentest.TestCase): + + # issue 39 + def test_acquire_returns_false_after_timeout(self): + s = Semaphore(value=0) + result = s.acquire(timeout=0.01) + assert result is False, repr(result) + + def test_release_twice(self): + s = Semaphore() + result = [] + s.rawlink(lambda s: result.append('a')) + s.release() + s.rawlink(lambda s: result.append('b')) + s.release() + gevent.sleep(0.001) + # The order, though, is not guaranteed. + self.assertEqual(sorted(result), ['a', 'b']) + + def test_semaphore_weakref(self): + s = Semaphore() + r = weakref.ref(s) + self.assertEqual(s, r()) + + def test_semaphore_in_class_with_del(self): + # Issue #704. This used to crash the process + # under PyPy through at least 4.0.1 if the Semaphore + # was implemented with Cython. + class X(object): + def __init__(self): + self.s = Semaphore() + + def __del__(self): + self.s.acquire() + + X() + import gc + gc.collect() + gc.collect() + + test_semaphore_in_class_with_del.ignore_leakcheck = True + + def test_rawlink_on_unacquired_runs_notifiers(self): + # https://github.com/gevent/gevent/issues/1287 + + # Rawlinking a ready semaphore should fire immediately, + # not raise LoopExit + s = Semaphore() + gevent.wait([s]) + +class TestLock(greentest.TestCase): + + def test_release_unheld_lock(self): + std_lock = std_allocate_lock() + g_lock = allocate_lock() + try: + std_lock.release() + self.fail("Should have thrown an exception") + except Exception as e: + std_exc = e + + try: + g_lock.release() + self.fail("Should have thrown an exception") + except Exception as e: + g_exc = e + self.assertIsInstance(g_exc, type(std_exc)) + + +@greentest.skipOnPurePython("Needs C extension") +class TestCExt(greentest.TestCase): + + def test_c_extension(self): + self.assertEqual(Semaphore.__module__, + 'gevent.__semaphore') + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__server.py b/src/gevent/tests/test__server.py new file mode 100644 index 0000000..2ccdc85 --- /dev/null +++ b/src/gevent/tests/test__server.py @@ -0,0 +1,515 @@ +from __future__ import print_function, division +import unittest +import errno +import os + + +import gevent.testing as greentest +from gevent.testing import PY3 +from gevent.testing import DEFAULT_SOCKET_TIMEOUT as _DEFAULT_SOCKET_TIMEOUT +from gevent import socket +import gevent +from gevent.server import StreamServer + + +class SimpleStreamServer(StreamServer): + + def handle(self, client_socket, _address): # pylint:disable=method-hidden + fd = client_socket.makefile() + try: + request_line = fd.readline() + if not request_line: + return + try: + _method, path, _rest = request_line.split(' ', 3) + except Exception: + print('Failed to parse request line: %r' % (request_line, )) + raise + if path == '/ping': + client_socket.sendall(b'HTTP/1.0 200 OK\r\n\r\nPONG') + elif path in ['/long', '/short']: + client_socket.sendall(b'hello') + while True: + data = client_socket.recv(1) + if not data: + break + else: + client_socket.sendall(b'HTTP/1.0 404 WTF?\r\n\r\n') + finally: + fd.close() + + +class Settings(object): + ServerClass = StreamServer + ServerSubClass = SimpleStreamServer + restartable = True + close_socket_detected = True + + @staticmethod + def assertAcceptedConnectionError(inst): + conn = inst.makefile() + result = conn.read() + inst.assertFalse(result) + + assert500 = assertAcceptedConnectionError + + @staticmethod + def assert503(inst): + # regular reads timeout + inst.assert500() + # attempt to send anything reset the connection + try: + inst.send_request() + except socket.error as ex: + if ex.args[0] not in greentest.CONN_ABORTED_ERRORS: + raise + + @staticmethod + def assertPoolFull(inst): + with inst.assertRaises(socket.timeout): + inst.assertRequestSucceeded(timeout=0.01) + + @staticmethod + def fill_default_server_args(inst, kwargs): + kwargs.setdefault('spawn', inst.get_spawn()) + return kwargs + +class TestCase(greentest.TestCase): + # pylint: disable=too-many-public-methods + __timeout__ = greentest.LARGE_TIMEOUT + Settings = Settings + server = None + + def cleanup(self): + if getattr(self, 'server', None) is not None: + self.server.stop() + self.server = None + + def get_listener(self): + sock = socket.socket() + sock.bind(('127.0.0.1', 0)) + sock.listen(5) + self._close_on_teardown(sock) + return sock + + def get_server_host_port_family(self): + server_host = self.server.server_host + if not server_host: + server_host = greentest.DEFAULT_LOCAL_HOST_ADDR + elif server_host == '::': + server_host = greentest.DEFAULT_LOCAL_HOST_ADDR6 + + try: + family = self.server.socket.family + except AttributeError: + # server deletes socket when closed + family = socket.AF_INET + + return server_host, self.server.server_port, family + + def makefile(self, timeout=_DEFAULT_SOCKET_TIMEOUT, bufsize=1): + server_host, server_port, family = self.get_server_host_port_family() + + sock = socket.socket(family=family) + try: + sock.connect((server_host, server_port)) + except Exception: + # avoid ResourceWarning under Py3 + sock.close() + raise + + if PY3: + # Under Python3, you can't read and write to the same + # makefile() opened in r, and r+ is not allowed + kwargs = {'buffering': bufsize, 'mode': 'rwb'} + else: + kwargs = {'bufsize': bufsize} + + rconn = sock.makefile(**kwargs) + if PY3: + rconn._sock = sock + rconn._sock.settimeout(timeout) + sock.close() + return rconn + + def send_request(self, url='/', timeout=_DEFAULT_SOCKET_TIMEOUT, bufsize=1): + conn = self.makefile(timeout=timeout, bufsize=bufsize) + conn.write(('GET %s HTTP/1.0\r\n\r\n' % url).encode('latin-1')) + conn.flush() + return conn + + def assertConnectionRefused(self): + with self.assertRaises(socket.error) as exc: + conn = self.makefile() + conn.close() + + ex = exc.exception + self.assertIn(ex.args[0], (errno.ECONNREFUSED, errno.EADDRNOTAVAIL)) + + def assert500(self): + self.Settings.assert500(self) + + def assert503(self): + self.Settings.assert503(self) + + def assertAcceptedConnectionError(self): + self.Settings.assertAcceptedConnectionError(self) + + def assertPoolFull(self): + self.Settings.assertPoolFull(self) + + def assertNotAccepted(self): + conn = self.makefile() + conn.write(b'GET / HTTP/1.0\r\n\r\n') + conn.flush() + result = b'' + try: + while True: + data = conn._sock.recv(1) + if not data: + break + result += data + except socket.timeout: + self.assertFalse(result) + return + self.assertTrue(result.startswith(b'HTTP/1.0 500 Internal Server Error'), repr(result)) + conn.close() + + def assertRequestSucceeded(self, timeout=_DEFAULT_SOCKET_TIMEOUT): + conn = self.makefile(timeout=timeout) + conn.write(b'GET /ping HTTP/1.0\r\n\r\n') + result = conn.read() + conn.close() + assert result.endswith(b'\r\n\r\nPONG'), repr(result) + + def start_server(self): + self.server.start() + self.assertRequestSucceeded() + self.assertRequestSucceeded() + + def stop_server(self): + self.server.stop() + self.assertConnectionRefused() + + def report_netstat(self, _msg): + # At one point this would call 'sudo netstat -anp | grep PID' + # with os.system. We can probably do better with psutil. + return + + def _create_server(self): + return self.ServerSubClass((greentest.DEFAULT_BIND_ADDR, 0)) + + def init_server(self): + self.server = self._create_server() + self.server.start() + gevent.sleep(0.01) + + @property + def socket(self): + return self.server.socket + + def _test_invalid_callback(self): + try: + self.server = self.ServerClass((greentest.DEFAULT_BIND_ADDR, 0), lambda: None) + self.server.start() + + self.expect_one_error() + + self.assert500() + self.assert_error(TypeError) + finally: + self.server.stop() + # XXX: There's something unreachable (with a traceback?) + # We need to clear it to make the leak checks work on Travis; + # so far I can't reproduce it locally on OS X. + import gc; gc.collect() + + def fill_default_server_args(self, kwargs): + return self.Settings.fill_default_server_args(self, kwargs) + + def ServerClass(self, *args, **kwargs): + return self.Settings.ServerClass(*args, + **self.fill_default_server_args(kwargs)) + + def ServerSubClass(self, *args, **kwargs): + return self.Settings.ServerSubClass(*args, + **self.fill_default_server_args(kwargs)) + + def get_spawn(self): + return None + +class TestDefaultSpawn(TestCase): + + def get_spawn(self): + return gevent.spawn + + def _test_server_start_stop(self, restartable): + self.report_netstat('before start') + self.start_server() + self.report_netstat('after start') + if restartable and self.Settings.restartable: + self.server.stop_accepting() + self.report_netstat('after stop_accepting') + self.assertNotAccepted() + self.server.start_accepting() + self.report_netstat('after start_accepting') + self.assertRequestSucceeded() + self.stop_server() + self.report_netstat('after stop') + + def test_backlog_is_not_accepted_for_socket(self): + self.switch_expected = False + with self.assertRaises(TypeError): + self.ServerClass(self.get_listener(), backlog=25, handle=False) + + def test_backlog_is_accepted_for_address(self): + self.server = self.ServerSubClass((greentest.DEFAULT_BIND_ADDR, 0), backlog=25) + self.assertConnectionRefused() + self._test_server_start_stop(restartable=False) + + def test_subclass_just_create(self): + self.server = self.ServerSubClass(self.get_listener()) + self.assertNotAccepted() + + def test_subclass_with_socket(self): + self.server = self.ServerSubClass(self.get_listener()) + # the connection won't be refused, because there exists a + # listening socket, but it won't be handled also + self.assertNotAccepted() + self._test_server_start_stop(restartable=True) + + def test_subclass_with_address(self): + self.server = self.ServerSubClass((greentest.DEFAULT_BIND_ADDR, 0)) + self.assertConnectionRefused() + self._test_server_start_stop(restartable=True) + + def test_invalid_callback(self): + self._test_invalid_callback() + + @greentest.reraises_flaky_timeout(socket.timeout) + def _test_serve_forever(self): + g = gevent.spawn(self.server.serve_forever) + try: + gevent.sleep(0.01) + self.assertRequestSucceeded() + self.server.stop() + assert not self.server.started + self.assertConnectionRefused() + finally: + g.kill() + g.get() + + def test_serve_forever(self): + self.server = self.ServerSubClass(('127.0.0.1', 0)) + assert not self.server.started + self.assertConnectionRefused() + self._test_serve_forever() + + def test_serve_forever_after_start(self): + self.server = self.ServerSubClass((greentest.DEFAULT_BIND_ADDR, 0)) + self.assertConnectionRefused() + assert not self.server.started + self.server.start() + assert self.server.started + self._test_serve_forever() + + def test_server_closes_client_sockets(self): + self.server = self.ServerClass((greentest.DEFAULT_BIND_ADDR, 0), lambda *args: []) + self.server.start() + conn = self.send_request() + # use assert500 below? + with gevent.Timeout._start_new_or_dummy(1): + try: + result = conn.read() + if result: + assert result.startswith('HTTP/1.0 500 Internal Server Error'), repr(result) + except socket.error as ex: + if ex.args[0] == 10053: + pass # "established connection was aborted by the software in your host machine" + elif ex.args[0] == errno.ECONNRESET: + pass + else: + raise + finally: + conn.close() + + self.stop_server() + + def init_server(self): + self.server = self._create_server() + self.server.start() + gevent.sleep(0.01) + + @property + def socket(self): + return self.server.socket + + def test_error_in_spawn(self): + self.init_server() + self.assertTrue(self.server.started) + error = ExpectedError('test_error_in_spawn') + self.server._spawn = lambda *args: gevent.getcurrent().throw(error) + self.expect_one_error() + self.assertAcceptedConnectionError() + self.assert_error(ExpectedError, error) + + def test_server_repr_when_handle_is_instancemethod(self): + # PR 501 + self.init_server() + self.start_server() + self.assertIn('Server', repr(self.server)) + + self.server.set_handle(self.server.handle) + self.assertIn('handle=', repr(self.server)) + + self.server.set_handle(self.test_server_repr_when_handle_is_instancemethod) + self.assertIn('test_server_repr_when_handle_is_instancemethod', repr(self.server)) + + def handle(): + pass + self.server.set_handle(handle) + self.assertIn('handle= returned a result with an error set + + # It's not safe to continue after a SystemError, so we just skip the test there. + + # As of Jan 2018 with CFFI 1.11.2 this happens reliably on macOS 3.6 and 3.7 + # as well. + + # See https://bitbucket.org/cffi/cffi/issues/352/systemerror-returned-a-result-with-an + + # This is fixed in 1.11.3 + + import gevent.signal # make sure it's in sys.modules pylint:disable=redefined-outer-name + assert gevent.signal + import site + if greentest.PY34: + from importlib import reload as reload_module + elif greentest.PY3: + from imp import reload as reload_module + else: + # builtin on py2 + reload_module = reload # pylint:disable=undefined-variable + + try: + reload_module(site) + except TypeError: + # Non-CFFI on Travis triggers this, for some reason, + # but only on 3.6, not 3.4 or 3.5, and not yet on 3.7. + + # The only module seen to trigger this is __main__, i.e., this module. + + # This is hard to trigger in a virtualenv since it appears they + # install their own site.py, different from the one that ships with + # Python 3.6., and at least the version I have doesn't mess with + # __cached__ + assert greentest.PY36 + import sys + for m in set(sys.modules.values()): + try: + if m.__cached__ is None: + print("Module has None __cached__", m, file=sys.stderr) + except AttributeError: + continue + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__sleep0.py b/src/gevent/tests/test__sleep0.py new file mode 100644 index 0000000..d95d677 --- /dev/null +++ b/src/gevent/tests/test__sleep0.py @@ -0,0 +1,10 @@ +import gevent +from gevent.testing.util import alarm + + +alarm(3) + + +with gevent.Timeout(0.01, False): + while True: + gevent.sleep(0) diff --git a/src/gevent/tests/test__socket.py b/src/gevent/tests/test__socket.py new file mode 100644 index 0000000..1ee6c23 --- /dev/null +++ b/src/gevent/tests/test__socket.py @@ -0,0 +1,452 @@ +from gevent import monkey; monkey.patch_all() + +import sys +import os +import array +import socket +import traceback +import time +import unittest +import gevent.testing as greentest +from functools import wraps +from gevent.testing import six +from gevent.testing import LARGE_TIMEOUT + +# we use threading on purpose so that we can test both regular and gevent sockets with the same code +from threading import Thread as _Thread + +errno_types = int + +def wrap_error(func): + + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except: # pylint:disable=bare-except + traceback.print_exc() + os._exit(2) + + return wrapper + + +class Thread(_Thread): + + def __init__(self, **kwargs): + target = kwargs.pop('target') + target = wrap_error(target) + _Thread.__init__(self, target=target, **kwargs) + self.start() + + +class TestTCP(greentest.TestCase): + + __timeout__ = None + TIMEOUT_ERROR = socket.timeout + long_data = ", ".join([str(x) for x in range(20000)]) + if not isinstance(long_data, bytes): + long_data = long_data.encode('ascii') + + def setUp(self): + super(TestTCP, self).setUp() + self.listener = self._close_on_teardown(self._setup_listener()) + + # XXX: On Windows (at least with libev), if we have a cleanup/tearDown method + # that does 'del self.listener' AND we haven't sometime + # previously closed the listener (while the test body was executing) + # we tend to sometimes see hangs when tests run in succession; + # notably test_empty_send followed by test_makefile produces a hang + # in test_makefile when it tries to read from the client_file, because + # the accept() call in accept_once has not yet returned a new socket to + # write to. + + # The cause *seems* to be that the listener socket in both tests gets the + # same fileno(); or, at least, if we don't del the listener object, + # we get a different fileno, and that scenario works. + + # Perhaps our logic is wrong in libev_vfd in the way we use + # _open_osfhandle and determine we can close it? + self.port = self.listener.getsockname()[1] + + def _setup_listener(self): + listener = socket.socket() + greentest.bind_and_listen(listener, ('127.0.0.1', 0)) + return listener + + def create_connection(self, host='127.0.0.1', port=None, timeout=None, + blocking=None): + sock = socket.socket() + sock.connect((host, port or self.port)) + if timeout is not None: + sock.settimeout(timeout) + if blocking is not None: + sock.setblocking(blocking) + return self._close_on_teardown(sock) + + def _test_sendall(self, data, match_data=None, client_method='sendall', + **client_args): + + read_data = [] + server_exc_info = [] + + def accept_and_read(): + conn = None + try: + conn, _ = self.listener.accept() + r = conn.makefile(mode='rb') + read_data.append(r.read()) + r.flush() + r.close() + except: # pylint:disable=bare-except + server_exc_info.append(sys.exc_info()) + finally: + if conn: + conn.close() + self.listener.close() + + server = Thread(target=accept_and_read) + client = self.create_connection(**client_args) + + try: + getattr(client, client_method)(data) + finally: + client.shutdown(socket.SHUT_RDWR) + client.close() + + server.join() + if match_data is None: + match_data = self.long_data + self.assertEqual(read_data[0], match_data) + + if server_exc_info: + six.reraise(*server_exc_info[0]) + + def test_sendall_str(self): + self._test_sendall(self.long_data) + + if not six.PY3: + def test_sendall_unicode(self): + self._test_sendall(six.text_type(self.long_data)) + + def test_sendall_array(self): + data = array.array("B", self.long_data) + self._test_sendall(data) + + def test_sendall_empty(self): + data = b'' + self._test_sendall(data, data) + + def test_sendall_empty_with_timeout(self): + # Issue 719 + data = b'' + self._test_sendall(data, data, timeout=10) + + def test_sendall_nonblocking(self): + # https://github.com/benoitc/gunicorn/issues/1282 + # Even if the socket is non-blocking, we make at least + # one attempt to send data. Under Py2 before this fix, we + # would incorrectly immediately raise a timeout error + data = b'hi\n' + self._test_sendall(data, data, blocking=False) + + def test_empty_send(self): + # Issue 719 + data = b'' + self._test_sendall(data, data, client_method='send') + + def test_fullduplex(self): + N = 100000 + + def server(): + (remote_client, _) = self.listener.accept() + # start reading, then, while reading, start writing. the reader should not hang forever + + def sendall(): + remote_client.sendall(b't' * N) + + sender = Thread(target=sendall) + result = remote_client.recv(1000) + self.assertEqual(result, b'hello world') + sender.join() + remote_client.close() + self.listener.close() + + server_thread = Thread(target=server) + client = self.create_connection() + client_file = client.makefile() + client_reader = Thread(target=client_file.read, args=(N, )) + time.sleep(0.1) + client.sendall(b'hello world') + time.sleep(0.1) + + # close() used to hang + client_file.close() + client.close() + + # this tests "full duplex" bug; + server_thread.join() + + client_reader.join() + + def test_recv_timeout(self): + client_sock = [] + acceptor = Thread(target=lambda: client_sock.append(self.listener.accept())) + client = self.create_connection() + client.settimeout(1) + start = time.time() + self.assertRaises(self.TIMEOUT_ERROR, client.recv, 1024) + took = time.time() - start + self.assertTimeWithinRange(took, 1 - 0.1, 1 + 0.1) + acceptor.join() + client.close() + client_sock[0][0].close() + + # Subclasses can disable this + _test_sendall_timeout_check_time = True + + # Travis-CI container infrastructure is configured with + # large socket buffers, at least 2MB, as-of Jun 3, 2015, + # so we must be sure to send more data than that. + # In 2018, this needs to be increased *again* as a smaller value was + # still often being sent. + _test_sendall_data = b'hello' * 100000000 + + # This doesn't make much sense...why are we really skipping this? + @greentest.skipOnWindows("On Windows send() accepts whatever is thrown at it") + def test_sendall_timeout(self): + client_sock = [] + acceptor = Thread(target=lambda: client_sock.append(self.listener.accept())) + client = self.create_connection() + time.sleep(0.1) + assert client_sock + client.settimeout(0.1) + start = time.time() + try: + with self.assertRaises(self.TIMEOUT_ERROR): + client.sendall(self._test_sendall_data) + if self._test_sendall_timeout_check_time: + took = time.time() - start + self.assertTimeWithinRange(took, 0.09, 0.2) + finally: + acceptor.join() + client.close() + client_sock[0][0].close() + + def test_makefile(self): + def accept_once(): + conn, _ = self.listener.accept() + fd = conn.makefile(mode='wb') + fd.write(b'hello\n') + fd.flush() + fd.close() + conn.close() # for pypy + self.listener.close() + + acceptor = Thread(target=accept_once) + client = self.create_connection() + # Closing the socket doesn't close the file + client_file = client.makefile(mode='rb') + client.close() + line = client_file.readline() + self.assertEqual(line, b'hello\n') + self.assertEqual(client_file.read(), b'') + client_file.close() + acceptor.join() + + def test_makefile_timeout(self): + + def accept_once(): + conn, _ = self.listener.accept() + try: + time.sleep(0.3) + finally: + conn.close() # for pypy + + acceptor = Thread(target=accept_once) + client = self.create_connection() + client.settimeout(0.1) + fd = client.makefile(mode='rb') + self.assertRaises(self.TIMEOUT_ERROR, fd.readline) + client.close() + fd.close() + acceptor.join() + + def test_attributes(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0) + self.assertEqual(socket.AF_INET, s.type) + self.assertEqual(socket.SOCK_DGRAM, s.family) + self.assertEqual(0, s.proto) + + if hasattr(socket, 'SOCK_NONBLOCK'): + s.settimeout(1) + self.assertEqual(socket.AF_INET, s.type) + + s.setblocking(0) + std_socket = monkey.get_original('socket', 'socket')(socket.AF_INET, socket.SOCK_DGRAM, 0) + try: + std_socket.setblocking(0) + self.assertEqual(std_socket.type, s.type) + finally: + std_socket.close() + + s.close() + + def test_connect_ex_nonblocking_bad_connection(self): + # Issue 841 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setblocking(False) + ret = s.connect_ex((greentest.DEFAULT_LOCAL_HOST_ADDR, get_port())) + self.assertIsInstance(ret, errno_types) + s.close() + + def test_connect_ex_gaierror(self): + # Issue 841 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with self.assertRaises(socket.gaierror): + s.connect_ex(('foo.bar.fizzbuzz', get_port())) + s.close() + + def test_connect_ex_nonblocking_overflow(self): + # Issue 841 + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setblocking(False) + with self.assertRaises(OverflowError): + s.connect_ex((greentest.DEFAULT_LOCAL_HOST_ADDR, 65539)) + s.close() + + @unittest.skipUnless(hasattr(socket, 'SOCK_CLOEXEC'), + "Requires SOCK_CLOEXEC") + def test_connect_with_type_flags_ignored(self): + # Issue 944 + # If we have SOCK_CLOEXEC or similar, we shouldn't be passing + # them through to the getaddrinfo call that connect() makes + SOCK_CLOEXEC = socket.SOCK_CLOEXEC # pylint:disable=no-member + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM | SOCK_CLOEXEC) + + def accept_once(): + conn, _ = self.listener.accept() + fd = conn.makefile(mode='wb') + fd.write(b'hello\n') + fd.close() + conn.close() + + acceptor = Thread(target=accept_once) + s.connect(('127.0.0.1', self.port)) + fd = s.makefile(mode='rb') + self.assertEqual(fd.readline(), b'hello\n') + + fd.close() + s.close() + + acceptor.join() + +def get_port(): + tempsock = socket.socket() + tempsock.bind(('', 0)) + port = tempsock.getsockname()[1] + tempsock.close() + return port + + +class TestCreateConnection(greentest.TestCase): + + __timeout__ = LARGE_TIMEOUT + + def test_refuses(self): + with self.assertRaises(socket.error) as cm: + socket.create_connection((greentest.DEFAULT_BIND_ADDR, get_port()), + timeout=30, + source_address=('', get_port())) + ex = cm.exception + self.assertIn('refused', str(ex).lower()) + + @greentest.ignores_leakcheck + def test_base_exception(self): + # such as a GreenletExit or a gevent.timeout.Timeout + + class E(BaseException): + pass + + class MockSocket(object): + + created = () + closed = False + + def __init__(self, *_): + MockSocket.created += (self,) + + def connect(self, _): + raise E() + + def close(self): + self.closed = True + + def mockgetaddrinfo(*_): + return [(1, 2, 3, 3, 5),] + + import gevent.socket as gsocket + # Make sure we're monkey patched + self.assertEqual(gsocket.create_connection, socket.create_connection) + orig_socket = gsocket.socket + orig_getaddrinfo = gsocket.getaddrinfo + + try: + gsocket.socket = MockSocket + gsocket.getaddrinfo = mockgetaddrinfo + + with self.assertRaises(E): + socket.create_connection(('host', 'port')) + + self.assertEqual(1, len(MockSocket.created)) + self.assertTrue(MockSocket.created[0].closed) + + finally: + MockSocket.created = () + gsocket.socket = orig_socket + gsocket.getaddrinfo = orig_getaddrinfo + +class TestFunctions(greentest.TestCase): + + @greentest.ignores_leakcheck + # Creating new types in the function takes a cycle to cleanup. + def test_wait_timeout(self): + # Issue #635 + import gevent.socket + import gevent._socketcommon + + class io(object): + callback = None + + def start(self, *_args): + gevent.sleep(10) + + with self.assertRaises(gevent.socket.timeout): + gevent.socket.wait(io(), timeout=0.01) # pylint:disable=no-member + + + def test_signatures(self): + # https://github.com/gevent/gevent/issues/960 + exclude = [] + if greentest.PYPY: + # Up through at least PyPy 5.7.1, they define these as + # gethostbyname(host), whereas the official CPython argument name + # is hostname. But cpython doesn't allow calling with keyword args. + # Likewise for gethostbyaddr: PyPy uses host, cpython uses ip_address + exclude.append('gethostbyname') + exclude.append('gethostbyname_ex') + exclude.append('gethostbyaddr') + self.assertMonkeyPatchedFuncSignatures('socket', exclude=exclude) + + +class TestSocket(greentest.TestCase): + + def test_shutdown_when_closed(self): + # https://github.com/gevent/gevent/issues/1089 + # we once raised an AttributeError. + s = socket.socket() + s.close() + with self.assertRaises(socket.error): + s.shutdown(socket.SHUT_RDWR) + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__socket_close.py b/src/gevent/tests/test__socket_close.py new file mode 100644 index 0000000..24a7168 --- /dev/null +++ b/src/gevent/tests/test__socket_close.py @@ -0,0 +1,58 @@ +import gevent +from gevent import socket +from gevent import server +import gevent.testing as greentest + +# XXX also test: send, sendall, recvfrom, recvfrom_into, sendto + + +def readall(sock, _): + while sock.recv(1024): + pass # pragma: no cover we never actually send the data + sock.close() + + +class Test(greentest.TestCase): + + error_fatal = False + + def setUp(self): + self.server = server.StreamServer(('127.0.0.1', 0), readall) + self.server.start() + + def tearDown(self): + self.server.stop() + + def test_recv_closed(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', self.server.server_port)) + receiver = gevent.spawn(sock.recv, 25) + try: + gevent.sleep(0.001) + sock.close() + receiver.join(timeout=0.1) + self.assertTrue(receiver.ready(), receiver) + self.assertEqual(receiver.value, None) + self.assertIsInstance(receiver.exception, socket.error) + self.assertEqual(receiver.exception.errno, socket.EBADF) + finally: + receiver.kill() + + # XXX: This is possibly due to the bad behaviour of small sleeps? + # The timeout is the global test timeout, 10s + @greentest.skipOnLibuvOnCI("Sometimes randomly times out") + def test_recv_twice(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', self.server.server_port)) + receiver = gevent.spawn(sock.recv, 25) + try: + gevent.sleep(0.001) + self.assertRaises(AssertionError, sock.recv, 25) + self.assertRaises(AssertionError, sock.recv, 25) + finally: + receiver.kill() + sock.close() + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__socket_dns.py b/src/gevent/tests/test__socket_dns.py new file mode 100644 index 0000000..d9986dd --- /dev/null +++ b/src/gevent/tests/test__socket_dns.py @@ -0,0 +1,718 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# pylint:disable=broad-except +import gevent +from gevent import monkey + +import os +import re +import gevent.testing as greentest +import unittest +import socket +from time import time +import traceback +import gevent.socket as gevent_socket +from gevent.testing.util import log +from gevent.testing import six +from gevent.testing.six import xrange + + +resolver = gevent.get_hub().resolver +log('Resolver: %s', resolver) + +if getattr(resolver, 'pool', None) is not None: + resolver.pool.size = 1 + +from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM +from gevent.testing.sysinfo import RESOLVER_DNSPYTHON +from gevent.testing.sysinfo import PY2 +import gevent.testing.timing + + +assert gevent_socket.gaierror is socket.gaierror +assert gevent_socket.error is socket.error + +DEBUG = os.getenv('GEVENT_DEBUG', '') == 'trace' + + +def _run(function, *args): + try: + result = function(*args) + assert not isinstance(result, BaseException), repr(result) + return result + except Exception as ex: + if DEBUG: + traceback.print_exc() + return ex + + +def format_call(function, args): + args = repr(args) + if args.endswith(',)'): + args = args[:-2] + ')' + try: + module = function.__module__.replace('gevent._socketcommon', 'gevent') + name = function.__name__ + return '%s:%s%s' % (module, name, args) + except AttributeError: + return function + args + + +def log_fresult(result, seconds): + if isinstance(result, Exception): + msg = ' -=> raised %r' % (result, ) + else: + msg = ' -=> returned %r' % (result, ) + time_ms = ' %.2fms' % (seconds * 1000.0, ) + space = 80 - len(msg) - len(time_ms) + if space > 0: + space = ' ' * space + else: + space = '' + log(msg + space + time_ms) + + +def run(function, *args): + if DEBUG: + log(format_call(function, args)) + delta = time() + result = _run(function, *args) + delta = time() - delta + if DEBUG: + log_fresult(result, delta) + return result, delta + + +def log_call(result, runtime, function, *args): + log(format_call(function, args)) + log_fresult(result, runtime) + + +def compare_relaxed(a, b): + """ + >>> compare_relaxed('2a00:1450:400f:801::1010', '2a00:1450:400f:800::1011') + True + + >>> compare_relaxed('2a00:1450:400f:801::1010', '2aXX:1450:400f:900::1011') + False + + >>> compare_relaxed('2a00:1450:4016:800::1013', '2a00:1450:4008:c01::93') + True + + >>> compare_relaxed('2001:470::e852:4a38:9d7f:0', '2001:470:6d00:1c:1::d00') + True + + >>> compare_relaxed('2001:470:4147:4943:6161:6161:2e74:6573', '2001:470::') + True + + >>> compare_relaxed('2607:f8b0:6708:24af:1fd:700:60d4:4af', '2607:f8b0:2d00::f000:0') + True + + >>> compare_relaxed('a.google.com', 'b.google.com') + True + + >>> compare_relaxed('a.google.com', 'a.gevent.org') + False + """ + # IPv6 address from different requests might be different + a_segments = a.count(':') + b_segments = b.count(':') + if a_segments and b_segments: + if a_segments == b_segments and a_segments in (4, 5, 6, 7): + return True + if a.rstrip(':').startswith(b.rstrip(':')) or b.rstrip(':').startswith(a.rstrip(':')): + return True + if a_segments >= 2 and b_segments >= 2 and a.split(':')[:2] == b.split(':')[:2]: + return True + + return a.split('.', 1)[-1] == b.split('.', 1)[-1] + + +def contains_5tuples(lst): + for item in lst: + if not (isinstance(item, tuple) and len(item) == 5): + return False + return True + + +def relaxed_is_equal(a, b): + """ + >>> relaxed_is_equal([(10, 1, 6, '', ('2a00:1450:400f:801::1010', 80, 0, 0))], [(10, 1, 6, '', ('2a00:1450:400f:800::1011', 80, 0, 0))]) + True + + >>> relaxed_is_equal([1, '2'], (1, '2')) + False + + >>> relaxed_is_equal([1, '2'], [1, '2']) + True + + >>> relaxed_is_equal(('wi-in-x93.1e100.net', 'http'), ('we-in-x68.1e100.net', 'http')) + True + """ + if type(a) is not type(b): + return False + if a == b: + return True + if isinstance(a, six.string_types): + return compare_relaxed(a, b) + try: + if len(a) != len(b): + return False + except TypeError: + return False + if contains_5tuples(a) and contains_5tuples(b): + # getaddrinfo results + a = sorted(a) + b = sorted(b) + return all(relaxed_is_equal(x, y) for (x, y) in zip(a, b)) + + +def add(klass, hostname, name=None, + skip=None, skip_reason=None): + + call = callable(hostname) + + def _setattr(k, n, func): + if skip: + func = greentest.skipIf(skip, skip_reason,)(func) + if not hasattr(k, n): + setattr(k, n, func) + + if name is None: + if call: + name = hostname.__name__ + else: + name = re.sub(r'[^\w]+', '_', repr(hostname)) + assert name, repr(hostname) + + def test1(self): + x = hostname() if call else hostname + self._test('getaddrinfo', x, 'http') + test1.__name__ = 'test_%s_getaddrinfo' % name + _setattr(klass, test1.__name__, test1) + + def test2(self): + x = hostname() if call else hostname + ipaddr = self._test('gethostbyname', x) + if not isinstance(ipaddr, Exception): + self._test('gethostbyaddr', ipaddr) + test2.__name__ = 'test_%s_gethostbyname' % name + _setattr(klass, test2.__name__, test2) + + def test3(self): + x = hostname() if call else hostname + self._test('gethostbyname_ex', x) + test3.__name__ = 'test_%s_gethostbyname_ex' % name + _setattr(klass, test3.__name__, test3) + + def test4(self): + x = hostname() if call else hostname + self._test('gethostbyaddr', x) + test4.__name__ = 'test_%s_gethostbyaddr' % name + _setattr(klass, test4.__name__, test4) + + def test5(self): + x = hostname() if call else hostname + self._test('getnameinfo', (x, 80), 0) + test5.__name__ = 'test_%s_getnameinfo' % name + _setattr(klass, test5.__name__, test5) + + +class TestCase(greentest.TestCase): + + __timeout__ = 30 + switch_expected = None + verbose_dns = False + + def should_log_results(self, result1, result2): + if not self.verbose_dns: + return False + + if isinstance(result1, BaseException) and isinstance(result2, BaseException): + return type(result1) is not type(result2) + return repr(result1) != repr(result2) + + def _test(self, func, *args): + gevent_func = getattr(gevent_socket, func) + real_func = monkey.get_original('socket', func) + real_result, time_real = run(real_func, *args) + gevent_result, time_gevent = run(gevent_func, *args) + if not DEBUG and self.should_log_results(real_result, gevent_result): + log('') + log_call(real_result, time_real, real_func, *args) + log_call(gevent_result, time_gevent, gevent_func, *args) + self.assertEqualResults(real_result, gevent_result, func) + + if self.verbose_dns and time_gevent > time_real + 0.01 and time_gevent > 0.02: + msg = 'gevent:%s%s took %dms versus %dms stdlib' % (func, args, time_gevent * 1000.0, time_real * 1000.0) + + if time_gevent > time_real + 1: + word = 'VERY' + else: + word = 'quite' + + log('\nWARNING: %s slow: %s', word, msg) + + return gevent_result + + def _normalize_result(self, result, func_name): + norm_name = '_normalize_result_' + func_name + if hasattr(self, norm_name): + return getattr(self, norm_name)(result) + return result + + def _normalize_result_gethostbyname_ex(self, result): + # Often the second and third part of the tuple (hostname, aliaslist, ipaddrlist) + # can be in different orders if we're hitting different servers, + # or using the native and ares resolvers due to load-balancing techniques. + # We sort them. + if not RESOLVER_NOT_SYSTEM or isinstance(result, BaseException): + return result + # result[1].sort() # we wind up discarding this + + # On Py2 in test_russion_gethostbyname_ex, this + # is actually an integer, for some reason. In TestLocalhost.tets__ip6_localhost, + # the result isn't this long (maybe an error?). + try: + result[2].sort() + except AttributeError: + pass + except IndexError: + return result + # On some systems, a random alias is found in the aliaslist + # by the system resolver, but not by cares, and vice versa. We deem the aliaslist + # unimportant and discard it. + # On some systems (Travis CI), the ipaddrlist for 'localhost' can come back + # with two entries 127.0.0.1 (presumably two interfaces?) for c-ares + ips = result[2] + if ips == ['127.0.0.1', '127.0.0.1']: + ips = ['127.0.0.1'] + # On some systems, the hostname can get caps + return (result[0].lower(), [], ips) + + def _normalize_result_getaddrinfo(self, result): + if not RESOLVER_NOT_SYSTEM: + return result + # On Python 3, the builtin resolver can return SOCK_RAW results, but + # c-ares doesn't do that. So we remove those if we find them. + if hasattr(socket, 'SOCK_RAW') and isinstance(result, list): + result = [x for x in result if x[1] != socket.SOCK_RAW] + if isinstance(result, list): + result.sort() + return result + + def _normalize_result_gethostbyaddr(self, result): + if not RESOLVER_NOT_SYSTEM: + return result + + if isinstance(result, tuple): + # On some systems, a random alias is found in the aliaslist + # by the system resolver, but not by cares and vice versa. We deem the aliaslist + # unimportant and discard it. + return (result[0], [], result[2]) + return result + + def assertEqualResults(self, real_result, gevent_result, func): + errors = (socket.gaierror, socket.herror, TypeError) + if isinstance(real_result, errors) and isinstance(gevent_result, errors): + if type(real_result) is not type(gevent_result): + log('WARNING: error type mismatch: %r (gevent) != %r (stdlib)', gevent_result, real_result) + return + + real_result = self._normalize_result(real_result, func) + gevent_result = self._normalize_result(gevent_result, func) + + real_result_repr = repr(real_result) + gevent_result_repr = repr(gevent_result) + if real_result_repr == gevent_result_repr: + return + if relaxed_is_equal(gevent_result, real_result): + return + + # If we're using the ares resolver, allow the real resolver to generate an + # error that the ares resolver actually gets an answer to. + + if ( + RESOLVER_NOT_SYSTEM + and isinstance(real_result, errors) + and not isinstance(gevent_result, errors) + ): + return + + # From 2.7 on, assertEqual does a better job highlighting the results than we would + # because it calls assertSequenceEqual, which highlights the exact + # difference in the tuple + self.assertEqual(real_result, gevent_result) + + +class TestTypeError(TestCase): + pass + +add(TestTypeError, None) +add(TestTypeError, 25) + + +class TestHostname(TestCase): + pass + +add(TestHostname, socket.gethostname) + + +class TestLocalhost(TestCase): + # certain tests in test_patched_socket.py only work if getaddrinfo('localhost') does not switch + # (e.g. NetworkConnectionAttributesTest.testSourceAddress) + #switch_expected = False + # XXX: The above has been commented out for some time. Apparently this isn't the case + # anymore. + + def _normalize_result_getaddrinfo(self, result): + if RESOLVER_NOT_SYSTEM: + # We see that some impls (OS X) return extra results + # like DGRAM that ares does not. + return () + return super(TestLocalhost, self)._normalize_result_getaddrinfo(result) + + if greentest.RUNNING_ON_TRAVIS and greentest.PY2 and RESOLVER_NOT_SYSTEM: + def _normalize_result_gethostbyaddr(self, result): + # Beginning in November 2017 after an upgrade to Travis, + # we started seeing ares return ::1 for localhost, but + # the system resolver is still returning 127.0.0.1 under Python 2 + result = super(TestLocalhost, self)._normalize_result_gethostbyaddr(result) + if isinstance(result, tuple): + result = (result[0], result[1], ['127.0.0.1']) + return result + + +add( + TestLocalhost, 'ip6-localhost', + skip=greentest.RUNNING_ON_TRAVIS, + skip_reason="ares fails here, for some reason, presumably a badly " + "configured /etc/hosts" +) +add( + TestLocalhost, 'localhost', + skip=greentest.RUNNING_ON_TRAVIS, + skip_reason="Beginning Dec 1 2017, ares started returning ip6-localhost " + "instead of localhost" +) + + +class TestNonexistent(TestCase): + pass + +add(TestNonexistent, 'nonexistentxxxyyy') + + +class Test1234(TestCase): + pass + +add(Test1234, '1.2.3.4') + + +class Test127001(TestCase): + pass + +add( + Test127001, '127.0.0.1', + skip=greentest.RUNNING_ON_TRAVIS, + skip_reason="Beginning Dec 1 2017, ares started returning ip6-localhost " + "instead of localhost" +) + + + +class TestBroadcast(TestCase): + switch_expected = False + + if RESOLVER_NOT_SYSTEM: + # ares and dnspython raises errors for broadcasthost/255.255.255.255 + + @unittest.skip('ares raises errors for broadcasthost/255.255.255.255') + def test__broadcast__gethostbyaddr(self): + return + + test__broadcast__gethostbyname = test__broadcast__gethostbyaddr + +add(TestBroadcast, '') + + +from gevent.resolver.dnspython import HostsFile +class SanitizedHostsFile(HostsFile): + def iter_all_host_addr_pairs(self): + for name, addr in super(SanitizedHostsFile, self).iter_all_host_addr_pairs(): + if (RESOLVER_NOT_SYSTEM + and (name.endswith('local') # ignore bonjour, ares can't find them + # ignore common aliases that ares can't find + or addr == '255.255.255.255' + or name == 'broadcasthost' + # We get extra results from some impls, like OS X + # it returns DGRAM results + or name == 'localhost')): + continue # pragma: no cover + if name.endswith('local'): + # These can only be found if bonjour is running, + # and are very slow to do so with the system resolver on OS X + continue + yield name, addr + +@greentest.skipIf(greentest.RUNNING_ON_CI, + "This sometimes randomly fails on Travis with ares and on appveyor, beginning Feb 13, 2018") +# Probably due to round-robin DNS, +# since this is not actually the system's etc hosts file. +# TODO: Rethink this. We need something reliable. Go back to using +# the system's etc hosts? +class TestEtcHosts(TestCase): + + MAX_HOSTS = int(os.getenv('GEVENTTEST_MAX_ETC_HOSTS', '10')) + + @classmethod + def populate_tests(cls): + hf = SanitizedHostsFile(os.path.join(os.path.dirname(__file__), + 'hosts_file.txt')) + all_etc_hosts = sorted(hf.iter_all_host_addr_pairs()) + if len(all_etc_hosts) > cls.MAX_HOSTS and not DEBUG: + all_etc_hosts = all_etc_hosts[:cls.MAX_HOSTS] + + for host, ip in all_etc_hosts: + add(cls, host) + add(cls, ip) + + + +TestEtcHosts.populate_tests() + + + +class TestGeventOrg(TestCase): + + HOSTNAME = 'www.gevent.org' + +# For this test to work correctly, it needs to resolve to +# an address with a single A record; round-robin DNS and multiple A records +# may mess it up (subsequent requests---and we always make two---may return +# unequal results). We used to use gevent.org, but that now has multiple A records; +# trying www.gevent.org which is a CNAME to readthedocs.org. +add(TestGeventOrg, TestGeventOrg.HOSTNAME) + + +class TestFamily(TestCase): + + @classmethod + def getresult(cls): + if not hasattr(cls, '_result'): + cls._result = getattr(socket, 'getaddrinfo')(TestGeventOrg.HOSTNAME, None) + return cls._result + + def test_inet(self): + self.assertEqualResults(self.getresult(), + gevent_socket.getaddrinfo(TestGeventOrg.HOSTNAME, None, socket.AF_INET), + 'getaddrinfo') + + def test_unspec(self): + self.assertEqualResults(self.getresult(), + gevent_socket.getaddrinfo(TestGeventOrg.HOSTNAME, None, socket.AF_UNSPEC), + 'getaddrinfo') + + def test_badvalue(self): + self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, 255) + self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, 255000) + self._test('getaddrinfo', TestGeventOrg.HOSTNAME, None, -1) + + def test_badtype(self): + self._test('getaddrinfo', TestGeventOrg.HOSTNAME, 'x') + + +class Test_getaddrinfo(TestCase): + + def _test_getaddrinfo(self, *args): + self._test('getaddrinfo', *args) + + def test_80(self): + self._test_getaddrinfo(TestGeventOrg.HOSTNAME, 80) + + def test_int_string(self): + self._test_getaddrinfo(TestGeventOrg.HOSTNAME, '80') + + def test_0(self): + self._test_getaddrinfo(TestGeventOrg.HOSTNAME, 0) + + def test_http(self): + self._test_getaddrinfo(TestGeventOrg.HOSTNAME, 'http') + + def test_notexistent_tld(self): + self._test_getaddrinfo('myhost.mytld', 53) + + def test_notexistent_dot_com(self): + self._test_getaddrinfo('sdfsdfgu5e66098032453245wfdggd.com', 80) + + def test1(self): + return self._test_getaddrinfo(TestGeventOrg.HOSTNAME, 52, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, 0) + + def test2(self): + return self._test_getaddrinfo(TestGeventOrg.HOSTNAME, 53, socket.AF_INET, socket.SOCK_DGRAM, 17) + + @unittest.skipIf(RESOLVER_DNSPYTHON, + "dnspython only returns some of the possibilities") + def test3(self): + return self._test_getaddrinfo('google.com', 'http', socket.AF_INET6) + + + @greentest.skipIf(PY2, "Enums only on Python 3.4+") + def test_enums(self): + # https://github.com/gevent/gevent/issues/1310 + + # On Python 3, getaddrinfo does special things to make sure that + # the fancy enums are returned. + + gai = gevent_socket.getaddrinfo('example.com', 80, + socket.AF_INET, + socket.SOCK_STREAM, socket.IPPROTO_TCP) + af, socktype, _proto, _canonname, _sa = gai[0] + self.assertIs(socktype, socket.SOCK_STREAM) + self.assertIs(af, socket.AF_INET) + +class TestInternational(TestCase): + pass + +# dns python can actually resolve these: it uses +# the 2008 version of idna encoding, whereas on Python 2, +# with the default resolver, it tries to encode to ascii and +# raises a UnicodeEncodeError. So we get different results. +add(TestInternational, u'президент.рф', 'russian', + skip=(PY2 and RESOLVER_DNSPYTHON), skip_reason="dnspython can actually resolve these") +add(TestInternational, u'президент.рф'.encode('idna'), 'idna') + + +class TestInterrupted_gethostbyname(gevent.testing.timing.AbstractGenericWaitTestCase): + + # There are refs to a Waiter in the C code that don't go + # away yet; one gc may or may not do it. + @greentest.ignores_leakcheck + def test_returns_none_after_timeout(self): + super(TestInterrupted_gethostbyname, self).test_returns_none_after_timeout() + + def wait(self, timeout): + with gevent.Timeout(timeout, False): + for index in xrange(1000000): + try: + gevent_socket.gethostbyname('www.x%s.com' % index) + except socket.error: + pass + raise AssertionError('Timeout was not raised') + + def cleanup(self): + # Depending on timing, this can raise: + # (This suddenly started happening on Apr 6 2016; www.x1000000.com + # is apparently no longer around) + + # File "test__socket_dns.py", line 538, in cleanup + # gevent.get_hub().threadpool.join() + # File "/home/travis/build/gevent/gevent/src/gevent/threadpool.py", line 108, in join + # sleep(delay) + # File "/home/travis/build/gevent/gevent/src/gevent/hub.py", line 169, in sleep + # hub.wait(loop.timer(seconds, ref=ref)) + # File "/home/travis/build/gevent/gevent/src/gevent/hub.py", line 651, in wait + # result = waiter.get() + # File "/home/travis/build/gevent/gevent/src/gevent/hub.py", line 899, in get + # return self.hub.switch() + # File "/home/travis/build/gevent/gevent/src/greentest/greentest.py", line 520, in switch + # return _original_Hub.switch(self, *args) + # File "/home/travis/build/gevent/gevent/src/gevent/hub.py", line 630, in switch + # return RawGreenlet.switch(self) + # gaierror: [Errno -2] Name or service not known + try: + gevent.get_hub().threadpool.join() + except Exception: # pragma: no cover pylint:disable=broad-except + traceback.print_exc() + + +# class TestInterrupted_getaddrinfo(greentest.GenericWaitTestCase): +# +# def wait(self, timeout): +# with gevent.Timeout(timeout, False): +# for index in range(1000): +# try: +# gevent_socket.getaddrinfo('www.a%s.com' % index, 'http') +# except socket.gaierror: +# pass + + +class TestBadName(TestCase): + pass + +add(TestBadName, 'xxxxxxxxxxxx') + + +class TestBadIP(TestCase): + pass + +add(TestBadIP, '1.2.3.400') + + +@greentest.skipIf(greentest.RUNNING_ON_TRAVIS, "Travis began returning ip6-localhost") +class Test_getnameinfo_127001(TestCase): + + def test(self): + self._test('getnameinfo', ('127.0.0.1', 80), 0) + + def test_DGRAM(self): + self._test('getnameinfo', ('127.0.0.1', 779), 0) + self._test('getnameinfo', ('127.0.0.1', 779), socket.NI_DGRAM) + + def test_NOFQDN(self): + # I get ('localhost', 'www') with _socket but ('localhost.localdomain', 'www') with gevent.socket + self._test('getnameinfo', ('127.0.0.1', 80), socket.NI_NOFQDN) + + def test_NAMEREQD(self): + self._test('getnameinfo', ('127.0.0.1', 80), socket.NI_NAMEREQD) + + +class Test_getnameinfo_geventorg(TestCase): + + def test_NUMERICHOST(self): + self._test('getnameinfo', (TestGeventOrg.HOSTNAME, 80), 0) + self._test('getnameinfo', (TestGeventOrg.HOSTNAME, 80), socket.NI_NUMERICHOST) + + def test_NUMERICSERV(self): + self._test('getnameinfo', (TestGeventOrg.HOSTNAME, 80), socket.NI_NUMERICSERV) + + def test_domain1(self): + self._test('getnameinfo', (TestGeventOrg.HOSTNAME, 80), 0) + + def test_domain2(self): + self._test('getnameinfo', ('www.gevent.org', 80), 0) + + def test_port_zero(self): + self._test('getnameinfo', ('www.gevent.org', 0), 0) + + +class Test_getnameinfo_fail(TestCase): + + def test_port_string(self): + self._test('getnameinfo', ('www.gevent.org', 'http'), 0) + + def test_bad_flags(self): + self._test('getnameinfo', ('localhost', 80), 55555555) + + +class TestInvalidPort(TestCase): + + def test1(self): + self._test('getnameinfo', ('www.gevent.org', -1), 0) + + def test2(self): + self._test('getnameinfo', ('www.gevent.org', None), 0) + + def test3(self): + self._test('getnameinfo', ('www.gevent.org', 'x'), 0) + + @unittest.skipIf(RESOLVER_DNSPYTHON, + "System resolvers do funny things with this: macOS raises gaierror, " + "Travis CI returns (readthedocs.org, '0'). It's hard to match that exactly. " + "dnspython raises OverflowError.") + def test4(self): + self._test('getnameinfo', ('www.gevent.org', 65536), 0) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__socket_dns6.py b/src/gevent/tests/test__socket_dns6.py new file mode 100644 index 0000000..6a706d8 --- /dev/null +++ b/src/gevent/tests/test__socket_dns6.py @@ -0,0 +1,81 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +from __future__ import print_function, absolute_import, division + +import gevent.testing as greentest +import socket +from gevent.tests.test__socket_dns import TestCase, add + +from gevent.testing.sysinfo import RESOLVER_NOT_SYSTEM +from gevent.testing.sysinfo import RESOLVER_DNSPYTHON + +if not greentest.RUNNING_ON_CI and not RESOLVER_DNSPYTHON: + + + # We can't control the DNS servers we use there + # for the system. This works best with the google DNS servers + # The getnameinfo test can fail on CI. + + # Previously only Test6_ds failed, but as of Jan 2018, Test6 + # and Test6_google begin to fail: + + # First differing element 0: + # 'vm2.test-ipv6.com' + # 'ip119.gigo.com' + + # - ('vm2.test-ipv6.com', [], ['2001:470:1:18::125']) + # ? --------- ^^ ^^ + + # + ('ip119.gigo.com', [], ['2001:470:1:18::119']) + # ? ^^^^^^^^ ^^ + + class Test6(TestCase): + + # host that only has AAAA record + host = 'aaaa.test-ipv6.com' + + def test_empty(self): + self._test('getaddrinfo', self.host, 'http') + + def test_inet(self): + self._test('getaddrinfo', self.host, None, socket.AF_INET) + + def test_inet6(self): + self._test('getaddrinfo', self.host, None, socket.AF_INET6) + + def test_unspec(self): + self._test('getaddrinfo', self.host, None, socket.AF_UNSPEC) + + + class Test6_google(Test6): + host = 'ipv6.google.com' + + def _normalize_result_getnameinfo(self, result): + if greentest.RUNNING_ON_CI and RESOLVER_NOT_SYSTEM: + # Disabled, there are multiple possibilities + # and we can get different ones, rarely. + return () + return result + + add(Test6, Test6.host) + add(Test6_google, Test6_google.host) + + + + class Test6_ds(Test6): + # host that has both A and AAAA records + host = 'ds.test-ipv6.com' + + def _normalize_result_gethostbyaddr(self, result): + # This test is effectively disabled. There are multiple address + # that resolve and which ones you get depend on the settings + # of the system and ares. They don't match exactly. + return () + + _normalize_result_gethostbyname = _normalize_result_gethostbyaddr + + add(Test6_ds, Test6_ds.host) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__socket_errors.py b/src/gevent/tests/test__socket_errors.py new file mode 100644 index 0000000..fec07dd --- /dev/null +++ b/src/gevent/tests/test__socket_errors.py @@ -0,0 +1,46 @@ +# Copyright (c) 2008-2009 AG Projects +# Author: Denis Bilenko +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import gevent.testing as greentest +from gevent.socket import socket, error + +try: + from errno import WSAECONNREFUSED as ECONNREFUSED +except ImportError: + from errno import ECONNREFUSED + + +class TestSocketErrors(greentest.TestCase): + + __timeout__ = 5 + + def test_connection_refused(self): + s = socket() + self._close_on_teardown(s) + try: + s.connect(('127.0.0.1', 81)) + except error as ex: + assert ex.args[0] == ECONNREFUSED, repr(ex) + assert 'refused' in str(ex).lower(), str(ex) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__socket_ex.py b/src/gevent/tests/test__socket_ex.py new file mode 100644 index 0000000..0926ff6 --- /dev/null +++ b/src/gevent/tests/test__socket_ex.py @@ -0,0 +1,43 @@ +import gevent.testing as greentest +from gevent import socket +import errno +import sys + + +class TestClosedSocket(greentest.TestCase): + + switch_expected = False + + def test(self): + sock = socket.socket() + sock.close() + try: + sock.send(b'a', timeout=1) + raise AssertionError("Should not get here") + except (socket.error, OSError) as ex: + if ex.args[0] != errno.EBADF: + if sys.platform.startswith('win'): + # Windows/Py3 raises "OSError: [WinError 10038] " + # which is not standard and not what it does + # on Py2. + pass + else: + raise + + +class TestRef(greentest.TestCase): + + switch_expected = False + + def test(self): + sock = socket.socket() + assert sock.ref is True, sock.ref + sock.ref = False + assert sock.ref is False, sock.ref + assert sock._read_event.ref is False, sock.ref + assert sock._write_event.ref is False, sock.ref + sock.close() + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__socket_send_memoryview.py b/src/gevent/tests/test__socket_send_memoryview.py new file mode 100644 index 0000000..c979fe2 --- /dev/null +++ b/src/gevent/tests/test__socket_send_memoryview.py @@ -0,0 +1,39 @@ +# See issue #466 +import unittest +import ctypes + + +class AnStructure(ctypes.Structure): + _fields_ = [("x", ctypes.c_int)] + + +def _send(socket): + for meth in ('sendall', 'send'): + anStructure = AnStructure() + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.connect(('127.0.0.1', 12345)) + getattr(sock, meth)(anStructure) + sock.close() + + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.connect(('127.0.0.1', 12345)) + sock.settimeout(1.0) + getattr(sock, meth)(anStructure) + sock.close() + +class TestSendBuiltinSocket(unittest.TestCase): + + def test_send(self): + import socket + _send(socket) + + +class TestSendGeventSocket(unittest.TestCase): + + def test_send(self): + import gevent.socket + _send(gevent.socket) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__socket_ssl.py b/src/gevent/tests/test__socket_ssl.py new file mode 100644 index 0000000..8f47940 --- /dev/null +++ b/src/gevent/tests/test__socket_ssl.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +from gevent import monkey +monkey.patch_all() + +import unittest +try: + import httplib +except ImportError: + from http import client as httplib +import socket + + +import gevent.testing as greentest + + +@unittest.skipUnless( + hasattr(socket, 'ssl'), + "Needs socket.ssl" +) +class AmazonHTTPSTests(greentest.TestCase): + + __timeout__ = 30 + + def test_amazon_response(self): + conn = httplib.HTTPSConnection('sdb.amazonaws.com') + conn.debuglevel = 1 + conn.request('GET', '/') + conn.getresponse() + + def test_str_and_repr(self): + conn = socket.socket() + conn.connect(('sdb.amazonaws.com', 443)) + ssl_conn = socket.ssl(conn) # pylint:disable=no-member + assert str(ssl_conn) + assert repr(ssl_conn) + + +if __name__ == "__main__": + greentest.main() diff --git a/src/gevent/tests/test__socket_timeout.py b/src/gevent/tests/test__socket_timeout.py new file mode 100644 index 0000000..f98d1bb --- /dev/null +++ b/src/gevent/tests/test__socket_timeout.py @@ -0,0 +1,54 @@ +import gevent +from gevent import socket +import gevent.testing as greentest + + +class Test(greentest.TestCase): + + server = None + acceptor = None + server_port = None + + def _accept(self): + self.server.listen(1) + try: + conn, _ = self.server.accept() + self._close_on_teardown(conn) + except socket.error: + pass + + def setUp(self): + super(Test, self).setUp() + self.server = socket.socket() + self._close_on_teardown(self.server) + self.server.bind(('127.0.0.1', 0)) + self.server_port = self.server.getsockname()[1] + self.acceptor = gevent.spawn(self._accept) + gevent.sleep(0) + + def tearDown(self): + if self.acceptor is not None: + self.acceptor.kill() + self.acceptor = None + if self.server is not None: + self.server.close() + self.server = None + super(Test, self).tearDown() + + def test_timeout(self): + gevent.sleep(0) + sock = socket.socket() + self._close_on_teardown(sock) + sock.connect(('127.0.0.1', self.server_port)) + + sock.settimeout(0.1) + with self.assertRaises(socket.error) as cm: + sock.recv(1024) + + ex = cm.exception + self.assertEqual(ex.args, ('timed out',)) + self.assertEqual(str(ex), 'timed out') + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__socketpair.py b/src/gevent/tests/test__socketpair.py new file mode 100644 index 0000000..308f033 --- /dev/null +++ b/src/gevent/tests/test__socketpair.py @@ -0,0 +1,35 @@ +from gevent import monkey; monkey.patch_all() +import socket +import unittest + + +class TestSocketpair(unittest.TestCase): + + def test_makefile(self): + msg = b'hello world' + x, y = socket.socketpair() + x.sendall(msg) + x.close() + with y.makefile('rb') as f: + read = f.read() + self.assertEqual(msg, read) + y.close() + + def test_fromfd(self): + msg = b'hello world' + x, y = socket.socketpair() + xx = socket.fromfd(x.fileno(), x.family, socket.SOCK_STREAM) + x.close() + yy = socket.fromfd(y.fileno(), y.family, socket.SOCK_STREAM) + y.close() + + xx.sendall(msg) + xx.close() + with yy.makefile('rb') as f: + read = f.read() + self.assertEqual(msg, read) + yy.close() + + +if __name__ == '__main__': + unittest.main() diff --git a/src/gevent/tests/test__ssl.py b/src/gevent/tests/test__ssl.py new file mode 100644 index 0000000..e2b4df7 --- /dev/null +++ b/src/gevent/tests/test__ssl.py @@ -0,0 +1,104 @@ +from gevent import monkey; monkey.patch_all() +import os + +import socket +import gevent.testing as greentest +# Be careful not to have TestTCP as a bare attribute in this module, +# even aliased, to avoid running duplicate tests +import test__socket +import ssl + + +import unittest +from gevent.hub import LoopExit + +class TestSSL(test__socket.TestTCP): + + certfile = os.path.join(os.path.dirname(__file__), 'test_server.crt') + privfile = os.path.join(os.path.dirname(__file__), 'test_server.key') + # Python 2.x has socket.sslerror (which is an alias for + # ssl.SSLError); That's gone in Py3 though. In Python 2, most timeouts are raised + # as SSLError, but Python 3 raises the normal socket.timeout instead. So this has + # the effect of making TIMEOUT_ERROR be SSLError on Py2 and socket.timeout on Py3 + # See https://bugs.python.org/issue10272 + TIMEOUT_ERROR = getattr(socket, 'sslerror', socket.timeout) + + def _setup_listener(self): + listener, raw_listener = ssl_listener(('127.0.0.1', 0), self.privfile, self.certfile) + self._close_on_teardown(raw_listener) + return listener + + def create_connection(self, *args, **kwargs): # pylint:disable=arguments-differ + return ssl.wrap_socket(super(TestSSL, self).create_connection(*args, **kwargs)) + + # The SSL library can take a long time to buffer the large amount of data we're trying + # to send, so we can't compare to the timeout values + _test_sendall_timeout_check_time = False + + # The SSL layer has extra buffering, so test_sendall needs + # to send a very large amount to make it timeout + _test_sendall_data = data_sent = b'hello' * 100000000 + + @greentest.skipOnWindows("Not clear why we're skipping") + def test_ssl_sendall_timeout0(self): + # Issue #317: SSL_WRITE_PENDING in some corner cases + + server_sock = [] + acceptor = test__socket.Thread(target=lambda: server_sock.append(self.listener.accept())) + client = self.create_connection() + client.setblocking(False) + try: + # Python 3 raises ssl.SSLWantWriteError; Python 2 simply *hangs* + # on non-blocking sockets because it's a simple loop around + # send(). Python 2.6 doesn't have SSLWantWriteError + expected = getattr(ssl, 'SSLWantWriteError', ssl.SSLError) + with self.assertRaises(expected): + client.sendall(self._test_sendall_data) + finally: + acceptor.join() + client.close() + server_sock[0][0].close() + + def test_fullduplex(self): + try: + super(TestSSL, self).test_fullduplex() + except LoopExit: + if greentest.LIBUV and greentest.WIN: + # XXX: Unable to duplicate locally + raise unittest.SkipTest("libuv on Windows sometimes raises LoopExit") + raise + + + @greentest.ignores_leakcheck + def test_empty_send(self): + # Issue 719 + # Sending empty bytes with the 'send' method raises + # ssl.SSLEOFError in the stdlib. PyPy 4.0 and CPython 2.6 + # both just raise the superclass, ssl.SSLError. + + # Ignored during leakchecks because the third or fourth iteration of the + # test hangs on CPython 2/posix for some reason, likely due to + # the use of _close_on_teardown keeping something alive longer than intended. + # cf test__makefile_ref + with self.assertRaises(ssl.SSLError): + super(TestSSL, self).test_empty_send() + + @greentest.ignores_leakcheck + def test_sendall_nonblocking(self): + # Override; doesn't work with SSL sockets. + pass + + @greentest.ignores_leakcheck + def test_connect_with_type_flags_ignored(self): + # Override; doesn't work with SSL sockets. + pass + +def ssl_listener(address, private_key, certificate): + raw_listener = socket.socket() + greentest.bind_and_listen(raw_listener, address) + sock = ssl.wrap_socket(raw_listener, private_key, certificate) + return sock, raw_listener + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__subprocess.py b/src/gevent/tests/test__subprocess.py new file mode 100644 index 0000000..da5d2d5 --- /dev/null +++ b/src/gevent/tests/test__subprocess.py @@ -0,0 +1,491 @@ +import sys +import os +import errno +import unittest + +import time +import gc +import tempfile + +import gevent.testing as greentest +import gevent +from gevent.testing import mock +from gevent import subprocess + +if not hasattr(subprocess, 'mswindows'): + # PyPy3, native python subprocess + subprocess.mswindows = False + + +PYPY = hasattr(sys, 'pypy_version_info') +PY3 = sys.version_info[0] >= 3 + + +if subprocess.mswindows: + SETBINARY = 'import msvcrt; msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY);' +else: + SETBINARY = '' + + +python_universal_newlines = hasattr(sys.stdout, 'newlines') +# The stdlib of Python 3 on Windows doesn't properly handle universal newlines +# (it produces broken results compared to Python 2) +# See gevent.subprocess for more details. +python_universal_newlines_broken = PY3 and subprocess.mswindows + + +class Test(greentest.TestCase): + + def setUp(self): + super(Test, self).setUp() + gc.collect() + gc.collect() + + def test_exit(self): + popen = subprocess.Popen([sys.executable, '-c', 'import sys; sys.exit(10)']) + self.assertEqual(popen.wait(), 10) + + def test_wait(self): + popen = subprocess.Popen([sys.executable, '-c', 'import sys; sys.exit(11)']) + gevent.wait([popen]) + self.assertEqual(popen.poll(), 11) + + def test_child_exception(self): + try: + subprocess.Popen(['*']).wait() + except OSError as ex: + assert ex.errno == 2, ex + else: + raise AssertionError('Expected OSError: [Errno 2] No such file or directory') + + def test_leak(self): + num_before = greentest.get_number_open_files() + p = subprocess.Popen([sys.executable, "-c", "print()"], + stdout=subprocess.PIPE) + p.wait() + p.stdout.close() + del p + if PYPY: + gc.collect() + gc.collect() + + num_after = greentest.get_number_open_files() + self.assertEqual(num_before, num_after) + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_communicate(self): + p = subprocess.Popen([sys.executable, "-W", "ignore", + "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple");' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate(b"banana") + self.assertEqual(stdout, b"banana") + if sys.executable.endswith('-dbg'): + assert stderr.startswith(b'pineapple') + else: + self.assertEqual(stderr, b"pineapple") + + @greentest.skipIf(subprocess.mswindows, + "Windows does weird things here") + @greentest.skipOnLibuvOnCIOnPyPy("Sometimes segfaults") + def test_communicate_universal(self): + # Native string all the things. See https://github.com/gevent/gevent/issues/1039 + p = subprocess.Popen( + [ + sys.executable, + "-W", "ignore", + "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple\\r\\n\\xff\\xff\\xf2\\xf9\\r\\n");' + 'sys.stdout.write(sys.stdin.read())' + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + (stdout, stderr) = p.communicate('banana\r\n\xff\xff\xf2\xf9\r\n') + self.assertIsInstance(stdout, str) + self.assertIsInstance(stderr, str) + self.assertEqual(stdout, + 'banana\n\xff\xff\xf2\xf9\n') + + self.assertEqual(stderr, + 'pineapple\n\xff\xff\xf2\xf9\n') + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_universal1(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'sys.stdout.write("line1\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line2\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("line3\\r\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line4\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline5");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline6");'], + stdout=subprocess.PIPE, + universal_newlines=1, + bufsize=1) + try: + stdout = p.stdout.read() + if python_universal_newlines: + # Interpreter with universal newline support + if not python_universal_newlines_broken: + self.assertEqual(stdout, + "line1\nline2\nline3\nline4\nline5\nline6") + else: + # Note the extra newline after line 3 + self.assertEqual(stdout, + 'line1\nline2\nline3\n\nline4\n\nline5\nline6') + else: + # Interpreter without universal newline support + self.assertEqual(stdout, + "line1\nline2\rline3\r\nline4\r\nline5\nline6") + finally: + p.stdout.close() + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_universal2(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'sys.stdout.write("line1\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line2\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("line3\\r\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line4\\r\\nline5");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline6");'], + stdout=subprocess.PIPE, + universal_newlines=1, + bufsize=1) + try: + stdout = p.stdout.read() + if python_universal_newlines: + # Interpreter with universal newline support + if not python_universal_newlines_broken: + self.assertEqual(stdout, + "line1\nline2\nline3\nline4\nline5\nline6") + else: + # Note the extra newline after line 3 + self.assertEqual(stdout, + 'line1\nline2\nline3\n\nline4\n\nline5\nline6') + else: + # Interpreter without universal newline support + self.assertEqual(stdout, + "line1\nline2\rline3\r\nline4\r\nline5\nline6") + finally: + p.stdout.close() + + if sys.platform != 'win32': + + def test_nonblock_removed(self): + # see issue #134 + r, w = os.pipe() + stdin = subprocess.FileObject(r) + p = subprocess.Popen(['grep', 'text'], stdin=stdin) + try: + # Closing one half of the pipe causes Python 3 on OS X to terminate the + # child process; it exits with code 1 and the assert that p.poll is None + # fails. Removing the close lets it pass under both Python 3 and 2.7. + # If subprocess.Popen._remove_nonblock_flag is changed to a noop, then + # the test fails (as expected) even with the close removed + #os.close(w) + time.sleep(0.1) + self.assertEqual(p.poll(), None) + finally: + if p.poll() is None: + p.kill() + stdin.close() + os.close(w) + + def test_issue148(self): + for _ in range(7): + try: + subprocess.Popen('this_name_must_not_exist') + except OSError as ex: + if ex.errno != errno.ENOENT: + raise + else: + raise AssertionError('must fail with ENOENT') + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_check_output_keyword_error(self): + try: + subprocess.check_output([sys.executable, '-c', 'import sys; sys.exit(44)']) + except subprocess.CalledProcessError as e: # pylint:disable=no-member + self.assertEqual(e.returncode, 44) + else: + raise AssertionError('must fail with CalledProcessError') + + def test_popen_bufsize(self): + # Test that subprocess has unbuffered output by default + # (as the vanilla subprocess module) + if PY3: + # The default changed under python 3. + return + p = subprocess.Popen([sys.executable, '-u', '-c', + 'import sys; sys.stdout.write(sys.stdin.readline())'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.stdin.write(b'foobar\n') + r = p.stdout.readline() + self.assertEqual(r, b'foobar\n') + + @greentest.ignores_leakcheck + @greentest.skipOnWindows("Not sure why?") + def test_subprocess_in_native_thread(self): + # gevent.subprocess doesn't work from a background + # native thread. See #688 + from gevent import monkey + + # must be a native thread; defend against monkey-patching + ex = [] + Thread = monkey.get_original('threading', 'Thread') + + def fn(): + with self.assertRaises(TypeError) as exc: + gevent.subprocess.Popen('echo 123', shell=True) + raise AssertionError("Should not be able to construct Popen") + ex.append(exc.exception) + + thread = Thread(target=fn) + thread.start() + thread.join() + + self.assertEqual(len(ex), 1) + self.assertTrue(isinstance(ex[0], TypeError), ex) + self.assertEqual(ex[0].args[0], 'child watchers are only available on the default loop') + + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def __test_no_output(self, kwargs, kind): + proc = subprocess.Popen([sys.executable, '-c', 'pass'], + stdout=subprocess.PIPE, + **kwargs) + stdout, stderr = proc.communicate() + + self.assertIsInstance(stdout, kind) + self.assertIsNone(stderr) + + @greentest.skipOnLibuvOnCIOnPyPy("Sometimes segfaults; " + "https://travis-ci.org/gevent/gevent/jobs/327357682") + def test_universal_newlines_text_mode_no_output_is_always_str(self): + # If the file is in universal_newlines mode, we should always get a str when + # there is no output. + # https://github.com/gevent/gevent/pull/939 + self.__test_no_output({'universal_newlines': True}, str) + + @greentest.skipIf(sys.version_info[:2] < (3, 6), "Need encoding argument") + def test_encoded_text_mode_no_output_is_str(self): + # If the file is in universal_newlines mode, we should always get a str when + # there is no output. + # https://github.com/gevent/gevent/pull/939 + self.__test_no_output({'encoding': 'utf-8'}, str) + + def test_default_mode_no_output_is_always_str(self): + # If the file is in default mode, we should always get a str when + # there is no output. + # https://github.com/gevent/gevent/pull/939 + self.__test_no_output({}, bytes) + +@greentest.skipOnWindows("Testing POSIX fd closing") +class TestFDs(unittest.TestCase): + + @mock.patch('os.closerange') + @mock.patch('gevent.subprocess._set_inheritable') + @mock.patch('os.close') + def test_close_fds_brute_force(self, close, set_inheritable, closerange): + keep = ( + 4, 5, + # Leave a hole + # 6, + 7, + ) + subprocess.Popen._close_fds_brute_force(keep, None) + + closerange.assert_has_calls([ + mock.call(3, 4), + mock.call(8, subprocess.MAXFD), + ]) + + set_inheritable.assert_has_calls([ + mock.call(4, True), + mock.call(5, True), + ]) + + close.assert_called_once_with(6) + + @mock.patch('gevent.subprocess.Popen._close_fds_brute_force') + @mock.patch('os.listdir') + def test_close_fds_from_path_bad_values(self, listdir, brute_force): + listdir.return_value = 'Not an Integer' + + subprocess.Popen._close_fds_from_path('path', [], 42) + brute_force.assert_called_once_with([], 42) + + @mock.patch('os.listdir') + @mock.patch('os.closerange') + @mock.patch('gevent.subprocess._set_inheritable') + @mock.patch('os.close') + def test_close_fds_from_path(self, close, set_inheritable, closerange, listdir): + keep = ( + 4, 5, + # Leave a hole + # 6, + 7, + ) + listdir.return_value = ['1', '6', '37'] + + subprocess.Popen._close_fds_from_path('path', keep, 5) + + self.assertEqual([], closerange.mock_calls) + + set_inheritable.assert_has_calls([ + mock.call(4, True), + mock.call(7, True), + ]) + + close.assert_has_calls([ + mock.call(6), + mock.call(37), + ]) + + @mock.patch('gevent.subprocess.Popen._close_fds_brute_force') + @mock.patch('os.path.isdir') + def test_close_fds_no_dir(self, isdir, brute_force): + isdir.return_value = False + + subprocess.Popen._close_fds([], 42) + brute_force.assert_called_once_with([], 42) + isdir.assert_has_calls([ + mock.call('/proc/self/fd'), + mock.call('/dev/fd'), + ]) + + @mock.patch('gevent.subprocess.Popen._close_fds_from_path') + @mock.patch('gevent.subprocess.Popen._close_fds_brute_force') + @mock.patch('os.path.isdir') + def test_close_fds_with_dir(self, isdir, brute_force, from_path): + isdir.return_value = True + + subprocess.Popen._close_fds([7], 42) + + self.assertEqual([], brute_force.mock_calls) + from_path.assert_called_once_with('/proc/self/fd', [7], 42) + +class RunFuncTestCase(greentest.TestCase): + # Based on code from python 3.6 + + __timeout__ = greentest.LARGE_TIMEOUT + + def run_python(self, code, **kwargs): + """Run Python code in a subprocess using subprocess.run""" + argv = [sys.executable, "-c", code] + return subprocess.run(argv, **kwargs) + + def test_returncode(self): + # call() function with sequence argument + cp = self.run_python("import sys; sys.exit(47)") + self.assertEqual(cp.returncode, 47) + with self.assertRaises(subprocess.CalledProcessError): # pylint:disable=no-member + cp.check_returncode() + + def test_check(self): + with self.assertRaises(subprocess.CalledProcessError) as c: # pylint:disable=no-member + self.run_python("import sys; sys.exit(47)", check=True) + self.assertEqual(c.exception.returncode, 47) + + def test_check_zero(self): + # check_returncode shouldn't raise when returncode is zero + cp = self.run_python("import sys; sys.exit(0)", check=True) + self.assertEqual(cp.returncode, 0) + + def test_timeout(self): + # run() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.run waits for the + # child. + with self.assertRaises(subprocess.TimeoutExpired): + self.run_python("while True: pass", timeout=0.0001) + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_capture_stdout(self): + # capture stdout with zero return code + cp = self.run_python("print('BDFL')", stdout=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stdout) + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_capture_stderr(self): + cp = self.run_python("import sys; sys.stderr.write('BDFL')", + stderr=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stderr) + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_check_output_stdin_arg(self): + # run() can be called with stdin set to a file + with tempfile.TemporaryFile() as tf: + tf.write(b'pear') + tf.seek(0) + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + stdin=tf, stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + input=b'pear', stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_check_output_stdin_with_input_arg(self): + # run() refuses to accept 'stdin' with 'input' + with tempfile.TemporaryFile() as tf: + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError, + msg="Expected ValueError when stdin and input args supplied.") as c: + self.run_python("print('will not be run')", + stdin=tf, input=b'hare') + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_check_output_timeout(self): + with self.assertRaises(subprocess.TimeoutExpired) as c: + self.run_python( + ( + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)" + ), + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3, stdout=subprocess.PIPE) + self.assertEqual(c.exception.output, b'BDFL') + # output is aliased to stdout + self.assertEqual(c.exception.stdout, b'BDFL') + + def test_run_kwargs(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + cp = self.run_python(('import sys, os;' + 'sys.exit(33 if os.getenv("FRUIT")=="banana" else 31)'), + env=newenv) + self.assertEqual(cp.returncode, 33) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__subprocess_interrupted.py b/src/gevent/tests/test__subprocess_interrupted.py new file mode 100644 index 0000000..ec27b0e --- /dev/null +++ b/src/gevent/tests/test__subprocess_interrupted.py @@ -0,0 +1,20 @@ +import sys + +if 'runtestcase' in sys.argv[1:]: # pragma: no cover + import gevent + import gevent.subprocess + gevent.spawn(sys.exit, 'bye') + # Look closely, this doesn't actually do anything, that's a string + # not a division + gevent.subprocess.Popen([sys.executable, '-c', '"1/0"']) + gevent.sleep(1) +else: + import subprocess + for _ in range(5): + out, err = subprocess.Popen([sys.executable, '-W', 'ignore', + __file__, 'runtestcase'], + stderr=subprocess.PIPE).communicate() + if b'refs' in err: # Something to do with debug mode python builds? + assert err.startswith(b'bye'), repr(err) # pragma: no cover + else: + assert err.strip() == b'bye', repr(err) diff --git a/src/gevent/tests/test__subprocess_poll.py b/src/gevent/tests/test__subprocess_poll.py new file mode 100644 index 0000000..e817dff --- /dev/null +++ b/src/gevent/tests/test__subprocess_poll.py @@ -0,0 +1,9 @@ +import sys +from gevent.subprocess import Popen +from gevent.testing.util import alarm + +alarm(3) + +popen = Popen([sys.executable, '-c', 'pass']) +while popen.poll() is None: + pass diff --git a/src/gevent/tests/test__systemerror.py b/src/gevent/tests/test__systemerror.py new file mode 100644 index 0000000..7101509 --- /dev/null +++ b/src/gevent/tests/test__systemerror.py @@ -0,0 +1,108 @@ +import sys +import gevent.testing as greentest +import gevent +from gevent.hub import get_hub + +def raise_(ex): + raise ex + + +MSG = 'should be re-raised and caught' + + +class Test(greentest.TestCase): + x = None + error_fatal = False + + def start(self, *args): + raise NotImplementedError + + def setUp(self): + self.x = None + + def test_sys_exit(self): + self.start(sys.exit, MSG) + + try: + gevent.sleep(0.001) + except SystemExit as ex: + assert str(ex) == MSG, repr(str(ex)) + else: + raise AssertionError('must raise SystemExit') + + def test_keyboard_interrupt(self): + self.start(raise_, KeyboardInterrupt) + + try: + gevent.sleep(0.001) + except KeyboardInterrupt: + pass + else: + raise AssertionError('must raise KeyboardInterrupt') + + def test_keyboard_interrupt_stderr_patched(self): + from gevent import monkey + monkey.patch_sys(stdin=False, stdout=False, stderr=True) + try: + try: + self.start(raise_, KeyboardInterrupt) + while True: + gevent.sleep(0.1) + except KeyboardInterrupt: + pass # expected + finally: + sys.stderr = monkey.get_original('sys', 'stderr') + + def test_system_error(self): + self.start(raise_, SystemError(MSG)) + + with self.assertRaisesRegex(SystemError, + MSG): + gevent.sleep(0.002) + + def test_exception(self): + self.start(raise_, Exception('regular exception must not kill the program')) + gevent.sleep(0.001) + + +class TestCallback(Test): + + def tearDown(self): + if self.x is not None: + # libuv: See the notes in libuv/loop.py:loop._start_callback_timer + # If that's broken, test_exception can fail sporadically. + # If the issue is the same, then adding `gevent.sleep(0)` here + # will solve it. There's also a race condition for the first loop, + # so we sleep twice. + assert not self.x.pending, self.x + + def start(self, *args): + self.x = get_hub().loop.run_callback(*args) + + if greentest.LIBUV: + def test_exception(self): + # This call will enter the loop for the very first time (if we're running + # standalone). On libuv, where timers run first, that means that depending on the + # amount of time that elapses between the call to uv_timer_start and uv_run, + # this timer might fire before our check or prepare watchers, and hence callbacks, + # run. + # We make this call now so that the call in the super class is guaranteed to be + # somewhere in the loop and not subject to that race condition. + gevent.sleep(0.001) + super(TestCallback, self).test_exception() + +class TestSpawn(Test): + + def tearDown(self): + gevent.sleep(0.0001) + if self.x is not None: + assert self.x.dead, self.x + + def start(self, *args): + self.x = gevent.spawn(*args) + + +del Test + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__threading.py b/src/gevent/tests/test__threading.py new file mode 100644 index 0000000..0a591fc --- /dev/null +++ b/src/gevent/tests/test__threading.py @@ -0,0 +1,50 @@ +from gevent import monkey; monkey.patch_all() +import gevent.hub + +# check that the locks initialized by 'threading' did not init the hub +assert gevent.hub._get_hub() is None, 'monkey.patch_all() should not init hub' + +import gevent +import gevent.testing as greentest +import threading + + +def helper(): + threading.currentThread() + gevent.sleep(0.2) + + +class Test(greentest.TestCase): + + def _do_test(self, spawn): + before = len(threading._active) + g = spawn(helper) + gevent.sleep(0.1) + self.assertEqual(len(threading._active), before + 1) + try: + g.join() + except AttributeError: + while not g.dead: + gevent.sleep() + # Raw greenlet has no join(), uses a weakref to cleanup. + # so the greenlet has to die. On CPython, it's enough to + # simply delete our reference. + del g + # On PyPy, it might take a GC, but for some reason, even + # running several GC's doesn't clean it up under 5.6.0. + # So we skip the test. + #import gc + #gc.collect() + + self.assertEqual(len(threading._active), before) + + + def test_cleanup_gevent(self): + self._do_test(gevent.spawn) + + @greentest.skipOnPyPy("weakref is not cleaned up in a timely fashion") + def test_cleanup_raw(self): + self._do_test(gevent.spawn_raw) + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__threading_2.py b/src/gevent/tests/test__threading_2.py new file mode 100644 index 0000000..b425c88 --- /dev/null +++ b/src/gevent/tests/test__threading_2.py @@ -0,0 +1,610 @@ +# testing gevent's Event, Lock, RLock, Semaphore, BoundedSemaphore with standard test_threading +from __future__ import print_function + +from gevent.testing.six import xrange +import gevent.testing as greentest + +setup_ = '''from gevent import monkey; monkey.patch_all() +from gevent.event import Event +from gevent.lock import RLock, Semaphore, BoundedSemaphore +from gevent.thread import allocate_lock as Lock +import threading +threading.Event = Event +threading.Lock = Lock +# NOTE: We're completely patching around the allocate_lock +# patch we try to do with RLock; our monkey patch doesn't +# behave this way, but we do it in tests to make sure that +# our RLock implementation behaves correctly by itself. +# However, we must test the patched version too, so make it +# available. +threading.NativeRLock = threading.RLock +threading.RLock = RLock +threading.Semaphore = Semaphore +threading.BoundedSemaphore = BoundedSemaphore +''' + +exec(setup_) + +setup_3 = '\n'.join(' %s' % line for line in setup_.split('\n')) +setup_4 = '\n'.join(' %s' % line for line in setup_.split('\n')) + + +try: + from test.support import verbose +except ImportError: + from test.test_support import verbose +import random +import re +import sys +import threading +try: + import thread +except ImportError: + import _thread as thread +import time +import unittest +import weakref + +from gevent.tests import lock_tests + +# A trivial mutable counter. + +def skipDueToHang(cls): + return unittest.skipIf( + greentest.PYPY3 and greentest.RUNNING_ON_CI, + "SKIPPED: Timeout on PyPy3 on Travis" + )(cls) + +class Counter(object): + def __init__(self): + self.value = 0 + + def inc(self): + self.value += 1 + + def dec(self): + self.value -= 1 + + def get(self): + return self.value + + +class TestThread(threading.Thread): + def __init__(self, name, testcase, sema, mutex, nrunning): + threading.Thread.__init__(self, name=name) + self.testcase = testcase + self.sema = sema + self.mutex = mutex + self.nrunning = nrunning + + def run(self): + delay = random.random() / 10000.0 + if verbose: + print('task %s will run for %.1f usec' % ( + self.name, delay * 1e6)) + + with self.sema: + with self.mutex: + self.nrunning.inc() + if verbose: + print(self.nrunning.get(), 'tasks are running') + self.testcase.assertLessEqual(self.nrunning.get(), 3) + + time.sleep(delay) + if verbose: + print('task', self.name, 'done') + + with self.mutex: + self.nrunning.dec() + self.testcase.assertGreaterEqual(self.nrunning.get(), 0) + if verbose: + print('%s is finished. %d tasks are running' % ( + self.name, self.nrunning.get())) + +@skipDueToHang +class ThreadTests(unittest.TestCase): + + # Create a bunch of threads, let each do some work, wait until all are + # done. + def test_various_ops(self): + # This takes about n/3 seconds to run (about n/3 clumps of tasks, + # times about 1 second per clump). + NUMTASKS = 10 + + # no more than 3 of the 10 can run at once + sema = threading.BoundedSemaphore(value=3) + mutex = threading.RLock() + numrunning = Counter() + + threads = [] + + for i in range(NUMTASKS): + t = TestThread("" % i, self, sema, mutex, numrunning) + threads.append(t) + t.daemon = False # Under PYPY we get daemon by default? + if hasattr(t, 'ident'): + self.assertIsNone(t.ident) + self.assertFalse(t.daemon) + self.assertTrue(re.match(r'', repr(t))) + t.start() + + if verbose: + print('waiting for all tasks to complete') + for t in threads: + t.join(NUMTASKS) + self.assertFalse(t.is_alive()) + if hasattr(t, 'ident'): + self.assertNotEqual(t.ident, 0) + self.assertFalse(t.ident is None) + self.assertTrue(re.match(r'', repr(t))) + if verbose: + print('all tasks done') + self.assertEqual(numrunning.get(), 0) + + def test_ident_of_no_threading_threads(self): + # The ident still must work for the main thread and dummy threads, + # as must the repr and str. + + t = threading.currentThread() + self.assertFalse(t.ident is None) + str(t) + repr(t) + + def f(): + t = threading.currentThread() + ident.append(t.ident) + str(t) + repr(t) + done.set() + + done = threading.Event() + ident = [] + thread.start_new_thread(f, ()) + done.wait() + self.assertFalse(ident[0] is None) + # Kill the "immortal" _DummyThread + del threading._active[ident[0]] + + # run with a small(ish) thread stack size (256kB) + def test_various_ops_small_stack(self): + if verbose: + print('with 256kB thread stack size...') + try: + threading.stack_size(262144) + except thread.error: + if verbose: + print('platform does not support changing thread stack size') + return + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1MB) + def test_various_ops_large_stack(self): + if verbose: + print('with 1MB thread stack size...') + try: + threading.stack_size(0x100000) + except thread.error: + if verbose: + print('platform does not support changing thread stack size') + return + self.test_various_ops() + threading.stack_size(0) + + def test_foreign_thread(self): + # Check that a "foreign" thread can use the threading module. + def f(mutex): + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + threading.current_thread() + mutex.release() + + mutex = threading.Lock() + mutex.acquire() + tid = thread.start_new_thread(f, (mutex,)) + # Wait for the thread to finish. + mutex.acquire() + self.assertIn(tid, threading._active) + self.assertIsInstance(threading._active[tid], + threading._DummyThread) + del threading._active[tid] + # in gevent, we actually clean up threading._active, but it's not happended there yet + + # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) + # exposed at the Python level. This test relies on ctypes to get at it. + def SKIP_test_PyThreadState_SetAsyncExc(self): + try: + import ctypes + except ImportError: + if verbose: + print("test_PyThreadState_SetAsyncExc can't import ctypes") + return # can't do anything + + set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc + + class AsyncExc(Exception): + pass + + exception = ctypes.py_object(AsyncExc) + + # `worker_started` is set by the thread when it's inside a try/except + # block waiting to catch the asynchronously set AsyncExc exception. + # `worker_saw_exception` is set by the thread upon catching that + # exception. + worker_started = threading.Event() + worker_saw_exception = threading.Event() + + class Worker(threading.Thread): + id = None + finished = False + + def run(self): + self.id = thread.get_ident() + self.finished = False + + try: + while True: + worker_started.set() + time.sleep(0.1) + except AsyncExc: + self.finished = True + worker_saw_exception.set() + + t = Worker() + t.daemon = True # so if this fails, we don't hang Python at shutdown + t.start() + if verbose: + print(" started worker thread") + + # Try a thread id that doesn't make sense. + if verbose: + print(" trying nonsensical thread id") + result = set_async_exc(ctypes.c_long(-1), exception) + self.assertEqual(result, 0) # no thread states modified + + # Now raise an exception in the worker thread. + if verbose: + print(" waiting for worker thread to get started") + worker_started.wait() + if verbose: + print(" verifying worker hasn't exited") + self.assertFalse(t.finished) + if verbose: + print(" attempting to raise asynch exception in worker") + result = set_async_exc(ctypes.c_long(t.id), exception) + self.assertEqual(result, 1) # one thread state modified + if verbose: + print(" waiting for worker to say it caught the exception") + worker_saw_exception.wait(timeout=10) + self.assertTrue(t.finished) + if verbose: + print(" all OK -- joining worker") + if t.finished: + t.join() + # else the thread is still running, and we have no way to kill it + + def test_limbo_cleanup(self): + # Issue 7481: Failure to start thread should cleanup the limbo map. + def fail_new_thread(*_args): + raise thread.error() + _start_new_thread = threading._start_new_thread + threading._start_new_thread = fail_new_thread + try: + t = threading.Thread(target=lambda: None) + self.assertRaises(thread.error, t.start) + self.assertFalse( + t in threading._limbo, + "Failed to cleanup _limbo map on failure of Thread.start().") + finally: + threading._start_new_thread = _start_new_thread + + def test_finalize_runnning_thread(self): + # Issue 1402: the PyGILState_Ensure / _Release functions may be called + # very late on python exit: on deallocation of a running thread for + # example. + try: + import ctypes + getattr(ctypes, 'pythonapi') # not available on PyPy + getattr(ctypes.pythonapi, 'PyGILState_Ensure') # not available on PyPy3 + except (ImportError, AttributeError): + if verbose: + print("test_finalize_with_runnning_thread can't import ctypes") + return # can't do anything + + del ctypes # pyflakes fix + + import subprocess + rc = subprocess.call([sys.executable, "-W", "ignore", "-c", """if 1: +%s + import ctypes, sys, time + try: + import thread + except ImportError: + import _thread as thread # Py3 + + # This lock is used as a simple event variable. + ready = thread.allocate_lock() + ready.acquire() + + # Module globals are cleared before __del__ is run + # So we save the functions in class dict + class C: + ensure = ctypes.pythonapi.PyGILState_Ensure + release = ctypes.pythonapi.PyGILState_Release + def __del__(self): + state = self.ensure() + self.release(state) + + def waitingThread(): + x = C() + ready.release() + time.sleep(100) + + thread.start_new_thread(waitingThread, ()) + ready.acquire() # Be sure the other thread is waiting. + sys.exit(42) + """ % setup_3]) + self.assertEqual(rc, 42) + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_join_nondaemon_on_shutdown(self): + # Issue 1722344 + # Raising SystemExit skipped threading._shutdown + import subprocess + p = subprocess.Popen([sys.executable, "-W", "ignore", "-c", """if 1: +%s + import threading + from time import sleep + + def child(): + sleep(1) + # As a non-daemon thread we SHOULD wake up and nothing + # should be torn down yet + print("Woke up, sleep function is: %%r" %% sleep) + + threading.Thread(target=child).start() + raise SystemExit + """ % setup_4], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + stdout = stdout.strip() + stdout = stdout.decode('utf-8') + stderr = stderr.decode('utf-8') + assert re.match('^Woke up, sleep function is: <.*?sleep.*?>$', stdout), repr(stdout) + stderr = re.sub(r"^\[\d+ refs\]", "", stderr, re.MULTILINE).strip() + # On Python 2, importing pkg_resources tends to result in some 'ImportWarning' + # being printed to stderr about packages missing __init__.py; the -W ignore is... + # ignored. + # self.assertEqual(stderr, "") + + def test_enumerate_after_join(self): + # Try hard to trigger #1703448: a thread is still returned in + # threading.enumerate() after it has been join()ed. + enum = threading.enumerate + import warnings + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + # get/set checkinterval are deprecated in Python 3 + old_interval = sys.getcheckinterval() + try: + for i in xrange(1, 100): + # Try a couple times at each thread-switching interval + # to get more interleavings. + sys.setcheckinterval(i // 5) + t = threading.Thread(target=lambda: None) + t.start() + t.join() + l = enum() + self.assertFalse(t in l, + "#1703448 triggered after %d trials: %s" % (i, l)) + finally: + sys.setcheckinterval(old_interval) + + if not hasattr(sys, 'pypy_version_info'): + def test_no_refcycle_through_target(self): + class RunSelfFunction(object): + def __init__(self, should_raise): + # The links in this refcycle from Thread back to self + # should be cleaned up when the thread completes. + self.should_raise = should_raise + self.thread = threading.Thread(target=self._run, + args=(self,), + kwargs={'yet_another': self}) + self.thread.start() + + def _run(self, _other_ref, _yet_another): + if self.should_raise: + raise SystemExit + + cyclic_object = RunSelfFunction(should_raise=False) + weak_cyclic_object = weakref.ref(cyclic_object) + cyclic_object.thread.join() + del cyclic_object + self.assertIsNone(weak_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_cyclic_object()))) + + raising_cyclic_object = RunSelfFunction(should_raise=True) + weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) + raising_cyclic_object.thread.join() + del raising_cyclic_object + self.assertIsNone(weak_raising_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_raising_cyclic_object()))) + +@skipDueToHang +class ThreadJoinOnShutdown(unittest.TestCase): + + def _run_and_join(self, script): + script = """if 1: +%s + import sys, os, time, threading + # a thread, which waits for the main program to terminate + def joiningfunc(mainthread): + mainthread.join() + print('end of thread') + \n""" % setup_3 + script + + import subprocess + p = subprocess.Popen([sys.executable, "-W", "ignore", "-c", script], stdout=subprocess.PIPE) + rc = p.wait() + data = p.stdout.read().replace(b'\r', b'') + p.stdout.close() + self.assertEqual(data, b"end of main\nend of thread\n") + self.assertNotEqual(rc, 2, b"interpreter was blocked") + self.assertEqual(rc, 0, b"Unexpected error") + + @greentest.skipOnLibuvOnPyPyOnWin("hangs") + def test_1_join_on_shutdown(self): + # The usual case: on exit, wait for a non-daemon thread + script = """if 1: + import os + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + time.sleep(0.1) + print('end of main') + """ + self._run_and_join(script) + + @greentest.skipOnPyPy3OnCI("Sometimes randomly times out") + def test_2_join_in_forked_process(self): + # Like the test above, but from a forked interpreter + import os + if not hasattr(os, 'fork'): + return + script = """if 1: + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + print('end of main') + """ + self._run_and_join(script) + + def test_3_join_in_forked_from_thread(self): + # Like the test above, but fork() was called from a worker thread + # In the forked process, the main Thread object must be marked as stopped. + import os + if not hasattr(os, 'fork'): + return + # Skip platforms with known problems forking from a worker thread. + # See http://bugs.python.org/issue3863. + # skip disable because I think the bug shouldn't apply to gevent -- denis + #if sys.platform in ('freebsd4', 'freebsd5', 'freebsd6', 'os2emx'): + # print(('Skipping test_3_join_in_forked_from_thread' + # ' due to known OS bugs on'), sys.platform, file=sys.stderr) + # return + + # A note on CFFI: Under Python 3, using CFFI tends to initialize the GIL, + # whether or not we spawn any actual threads. Now, os.fork() calls + # PyEval_ReInitThreads, which only does any work of the GIL has been taken. + # One of the things it does is call threading._after_fork to reset + # some thread state, which causes the main thread (threading._main_thread) + # to be reset to the current thread---which for Python >= 3.4 happens + # to be our version of thread, gevent.threading.Thread, which doesn't + # initialize the _tstate_lock ivar. This causes threading._shutdown to crash + # with an AssertionError and this test to fail. We hack around this by + # making sure _after_fork is not called in the child process. + # XXX: Figure out how to really fix that. + + script = """if 1: + main_thread = threading.current_thread() + def worker(): + threading._after_fork = lambda: None + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(main_thread,)) + print('end of main') + t.start() + t.join() # Should not block: main_thread is already stopped + + w = threading.Thread(target=worker) + w.start() + import sys + if sys.version_info[:2] >= (3, 7) or (sys.version_info[:2] >= (3, 5) and hasattr(sys, 'pypy_version_info') and sys.platform != 'darwin'): + w.join() + """ + # In PyPy3 5.8.0, if we don't wait on this top-level "thread", 'w', + # we never see "end of thread". It's not clear why, since that's being + # done in a child of this process. Yet in normal CPython 3, waiting on this + # causes the whole process to lock up (possibly because of some loop within + # the interpreter waiting on thread locks, like the issue described in threading.py + # for Python 3.4? in any case, it doesn't hang in Python 2.) This changed in + # 3.7a1 and waiting on it is again necessary and doesn't hang. + # PyPy3 5.10.1 is back to the "old" cpython behaviour, and waiting on it + # causes the whole process to hang, but apparently only on OS X---linux was fine without it + self._run_and_join(script) + + +@skipDueToHang +class ThreadingExceptionTests(unittest.TestCase): + # A RuntimeError should be raised if Thread.start() is called + # multiple times. + # pylint:disable=bad-thread-instantiation + def test_start_thread_again(self): + thread_ = threading.Thread() + thread_.start() + self.assertRaises(RuntimeError, thread_.start) + + def test_joining_current_thread(self): + current_thread = threading.current_thread() + self.assertRaises(RuntimeError, current_thread.join) + + def test_joining_inactive_thread(self): + thread_ = threading.Thread() + self.assertRaises(RuntimeError, thread_.join) + + def test_daemonize_active_thread(self): + thread_ = threading.Thread() + thread_.start() + self.assertRaises(RuntimeError, setattr, thread_, "daemon", True) + + +@skipDueToHang +class LockTests(lock_tests.LockTests): + locktype = staticmethod(threading.Lock) + +@skipDueToHang +class RLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading.RLock) + +@skipDueToHang +class NativeRLockTests(lock_tests.RLockTests): + # See comments at the top of the file for the difference + # between this and RLockTests, and why they both matter + locktype = staticmethod(threading.NativeRLock) + +@skipDueToHang +class EventTests(lock_tests.EventTests): + eventtype = staticmethod(threading.Event) + +@skipDueToHang +class ConditionAsRLockTests(lock_tests.RLockTests): + # An Condition uses an RLock by default and exports its API. + locktype = staticmethod(threading.Condition) + +@skipDueToHang +class ConditionTests(lock_tests.ConditionTests): + condtype = staticmethod(threading.Condition) + +@skipDueToHang +class SemaphoreTests(lock_tests.SemaphoreTests): + semtype = staticmethod(threading.Semaphore) + +@skipDueToHang +class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): + semtype = staticmethod(threading.BoundedSemaphore) + + +if __name__ == "__main__": + greentest.main() diff --git a/src/gevent/tests/test__threading_before_monkey.py b/src/gevent/tests/test__threading_before_monkey.py new file mode 100644 index 0000000..b60fb30 --- /dev/null +++ b/src/gevent/tests/test__threading_before_monkey.py @@ -0,0 +1,23 @@ +# If stdlib threading is imported *BEFORE* monkey patching, +# we can still get the current (main) thread, and it's not a DummyThread. + +import threading +from gevent import monkey +monkey.patch_all() + +import gevent.testing as greentest + + +class Test(greentest.TestCase): + + def test_main_thread(self): + current = threading.current_thread() + self.assertFalse(isinstance(current, threading._DummyThread)) + self.assertTrue(isinstance(current, monkey.get_original('threading', 'Thread'))) + # in 3.4, if the patch is incorrectly done, getting the repr + # of the thread fails + repr(current) + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__threading_holding_lock_while_monkey.py b/src/gevent/tests/test__threading_holding_lock_while_monkey.py new file mode 100644 index 0000000..c5aed4a --- /dev/null +++ b/src/gevent/tests/test__threading_holding_lock_while_monkey.py @@ -0,0 +1,8 @@ +from gevent import monkey +import threading +# Make sure that we can patch gevent while holding +# a threading lock. Under Python2, where RLock is implemented +# in python code, this used to throw RuntimeErro("Cannot release un-acquired lock") +# See https://github.com/gevent/gevent/issues/615 +with threading.RLock(): + monkey.patch_all() diff --git a/src/gevent/tests/test__threading_monkey_in_thread.py b/src/gevent/tests/test__threading_monkey_in_thread.py new file mode 100644 index 0000000..f3ff73d --- /dev/null +++ b/src/gevent/tests/test__threading_monkey_in_thread.py @@ -0,0 +1,65 @@ +# We can monkey-patch in a thread, but things don't work as expected. +from __future__ import print_function +import sys +import threading +from gevent import monkey +import gevent.testing as greentest + + +class Test(greentest.TestCase): + + @greentest.ignores_leakcheck # can't be run multiple times + def test_patch_in_thread(self): + all_warnings = [] + try: + get_ident = threading.get_ident + except AttributeError: + get_ident = threading._get_ident + + def process_warnings(warnings): + all_warnings.extend(warnings) + monkey._process_warnings = process_warnings + + current = threading.current_thread() + current_id = get_ident() + + def target(): + tcurrent = threading.current_thread() + monkey.patch_all() + tcurrent2 = threading.current_thread() + self.assertIsNot(tcurrent, current) + # We get a dummy thread now + self.assertIsNot(tcurrent, tcurrent2) + + thread = threading.Thread(target=target) + thread.start() + try: + thread.join() + except: # pylint:disable=bare-except + # XXX: This can raise LoopExit in some cases. + greentest.reraiseFlakyTestRaceCondition() + + self.assertNotIsInstance(current, threading._DummyThread) + self.assertIsInstance(current, monkey.get_original('threading', 'Thread')) + + + # We generated some warnings + if sys.version_info >= (3, 4): + self.assertEqual(all_warnings, + ['Monkey-patching outside the main native thread. Some APIs will not be ' + 'available. Expect a KeyError to be printed at shutdown.', + 'Monkey-patching not on the main thread; threading.main_thread().join() ' + 'will hang from a greenlet']) + else: + self.assertEqual(all_warnings, + ['Monkey-patching outside the main native thread. Some APIs will not be ' + 'available. Expect a KeyError to be printed at shutdown.']) + + + # Manual clean up so we don't get a KeyError + del threading._active[current_id] + threading._active[(getattr(threading, 'get_ident', None) or threading._get_ident)()] = current + + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test__threading_native_before_monkey.py b/src/gevent/tests/test__threading_native_before_monkey.py new file mode 100644 index 0000000..dd29ab6 --- /dev/null +++ b/src/gevent/tests/test__threading_native_before_monkey.py @@ -0,0 +1,57 @@ +# If stdlib threading is imported *BEFORE* monkey patching, *and* +# there is a native thread created, we can still get the current +# (main) thread, and it's not a DummyThread. +# Joining the native thread also does not fail + +import threading +from time import sleep as time_sleep + +import gevent.testing as greentest + +class NativeThread(threading.Thread): + do_run = True + + def run(self): + while self.do_run: + time_sleep(0.1) + + def stop(self, timeout=None): + self.do_run = False + self.join(timeout=timeout) + +native_thread = None + +class Test(greentest.TestCase): + + def test_main_thread(self): + current = threading.current_thread() + self.assertFalse(isinstance(current, threading._DummyThread)) + self.assertTrue(isinstance(current, monkey.get_original('threading', 'Thread'))) + # in 3.4, if the patch is incorrectly done, getting the repr + # of the thread fails + repr(current) + + if hasattr(threading, 'main_thread'): # py 3.4 + self.assertEqual(threading.current_thread(), threading.main_thread()) + + @greentest.ignores_leakcheck # because it can't be run multiple times + def test_join_native_thread(self): + self.assertTrue(native_thread.is_alive()) + + native_thread.stop(timeout=1) + self.assertFalse(native_thread.is_alive()) + + # again, idempotent + native_thread.stop() + self.assertFalse(native_thread.is_alive()) + + +if __name__ == '__main__': + native_thread = NativeThread() + native_thread.start() + + # Only patch after we're running + from gevent import monkey + monkey.patch_all() + + greentest.main() diff --git a/src/gevent/tests/test__threading_patched_local.py b/src/gevent/tests/test__threading_patched_local.py new file mode 100644 index 0000000..5ff3352 --- /dev/null +++ b/src/gevent/tests/test__threading_patched_local.py @@ -0,0 +1,24 @@ +from gevent import monkey; monkey.patch_all() +import threading + + +localdata = threading.local() +localdata.x = "hello" +assert localdata.x == 'hello' +success = [] + + +def func(): + try: + getattr(localdata, 'x') + raise AssertionError('localdata.x must raise AttributeError') + except AttributeError: + pass + assert localdata.__dict__ == {}, localdata.__dict__ + success.append(1) + +t = threading.Thread(None, func) +t.start() +t.join() +assert success == [1], 'test failed' +assert localdata.x == 'hello' diff --git a/src/gevent/tests/test__threading_vs_settrace.py b/src/gevent/tests/test__threading_vs_settrace.py new file mode 100644 index 0000000..4324829 --- /dev/null +++ b/src/gevent/tests/test__threading_vs_settrace.py @@ -0,0 +1,163 @@ +from __future__ import print_function +import sys +import subprocess +import unittest +from gevent.thread import allocate_lock +import gevent.testing as greentest + +script = """ +from gevent import monkey +monkey.patch_all() +import sys, os, threading, time + + +# A deadlock-killer, to prevent the +# testsuite to hang forever +def killer(): + time.sleep(0.1) + sys.stdout.write('..program blocked; aborting!') + sys.stdout.flush() + os._exit(2) +t = threading.Thread(target=killer) +t.daemon = True +t.start() + + +def trace(frame, event, arg): + if threading is not None: + threading.currentThread() + return trace + + +def doit(): + sys.stdout.write("..thread started..") + + +def test1(): + t = threading.Thread(target=doit) + t.start() + t.join() + sys.settrace(None) + +sys.settrace(trace) +if len(sys.argv) > 1: + test1() + +sys.stdout.write("..finishing..") +""" + + +class TestTrace(unittest.TestCase): + @greentest.skipOnPurePython("Locks can be traced in Pure Python") + def test_untraceable_lock(self): + # Untraceable locks were part of the solution to https://bugs.python.org/issue1733757 + # which details a deadlock that could happen if a trace function invoked + # threading.currentThread at shutdown time---the cleanup lock would be held + # by the VM, and calling currentThread would try to acquire it again. The interpreter + # changed in 2.6 to use the `with` statement (https://hg.python.org/cpython/rev/76f577a9ec03/), + # which apparently doesn't trace in quite the same way. + if hasattr(sys, 'gettrace'): + old = sys.gettrace() + else: + old = None + + lst = [] + try: + def trace(frame, ev, _arg): + lst.append((frame.f_code.co_filename, frame.f_lineno, ev)) + print("TRACE: %s:%s %s" % lst[-1]) + return trace + + with allocate_lock(): + sys.settrace(trace) + finally: + sys.settrace(old) + + self.assertEqual(lst, [], "trace not empty") + + @greentest.skipOnPurePython("Locks can be traced in Pure Python") + def test_untraceable_lock_uses_different_lock(self): + if hasattr(sys, 'gettrace'): + old = sys.gettrace() + else: + old = None + + PY3 = sys.version_info[0] > 2 + lst = [] + # we should be able to use unrelated locks from within the trace function + l = allocate_lock() + try: + def trace(frame, ev, _arg): + with l: + lst.append((frame.f_code.co_filename, frame.f_lineno, ev)) + print("TRACE: %s:%s %s" % lst[-1]) + return trace + + l2 = allocate_lock() + sys.settrace(trace) + # Separate functions, not the C-implemented `with` so the trace + # function gets a crack at them + l2.acquire() + l2.release() + finally: + sys.settrace(old) + + if not PY3: + # Py3 overrides acquire in Python to do argument checking + self.assertEqual(lst, [], "trace not empty") + else: + # Have an assert so that we know if we miscompile + self.assertTrue(lst, "should not compile on pypy") + + @greentest.skipOnPurePython("Locks can be traced in Pure Python") + def test_untraceable_lock_uses_same_lock(self): + from gevent.hub import LoopExit + if hasattr(sys, 'gettrace'): + old = sys.gettrace() + else: + old = None + PY3 = sys.version_info[0] > 2 + lst = [] + e = None + # we should not be able to use the same lock from within the trace function + # because it's over acquired but instead of deadlocking it raises an exception + l = allocate_lock() + try: + def trace(frame, ev, _arg): + with l: + lst.append((frame.f_code.co_filename, frame.f_lineno, ev)) + return trace + + sys.settrace(trace) + # Separate functions, not the C-implemented `with` so the trace + # function gets a crack at them + l.acquire() + except LoopExit as ex: + e = ex + finally: + sys.settrace(old) + + if not PY3: + # Py3 overrides acquire in Python to do argument checking + self.assertEqual(lst, [], "trace not empty") + else: + # Have an assert so that we know if we miscompile + self.assertTrue(lst, "should not compile on pypy") + self.assertTrue(isinstance(e, LoopExit)) + + def run_script(self, more_args=()): + args = [sys.executable, "-c", script] + args.extend(more_args) + rc = subprocess.call(args) + self.assertNotEqual(rc, 2, "interpreter was blocked") + self.assertEqual(rc, 0, "Unexpected error") + + def test_finalize_with_trace(self): + self.run_script() + + def test_bootstrap_inner_with_trace(self): + self.run_script(["1"]) + + +if __name__ == "__main__": + greentest.main() diff --git a/src/gevent/tests/test__threadpool.py b/src/gevent/tests/test__threadpool.py new file mode 100644 index 0000000..2bbcbb7 --- /dev/null +++ b/src/gevent/tests/test__threadpool.py @@ -0,0 +1,653 @@ +from __future__ import print_function + +from time import time, sleep +import contextlib +import random +import weakref +import gc + +import gevent.testing as greentest +import gevent.threadpool +from gevent.threadpool import ThreadPool +import gevent + +from gevent.testing import ExpectedException +from gevent.testing import PYPY + + + +# pylint:disable=too-many-ancestors + + +@contextlib.contextmanager +def disabled_gc(): + was_enabled = gc.isenabled() + gc.disable() + try: + yield + finally: + if was_enabled: + gc.enable() + + +class TestCase(greentest.TestCase): + # These generally need more time + __timeout__ = greentest.LARGE_TIMEOUT + pool = None + + ClassUnderTest = ThreadPool + def _FUT(self): + return self.ClassUnderTest + + def _makeOne(self, size, increase=greentest.RUN_LEAKCHECKS): + self.pool = pool = self._FUT()(size) + if increase: + # Max size to help eliminate false positives + self.pool.size = size + return pool + + def cleanup(self): + pool = self.pool + if pool is not None: + kill = getattr(pool, 'kill', None) or getattr(pool, 'shutdown') + kill() + del kill + del self.pool + + if greentest.RUN_LEAKCHECKS: + # Each worker thread created a greenlet object and switched to it. + # It's a custom subclass, but even if it's not, it appears that + # the root greenlet for the new thread sticks around until there's a + # gc. Simply calling 'getcurrent()' is enough to "leak" a greenlet.greenlet + # and a weakref. + for _ in range(3): + gc.collect() + + +class PoolBasicTests(TestCase): + + def test_execute_async(self): + pool = self._makeOne(2) + r = [] + first = pool.spawn(r.append, 1) + first.get() + self.assertEqual(r, [1]) + gevent.sleep(0) + + pool.apply_async(r.append, (2, )) + self.assertEqual(r, [1]) + + pool.apply_async(r.append, (3, )) + self.assertEqual(r, [1]) + + pool.apply_async(r.append, (4, )) + self.assertEqual(r, [1]) + gevent.sleep(0.01) + self.assertEqualFlakyRaceCondition(sorted(r), [1, 2, 3, 4]) + + def test_apply(self): + pool = self._makeOne(1) + result = pool.apply(lambda a: ('foo', a), (1, )) + self.assertEqual(result, ('foo', 1)) + + def test_apply_raises(self): + pool = self._makeOne(1) + + def raiser(): + raise ExpectedException() + + with self.assertRaises(ExpectedException): + pool.apply(raiser) + # Don't let the metaclass automatically force any error + # that reaches the hub from a spawned greenlet to become + # fatal; that defeats the point of the test. + test_apply_raises.error_fatal = False + + def test_init_valueerror(self): + self.switch_expected = False + with self.assertRaises(ValueError): + self._makeOne(-1) + +# +# tests from standard library test/test_multiprocessing.py + + +class TimingWrapper(object): + + def __init__(self, the_func): + self.func = the_func + self.elapsed = None + + def __call__(self, *args, **kwds): + t = time() + try: + return self.func(*args, **kwds) + finally: + self.elapsed = time() - t + + +def sqr(x, wait=0.0): + sleep(wait) + return x * x + + +def sqr_random_sleep(x): + sleep(random.random() * 0.1) + return x * x + + +TIMEOUT1, TIMEOUT2, TIMEOUT3 = 0.082, 0.035, 0.14 + +class _AbstractPoolTest(TestCase): + + size = 1 + + MAP_IS_GEN = False + + def setUp(self): + greentest.TestCase.setUp(self) + self._makeOne(self.size) + + @greentest.ignores_leakcheck + def test_map(self): + pmap = self.pool.map + if self.MAP_IS_GEN: + pmap = lambda f, i: list(self.pool.map(f, i)) + self.assertEqual(pmap(sqr, range(10)), list(map(sqr, range(10)))) + self.assertEqual(pmap(sqr, range(100)), list(map(sqr, range(100)))) + + self.pool.kill() + del self.pool + del pmap + +SMALL_RANGE = 10 +LARGE_RANGE = 1000 + +if (greentest.PYPY and (greentest.WIN or greentest.RUN_COVERAGE)) or greentest.RUN_LEAKCHECKS: + # PyPy 5.10 is *really* slow at spawning or switching between + # threads (especially on Windows or when coverage is enabled) Tests that happen + # instantaneously on other platforms time out due to the overhead. + + # Leakchecks also take much longer due to all the calls into the GC, + # most especially on Python 3 + LARGE_RANGE = 50 + +class TestPool(_AbstractPoolTest): + + def test_greenlet_class(self): + from greenlet import getcurrent + from gevent.threadpool import _WorkerGreenlet + worker_greenlet = self.pool.apply(getcurrent) + + self.assertIsInstance(worker_greenlet, _WorkerGreenlet) + r = repr(worker_greenlet) + self.assertIn('ThreadPoolWorker', r) + self.assertIn('thread_ident', r) + self.assertIn('hub=', r) + + from gevent.util import format_run_info + + info = '\n'.join(format_run_info()) + self.assertIn(" + : Parent: None + : Greenlet Locals: + : Local at X + : {'foo': 42} + +--- + : Parent: + +--- ; finished with value + | +--- ; finished with exception ExpectedException() + : Parent: + +--- ; finished with value + | +--- ; finished with exception ExpectedException() + : Parent: + +--- ; finished with value + : Spawn Tree Locals + : {'stl': 'STL'} + | +--- ; finished with value + | +--- ; finished with exception ExpectedException() + : Parent: + +--- >>; finished with value + """.strip() + self.assertEqual(expected, value) + + @greentest.ignores_leakcheck + def test_tree_no_track(self): + gevent.config.track_greenlet_tree = False + self._build_tree() + + + @greentest.ignores_leakcheck + def test_forest_fake_parent(self): + from greenlet import greenlet as RawGreenlet + + def t4(): + # Ignore this one, make the child the parent, + # and don't be a child of the hub. + c = RawGreenlet(util.GreenletTree.current_tree) + c.parent.greenlet_tree_is_ignored = True + c.greenlet_tree_is_root = True + return c.switch() + + + g = RawGreenlet(t4) + tree = g.switch() + + tree_format = tree.format(details={'running_stacks': False, + 'spawning_stacks': False}) + value = self._normalize_tree_format(tree_format) + + expected = """\ +; not running + : Parent: + """.strip() + + self.assertEqual(expected, value) + + +class TestAssertSwitches(unittest.TestCase): + + def test_time_sleep(self): + # A real blocking function + from time import sleep + + # No time given, we detect the failure to switch immediately + with self.assertRaises(util._FailedToSwitch) as exc: + with util.assert_switches(): + sleep(0.001) + + message = str(exc.exception) + self.assertIn('To any greenlet in', message) + + # Supply a max blocking allowed and exceed it + with self.assertRaises(util._FailedToSwitch): + with util.assert_switches(0.001): + sleep(0.1) + + + # Supply a max blocking allowed, and exit before that happens, + # but don't switch to the hub as requested + with self.assertRaises(util._FailedToSwitch) as exc: + with util.assert_switches(0.001, hub_only=True): + sleep(0) + + message = str(exc.exception) + self.assertIn('To the hub in', message) + self.assertIn('(max allowed 0.0010 seconds)', message) + + # Supply a max blocking allowed, and exit before that happens, + # and allow any switch (or no switch). + # Note that we need to use a relatively long duration; + # sleep(0) on Windows can actually take a substantial amount of time + # sometimes (more than 0.001s) + with util.assert_switches(1.0, hub_only=False): + sleep(0) + + + def test_no_switches_no_function(self): + # No blocking time given, no switch performed: exception + with self.assertRaises(util._FailedToSwitch): + with util.assert_switches(): + pass + + # blocking time given, for all greenlets, no switch performed: nothing + with util.assert_switches(max_blocking_time=1, hub_only=False): + pass + + def test_exception_not_supressed(self): + + with self.assertRaises(NameError): + with util.assert_switches(): + raise NameError() + + def test_nested(self): + from greenlet import gettrace + with util.assert_switches() as outer: + self.assertEqual(gettrace(), outer.tracer) + self.assertIsNotNone(outer.tracer.active_greenlet) + + with util.assert_switches() as inner: + self.assertEqual(gettrace(), inner.tracer) + self.assertEqual(inner.tracer.previous_trace_function, outer.tracer) + + inner.tracer('switch', (self, self)) + + self.assertIs(self, inner.tracer.active_greenlet) + self.assertIs(self, outer.tracer.active_greenlet) + + self.assertEqual(gettrace(), outer.tracer) + +if __name__ == '__main__': + greentest.main() diff --git a/src/gevent/tests/test_server.crt b/src/gevent/tests/test_server.crt new file mode 100644 index 0000000..1379e1d --- /dev/null +++ b/src/gevent/tests/test_server.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICYzCCAcwCCQD5jx1Aa0dytjANBgkqhkiG9w0BAQQFADB2MQswCQYDVQQGEwJU +UzENMAsGA1UECBMEVGVzdDENMAsGA1UEBxMEVGVzdDEWMBQGA1UEChMNVGVzdCBF +dmVudGxldDENMAsGA1UECxMEVGVzdDENMAsGA1UEAxMEVGVzdDETMBEGCSqGSIb3 +DQEJARYEVGVzdDAeFw0wODA3MDgyMTExNDJaFw0xMDAyMDgwODE1MTBaMHYxCzAJ +BgNVBAYTAlRTMQ0wCwYDVQQIEwRUZXN0MQ0wCwYDVQQHEwRUZXN0MRYwFAYDVQQK +Ew1UZXN0IEV2ZW50bGV0MQ0wCwYDVQQLEwRUZXN0MQ0wCwYDVQQDEwRUZXN0MRMw +EQYJKoZIhvcNAQkBFgRUZXN0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDM +WcyeIiHQuEGQxgTIvu0aOW4iRFAyUEi8pLWNCxMEHglF8k6OxFVq7XWZMDnDFVnb +ZjmQh5Tc21Ae6cXzxXln578fROXHEzXo3Is8HUlq3ug1yYOGHjxw++Opjf1uoHwP +EBUKsz/flS7knuscgFM9FO05KSPn2wHnZeIDta4yTwIDAQABMA0GCSqGSIb3DQEB +BAUAA4GBAKM71aP0r26gEEEBzovfXm1IwKav6R9/xiWsJ4pFsUXVotcaIjcVBDG1 +Z7tz688hokb+GNxsTI2gNfqanqUnfP9wZxnKRmfTSOvb5aWHIiaiMXSgjiPlqBcm +6mnSeEbSMM9cw479wWhh1YqY8tf3gYJa+sxznVWLSfVLpsjRMphe +-----END CERTIFICATE----- diff --git a/src/gevent/tests/test_server.key b/src/gevent/tests/test_server.key new file mode 100644 index 0000000..24cd8e5 --- /dev/null +++ b/src/gevent/tests/test_server.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDMWcyeIiHQuEGQxgTIvu0aOW4iRFAyUEi8pLWNCxMEHglF8k6O +xFVq7XWZMDnDFVnbZjmQh5Tc21Ae6cXzxXln578fROXHEzXo3Is8HUlq3ug1yYOG +Hjxw++Opjf1uoHwPEBUKsz/flS7knuscgFM9FO05KSPn2wHnZeIDta4yTwIDAQAB +AoGBAKWfvq0IIvok7Ncm92ew/0D6/R1+2rT8xwdGQ/Nt31q98WwkqLEjxctlbKPd +J2PLIUomf0955BhhFH4JoSwjiHJQ6uishY7srjQQDX/Dxdi5wZAyxYCIVW/kAA9N +/u2s75hSD3s/rqAwOZ182DwAPIqJc4KQoYzvlKERSMDT1PJhAkEA5SUFsiSzBEMX +FyZ++ZMMs1vHrTu5oTK7WHznh9lk7dvsnp9BoUPqhiu8iJ7Q23zj0u5asz2czu11 +nnczXgU6XwJBAORM5Ib4I7nAsoUWn9wDiTwVQeE+D9P1ac9p7EHm7XXuf8o2irRZ +wYYfpXXsjk496YfyQFcQRMk0tU0gegCP7hECQFWRWqwoajUoPIInnPjjwbVki48U +I4CfqjgkBG3Fb5wnKRgezmpDK1vJD1FRRRsBay4EVhhi5KCdKfPv/V2ZxC8CQQCu +U5SxBytofJ8UhxkcTErvaR/8GYLGi//21GAGVop+YdaMlydE3cCrZODYcgCb+CSp +nS7KDG8p4KiMMz9VzJGxAkEAv85K6Sa3H8g9h7LwopBZ5tFNZUaFWo7lEP7DDMH0 +eckZTb1JVpyT/8zrDtsis4WlV9zVkVHxkIaad503BjqvEQ== +-----END RSA PRIVATE KEY----- diff --git a/src/gevent/tests/tests_that_dont_do_leakchecks.txt b/src/gevent/tests/tests_that_dont_do_leakchecks.txt new file mode 100644 index 0000000..55288a3 --- /dev/null +++ b/src/gevent/tests/tests_that_dont_do_leakchecks.txt @@ -0,0 +1,9 @@ +test___monkey_patching.py +test__monkey_ssl_warning.py +test___monitor.py +test__monkey_scope.py +test__ares_timeout.py +test__close_backend_fd.py +test__hub_join.py +test__hub_join_timeout.py +test__issue112.py diff --git a/src/gevent/tests/tests_that_dont_monkeypatch.txt b/src/gevent/tests/tests_that_dont_monkeypatch.txt new file mode 100644 index 0000000..452e429 --- /dev/null +++ b/src/gevent/tests/tests_that_dont_monkeypatch.txt @@ -0,0 +1,26 @@ +test___example_servers.py +test__backdoor.py +test__example_echoserver.py +test__example_udp_client.py +test__getaddrinfo_import.py +test__example_portforwarder.py +test__pywsgi.py +test__server.py +test__server_pywsgi.py +test__socket_close.py +test__socket_dns6.py +test__socket_errors.py +test__socket_send_memoryview.py +test__socket_timeout.py +test__examples.py +test__issue330.py +test___ident.py +test___config.py +test___monitor.py +test__events.py +test__monkey_scope.py +test__iwait.py +test__ares_timeout.py +test__close_backend_fd.py +test__hub_join.py +test__hub_join_timeout.py diff --git a/src/gevent/tests/tests_that_dont_use_resolver.txt b/src/gevent/tests/tests_that_dont_use_resolver.txt new file mode 100644 index 0000000..654918e --- /dev/null +++ b/src/gevent/tests/tests_that_dont_use_resolver.txt @@ -0,0 +1,136 @@ +test__all__.py +#uses socket test__api.py +test__api_timeout.py +test__ares_host_result.py +test__ares_timeout.py # explicitly uses ares resolver +# uses socket test__backdoor.py +test__close_backend_fd.py +test__core_async.py +test__core_callback.py +test__core_loop_run.py +test__core.py +test__core_stat.py +test__core_timer.py +test__core_watcher.py +test__destroy.py +# uses socket test__doctests.py +test__environ.py +test__event.py +# uses socket test__example_echoserver.py +# uses socket test__example_portforwarder.py +# uses socket test___example_servers.py +# uses bunch of things test__examples.py +# uses socket test__example_udp_client.py +# uses socket test__example_udp_server.py +test__exc_info.py +#test__execmodules.py +test__fileobject.py +# uses socket test__greenio.py +test__GreenletExit.py +test__greenlet.py +test__greenletset.py +# uses socket test__greenness.py +test__hub_join.py +test__hub_join_timeout.py +# uses socket test__hub.py +test__issue112.py +test__joinall.py +test__local.py +test__loop_callback.py +test__memleak.py +# uses lots of things test___monkey_patching.py +test__monkey.py +test__order.py +test__os.py +test__pool.py +# uses socket test__pywsgi.py +test__queue.py +test__monkey_queue.py +# uses socket test__refcount.py +test__select.py +test__semaphore.py +# uses socket test__server.py +# test__server_pywsgi.py +test__signal.py +# uses socket test__socket_close.py +# test__socket_dns6.py +# test__socket_dns.py +# test__socket_errors.py +# test__socket.py +# test__socket_ssl.py +# test__socket_timeout.py +test__subprocess_interrupted.py +test__subprocess.py +test__systemerror.py +test__threading_2.py +test__threading_patched_local.py +test__threading_vs_settrace.py +test__threadpool.py +test__timeout.py + +test__compat.py +test__core_fork.py +test__doctests.py +test__core_loop_run_sig_mod.py +test__execmodules.py +test__greenio.py +test__greenness.py +test__hub.py +test__import_blocking_in_greenlet.py +test__import_wait.py +test__issue230.py +test__issue330.py +test__issue467.py +test__issue6.py +test__issue600.py +test__issue607.py +test__issue461_471.py +test__monkey_builtins_future.py +test__monkey_hub_in_thread.py +test__monkey_logging.py +test__monkey_multiple_imports.py +test__monkey_scope.py +test__monkey_selectors.py +test__monkey_sigchld.py +test__monkey_sigchld_2.py +test__nondefaultloop.py +test__monkey_sigchld_3.py +test__real_greenlet.py +test__refcount.py +test__sleep0.py +test__subprocess_poll.py +test__threading.py +test__threading_before_monkey.py +test__threading_holding_lock_while_monkey.py +test__threading_monkey_in_thread.py +test__threading_native_before_monkey.py +test__threadpool_executor_patched.py + + +# monkey patched standard tests: +test_queue.py +test_select.py +test_signal.py +test_subprocess.py +test_threading_local.py +test_threading.py +test_thread.py +test_selectors.py +test_timeout.py + +# test_asyncore probably does use the resolver, but only +# implicitly for localhost, which is covered well enough +# elsewhere that we don't need to spend the 20s (*2) +test_asyncore.py + +test___config.py +test__destroy_default_loop.py +test__util.py +test___ident.py +test__issue639.py +test__issue_728.py +test__refcount_core.py +test__api.py +test__monitor.py +test__events.py +test__iwait.py diff --git a/src/gevent/tests/wrongcert.pem b/src/gevent/tests/wrongcert.pem new file mode 100644 index 0000000..5f92f9b --- /dev/null +++ b/src/gevent/tests/wrongcert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH +FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T +f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB +AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq +1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW +7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg +SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe +19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg +ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666 +lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs +ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv +frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk +2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6 ++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK +24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G +A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9 +OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ +fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E +usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+ +43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw +eSHj5jpC8iZKjCHBn+mAi4cQ514= +-----END CERTIFICATE----- diff --git a/src/gevent/thread.py b/src/gevent/thread.py new file mode 100644 index 0000000..cceaf48 --- /dev/null +++ b/src/gevent/thread.py @@ -0,0 +1,114 @@ +""" +Implementation of the standard :mod:`thread` module that spawns greenlets. + +.. note:: + + This module is a helper for :mod:`gevent.monkey` and is not + intended to be used directly. For spawning greenlets in your + applications, prefer higher level constructs like + :class:`gevent.Greenlet` class or :func:`gevent.spawn`. +""" +from __future__ import absolute_import +import sys + +__implements__ = ['allocate_lock', + 'get_ident', + 'exit', + 'LockType', + 'stack_size', + 'start_new_thread', + '_local'] + +__imports__ = ['error'] +if sys.version_info[0] <= 2: + import thread as __thread__ # pylint:disable=import-error +else: + import _thread as __thread__ # pylint:disable=import-error + __target__ = '_thread' + __imports__ += ['RLock', + 'TIMEOUT_MAX', + 'allocate', + 'exit_thread', + 'interrupt_main', + 'start_new'] +error = __thread__.error +from gevent._compat import PY3 +from gevent._compat import PYPY +from gevent._util import copy_globals +from gevent.hub import getcurrent, GreenletExit +from gevent.greenlet import Greenlet +from gevent.lock import BoundedSemaphore +from gevent.local import local as _local + + +def get_ident(gr=None): + if gr is None: + gr = getcurrent() + return id(gr) + + +def start_new_thread(function, args=(), kwargs=None): + if kwargs is not None: + greenlet = Greenlet.spawn(function, *args, **kwargs) + else: + greenlet = Greenlet.spawn(function, *args) + return get_ident(greenlet) + + +class LockType(BoundedSemaphore): + # Change the ValueError into the appropriate thread error + # and any other API changes we need to make to match behaviour + _OVER_RELEASE_ERROR = __thread__.error + + if PYPY and PY3: + _OVER_RELEASE_ERROR = RuntimeError + + if PY3: + _TIMEOUT_MAX = __thread__.TIMEOUT_MAX # python 2: pylint:disable=no-member + + def acquire(self, blocking=True, timeout=-1): + # Transform the default -1 argument into the None that our + # semaphore implementation expects, and raise the same error + # the stdlib implementation does. + if timeout == -1: + timeout = None + if not blocking and timeout is not None: + raise ValueError("can't specify a timeout for a non-blocking call") + if timeout is not None: + if timeout < 0: + # in C: if(timeout < 0 && timeout != -1) + raise ValueError("timeout value must be strictly positive") + if timeout > self._TIMEOUT_MAX: + raise OverflowError('timeout value is too large') + + return BoundedSemaphore.acquire(self, blocking, timeout) + +allocate_lock = LockType + + +def exit(): + raise GreenletExit + + +if hasattr(__thread__, 'stack_size'): + _original_stack_size = __thread__.stack_size + + def stack_size(size=None): + if size is None: + return _original_stack_size() + if size > _original_stack_size(): + return _original_stack_size(size) + # not going to decrease stack_size, because otherwise other + # greenlets in this thread will suffer +else: + __implements__.remove('stack_size') + +__imports__ = copy_globals(__thread__, globals(), + only_names=__imports__, + ignore_missing_names=True) + +__all__ = __implements__ + __imports__ +__all__.remove('_local') + +# XXX interrupt_main +# XXX _count() diff --git a/src/gevent/threading.py b/src/gevent/threading.py new file mode 100644 index 0000000..570ccd6 --- /dev/null +++ b/src/gevent/threading.py @@ -0,0 +1,235 @@ +""" +Implementation of the standard :mod:`threading` using greenlets. + +.. note:: + + This module is a helper for :mod:`gevent.monkey` and is not + intended to be used directly. For spawning greenlets in your + applications, prefer higher level constructs like + :class:`gevent.Greenlet` class or :func:`gevent.spawn`. Attributes + in this module like ``__threading__`` are implementation artifacts subject + to change at any time. + +.. versionchanged:: 1.2.3 + + Defer adjusting the stdlib's list of active threads until we are + monkey patched. Previously this was done at import time. We are + documented to only be used as a helper for monkey patching, so this should + functionally be the same, but some applications ignore the documentation and + directly import this module anyway. + + A positive consequence is that ``import gevent.threading, + threading; threading.current_thread()`` will no longer return a DummyThread + before monkey-patching. +""" +from __future__ import absolute_import + + +__implements__ = [ + 'local', + '_start_new_thread', + '_allocate_lock', + 'Lock', + '_get_ident', + '_sleep', + '_DummyThread', +] + + +import threading as __threading__ +_DummyThread_ = __threading__._DummyThread +from gevent.local import local +from gevent.thread import start_new_thread as _start_new_thread, allocate_lock as _allocate_lock, get_ident as _get_ident +from gevent.hub import sleep as _sleep, getcurrent + +# Exports, prevent unused import warnings +local = local +start_new_thread = _start_new_thread +allocate_lock = _allocate_lock +_get_ident = _get_ident +_sleep = _sleep +getcurrent = getcurrent + +Lock = _allocate_lock + + +def _cleanup(g): + __threading__._active.pop(_get_ident(g), None) + +def _make_cleanup_id(gid): + def _(_r): + __threading__._active.pop(gid, None) + return _ + +_weakref = None + +class _DummyThread(_DummyThread_): + # We avoid calling the superclass constructor. This makes us about + # twice as fast (1.16 vs 0.68usec on PyPy, 29.3 vs 17.7usec on + # CPython 2.7), and has the important effect of avoiding + # allocation and then immediate deletion of _Thread__block, a + # lock. This is especially important on PyPy where locks go + # through the cpyext API and Cython, which is known to be slow and + # potentially buggy (e.g., + # https://bitbucket.org/pypy/pypy/issues/2149/memory-leak-for-python-subclass-of-cpyext#comment-22347393) + + # These objects are constructed quite frequently in some cases, so + # the optimization matters: for example, in gunicorn, which uses + # pywsgi.WSGIServer, every request is handled in a new greenlet, + # and every request uses a logging.Logger to write the access log, + # and every call to a log method captures the current thread (by + # default). + # + # (Obviously we have to duplicate the effects of the constructor, + # at least for external state purposes, which is potentially + # slightly fragile.) + + # For the same reason, instances of this class will cleanup their own entry + # in ``threading._active`` + + # This class also solves a problem forking process with subprocess: after forking, + # Thread.__stop is called, which throws an exception when __block doesn't + # exist. + + # Capture the static things as class vars to save on memory/ + # construction time. + # In Py2, they're all private; in Py3, they become protected + _Thread__stopped = _is_stopped = _stopped = False + _Thread__initialized = _initialized = True + _Thread__daemonic = _daemonic = True + _Thread__args = _args = () + _Thread__kwargs = _kwargs = None + _Thread__target = _target = None + _Thread_ident = _ident = None + _Thread__started = _started = __threading__.Event() + _Thread__started.set() + _tstate_lock = None + + def __init__(self): # pylint:disable=super-init-not-called + #_DummyThread_.__init__(self) + + # It'd be nice to use a pattern like "greenlet-%d", but maybe somebody out + # there is checking thread names... + self._name = self._Thread__name = __threading__._newname("DummyThread-%d") + # All dummy threads in the same native thread share the same ident + # (that of the native thread) + self._set_ident() + + g = getcurrent() + gid = _get_ident(g) + __threading__._active[gid] = self + rawlink = getattr(g, 'rawlink', None) + if rawlink is not None: + # raw greenlet.greenlet greenlets don't + # have rawlink... + rawlink(_cleanup) + else: + # ... so for them we use weakrefs. + # See https://github.com/gevent/gevent/issues/918 + global _weakref + if _weakref is None: + _weakref = __import__('weakref') + ref = _weakref.ref(g, _make_cleanup_id(gid)) + self.__raw_ref = ref + + def _Thread__stop(self): + pass + + _stop = _Thread__stop # py3 + + def _wait_for_tstate_lock(self, *args, **kwargs): + # pylint:disable=arguments-differ + pass + +if hasattr(__threading__, 'main_thread'): # py 3.4+ + def main_native_thread(): + return __threading__.main_thread() # pylint:disable=no-member +else: + def main_native_thread(): + main_threads = [v for v in __threading__._active.values() + if isinstance(v, __threading__._MainThread)] + assert len(main_threads) == 1, "Too many main threads" + + return main_threads[0] + +import sys +if sys.version_info[:2] >= (3, 4): + # XXX: Issue 18808 breaks us on Python 3.4. + # Thread objects now expect a callback from the interpreter itself + # (threadmodule.c:release_sentinel). Because this never happens + # when a greenlet exits, join() and friends will block forever. + # The solution below involves capturing the greenlet when it is + # started and deferring the known broken methods to it. + + class Thread(__threading__.Thread): + _greenlet = None + + def is_alive(self): + return bool(self._greenlet) + + isAlive = is_alive + + def _set_tstate_lock(self): + self._greenlet = getcurrent() + + def run(self): + try: + super(Thread, self).run() + finally: + # avoid ref cycles, but keep in __dict__ so we can + # distinguish the started/never-started case + self._greenlet = None + self._stop() # mark as finished + + def join(self, timeout=None): + if '_greenlet' not in self.__dict__: + raise RuntimeError("Cannot join an inactive thread") + if self._greenlet is None: + return + self._greenlet.join(timeout=timeout) + + def _wait_for_tstate_lock(self, *args, **kwargs): + # pylint:disable=arguments-differ + raise NotImplementedError() + + __implements__.append('Thread') + + class Timer(Thread, __threading__.Timer): # pylint:disable=abstract-method,inherit-non-class + pass + + __implements__.append('Timer') + + # The main thread is patched up with more care + # in _gevent_will_monkey_patch + +if sys.version_info[:2] >= (3, 3): + __implements__.remove('_get_ident') + __implements__.append('get_ident') + get_ident = _get_ident + __implements__.remove('_sleep') + + # Python 3 changed the implementation of threading.RLock + # Previously it was a factory function around threading._RLock + # which in turn used _allocate_lock. Now, it wants to use + # threading._CRLock, which is imported from _thread.RLock and as such + # is implemented in C. So it bypasses our _allocate_lock function. + # Fortunately they left the Python fallback in place + assert hasattr(__threading__, '_CRLock'), "Unsupported Python version" + _CRLock = None + __implements__.append('_CRLock') + +def _gevent_will_monkey_patch(native_module, items, warn): # pylint:disable=unused-argument + # Make sure the MainThread can be found by our current greenlet ID, + # otherwise we get a new DummyThread, which cannot be joined. + # Fixes tests in test_threading_2 under PyPy. + main_thread = main_native_thread() + if __threading__.current_thread() != main_thread: + warn("Monkey-patching outside the main native thread. Some APIs " + "will not be available. Expect a KeyError to be printed at shutdown.") + return + + if _get_ident() not in __threading__._active: + main_id = main_thread.ident + del __threading__._active[main_id] + main_thread._ident = main_thread._Thread__ident = _get_ident() + __threading__._active[_get_ident()] = main_thread diff --git a/src/gevent/threadpool.py b/src/gevent/threadpool.py new file mode 100644 index 0000000..7108e4d --- /dev/null +++ b/src/gevent/threadpool.py @@ -0,0 +1,580 @@ +# Copyright (c) 2012 Denis Bilenko. See LICENSE for details. +from __future__ import absolute_import +import sys +import os + +from weakref import ref as wref + +from greenlet import greenlet as RawGreenlet + +from gevent._compat import integer_types +from gevent.hub import _get_hub_noargs as get_hub +from gevent.hub import getcurrent +from gevent.hub import sleep +from gevent.hub import _get_hub +from gevent.event import AsyncResult +from gevent.greenlet import Greenlet +from gevent.pool import GroupMappingMixin +from gevent.lock import Semaphore + +from gevent._threading import Lock +from gevent._threading import Queue +from gevent._threading import start_new_thread +from gevent._threading import get_thread_ident + + +__all__ = [ + 'ThreadPool', + 'ThreadResult', +] + + +class _WorkerGreenlet(RawGreenlet): + # Exists to produce a more useful repr for worker pool + # threads/greenlets. + + def __init__(self, threadpool): + RawGreenlet.__init__(self, threadpool._worker) + self.thread_ident = get_thread_ident() + self._threadpool_wref = wref(threadpool) + + # Inform the gevent.util.GreenletTree that this should be + # considered the root (for printing purposes) and to + # ignore the parent attribute. (We can't set parent to None.) + self.greenlet_tree_is_root = True + self.parent.greenlet_tree_is_ignored = True + + def __repr__(self): + return "" % ( + id(self), + self.thread_ident, + self._threadpool_wref()) + +class ThreadPool(GroupMappingMixin): + """ + .. note:: The method :meth:`apply_async` will always return a new + greenlet, bypassing the threadpool entirely. + .. caution:: Instances of this class are only true if they have + unfinished tasks. + """ + + def __init__(self, maxsize, hub=None): + if hub is None: + hub = get_hub() + self.hub = hub + self._maxsize = 0 + self.manager = None + self.pid = os.getpid() + self.fork_watcher = hub.loop.fork(ref=False) + try: + self._init(maxsize) + except: + self.fork_watcher.close() + raise + + def _set_maxsize(self, maxsize): + if not isinstance(maxsize, integer_types): + raise TypeError('maxsize must be integer: %r' % (maxsize, )) + if maxsize < 0: + raise ValueError('maxsize must not be negative: %r' % (maxsize, )) + difference = maxsize - self._maxsize + self._semaphore.counter += difference + self._maxsize = maxsize + self.adjust() + # make sure all currently blocking spawn() start unlocking if maxsize increased + self._semaphore._start_notify() + + def _get_maxsize(self): + return self._maxsize + + maxsize = property(_get_maxsize, _set_maxsize) + + def __repr__(self): + return '<%s at 0x%x %s/%s/%s hub=<%s at 0x%x thread_ident=0x%s>>' % ( + self.__class__.__name__, + id(self), + len(self), self.size, self.maxsize, + self.hub.__class__.__name__, id(self.hub), self.hub.thread_ident) + + def __len__(self): + # XXX just do unfinished_tasks property + # Note that this becomes the boolean value of this class, + # that's probably not what we want! + return self.task_queue.unfinished_tasks + + def _get_size(self): + return self._size + + def _set_size(self, size): + if size < 0: + raise ValueError('Size of the pool cannot be negative: %r' % (size, )) + if size > self._maxsize: + raise ValueError('Size of the pool cannot be bigger than maxsize: %r > %r' % (size, self._maxsize)) + if self.manager: + self.manager.kill() + while self._size < size: + self._add_thread() + delay = self.hub.loop.approx_timer_resolution + while self._size > size: + while self._size - size > self.task_queue.unfinished_tasks: + self.task_queue.put(None) + if getcurrent() is self.hub: + break + sleep(delay) + delay = min(delay * 2, .05) + if self._size: + self.fork_watcher.start(self._on_fork) + else: + self.fork_watcher.stop() + + size = property(_get_size, _set_size) + + def _init(self, maxsize): + self._size = 0 + self._semaphore = Semaphore(1) + self._lock = Lock() + self.task_queue = Queue() + self._set_maxsize(maxsize) + + def _on_fork(self): + # fork() only leaves one thread; also screws up locks; + # let's re-create locks and threads. + # NOTE: See comment in gevent.hub.reinit. + pid = os.getpid() + if pid != self.pid: + self.pid = pid + # Do not mix fork() and threads; since fork() only copies one thread + # all objects referenced by other threads has refcount that will never + # go down to 0. + self._init(self._maxsize) + + def join(self): + """Waits until all outstanding tasks have been completed.""" + delay = max(0.0005, self.hub.loop.approx_timer_resolution) + while self.task_queue.unfinished_tasks > 0: + sleep(delay) + delay = min(delay * 2, .05) + + def kill(self): + self.size = 0 + self.fork_watcher.close() + + def _adjust_step(self): + # if there is a possibility & necessity for adding a thread, do it + while self._size < self._maxsize and self.task_queue.unfinished_tasks > self._size: + self._add_thread() + # while the number of threads is more than maxsize, kill one + # we do not check what's already in task_queue - it could be all Nones + while self._size - self._maxsize > self.task_queue.unfinished_tasks: + self.task_queue.put(None) + if self._size: + self.fork_watcher.start(self._on_fork) + else: + self.fork_watcher.stop() + + def _adjust_wait(self): + delay = 0.0001 + while True: + self._adjust_step() + if self._size <= self._maxsize: + return + sleep(delay) + delay = min(delay * 2, .05) + + def adjust(self): + self._adjust_step() + if not self.manager and self._size > self._maxsize: + # might need to feed more Nones into the pool + self.manager = Greenlet.spawn(self._adjust_wait) + + def _add_thread(self): + with self._lock: + self._size += 1 + try: + start_new_thread(self.__trampoline, ()) + except: + with self._lock: + self._size -= 1 + raise + + def spawn(self, func, *args, **kwargs): + """ + Add a new task to the threadpool that will run ``func(*args, **kwargs)``. + + Waits until a slot is available. Creates a new thread if necessary. + + :return: A :class:`gevent.event.AsyncResult`. + """ + while 1: + semaphore = self._semaphore + semaphore.acquire() + if semaphore is self._semaphore: + break + + thread_result = None + try: + task_queue = self.task_queue + result = AsyncResult() + # XXX We're calling the semaphore release function in the hub, otherwise + # we get LoopExit (why?). Previously it was done with a rawlink on the + # AsyncResult and the comment that it is "competing for order with get(); this is not + # good, just make ThreadResult release the semaphore before doing anything else" + thread_result = ThreadResult(result, self.hub, semaphore.release) + task_queue.put((func, args, kwargs, thread_result)) + self.adjust() + except: + if thread_result is not None: + thread_result.destroy() + semaphore.release() + raise + return result + + def _decrease_size(self): + if sys is None: + return + _lock = getattr(self, '_lock', None) + if _lock is not None: + with _lock: + self._size -= 1 + + # XXX: This used to be false by default. It really seems like + # it should be true to avoid leaking resources. + _destroy_worker_hub = True + + + def __ignore_current_greenlet_blocking(self, hub): + if hub is not None and hub.periodic_monitoring_thread is not None: + hub.periodic_monitoring_thread.ignore_current_greenlet_blocking() + + def __trampoline(self): + # The target that we create new threads with. It exists + # solely to create the _WorkerGreenlet and switch to it. + # (the __class__ of a raw greenlet cannot be changed.) + g = _WorkerGreenlet(self) + g.switch() + + def _worker(self): + # pylint:disable=too-many-branches + need_decrease = True + try: + while 1: # tiny bit faster than True on Py2 + h = _get_hub() + if h is not None: + h.name = 'ThreadPool Worker Hub' + task_queue = self.task_queue + # While we block, don't let the monitoring thread, if any, + # report us as blocked. Indeed, so long as we never + # try to switch greenlets, don't report us as blocked--- + # the threadpool is *meant* to run blocking tasks + self.__ignore_current_greenlet_blocking(h) + task = task_queue.get() + try: + if task is None: + need_decrease = False + self._decrease_size() + # we want first to decrease size, then decrease unfinished_tasks + # otherwise, _adjust might think there's one more idle thread that + # needs to be killed + return + func, args, kwargs, thread_result = task + try: + value = func(*args, **kwargs) + except: # pylint:disable=bare-except + exc_info = getattr(sys, 'exc_info', None) + if exc_info is None: + return + thread_result.handle_error((self, func), exc_info()) + else: + if sys is None: + return + thread_result.set(value) + del value + finally: + del func, args, kwargs, thread_result, task + finally: + if sys is None: + return # pylint:disable=lost-exception + task_queue.task_done() + finally: + if need_decrease: + self._decrease_size() + if sys is not None and self._destroy_worker_hub: + hub = _get_hub() + if hub is not None: + hub.destroy(True) + del hub + + def apply_e(self, expected_errors, function, args=None, kwargs=None): + """ + .. deprecated:: 1.1a2 + Identical to :meth:`apply`; the ``expected_errors`` argument is ignored. + """ + # pylint:disable=unused-argument + # Deprecated but never documented. In the past, before + # self.apply() allowed all errors to be raised to the caller, + # expected_errors allowed a caller to specify a set of errors + # they wanted to be raised, through the wrap_errors function. + # In practice, it always took the value Exception or + # BaseException. + return self.apply(function, args, kwargs) + + def _apply_immediately(self): + # If we're being called from a different thread than the one that + # created us, e.g., because a worker task is trying to use apply() + # recursively, we have no choice but to run the task immediately; + # if we try to AsyncResult.get() in the worker thread, it's likely to have + # nothing to switch to and lead to a LoopExit. + return get_hub() is not self.hub + + def _apply_async_cb_spawn(self, callback, result): + callback(result) + + def _apply_async_use_greenlet(self): + # Always go to Greenlet because our self.spawn uses threads + return True + +class _FakeAsync(object): + + def send(self): + pass + close = stop = send + + def __call_(self, result): + "fake out for 'receiver'" + + def __bool__(self): + return False + + __nonzero__ = __bool__ + +_FakeAsync = _FakeAsync() + +class ThreadResult(object): + + # Using slots here helps to debug reference cycles/leaks + __slots__ = ('exc_info', 'async_watcher', '_call_when_ready', 'value', + 'context', 'hub', 'receiver') + + def __init__(self, receiver, hub, call_when_ready): + self.receiver = receiver + self.hub = hub + self.context = None + self.value = None + self.exc_info = () + self.async_watcher = hub.loop.async_() + self._call_when_ready = call_when_ready + self.async_watcher.start(self._on_async) + + @property + def exception(self): + return self.exc_info[1] if self.exc_info else None + + def _on_async(self): + self.async_watcher.stop() + self.async_watcher.close() + + # Typically this is pool.semaphore.release and we have to + # call this in the Hub; if we don't we get the dreaded + # LoopExit (XXX: Why?) + self._call_when_ready() + + try: + if self.exc_info: + self.hub.handle_error(self.context, *self.exc_info) + self.context = None + self.async_watcher = _FakeAsync + self.hub = None + self._call_when_ready = _FakeAsync + + self.receiver(self) + finally: + self.receiver = _FakeAsync + self.value = None + if self.exc_info: + self.exc_info = (self.exc_info[0], self.exc_info[1], None) + + def destroy(self): + self.async_watcher.stop() + self.async_watcher.close() + self.async_watcher = _FakeAsync + + self.context = None + self.hub = None + self._call_when_ready = _FakeAsync + self.receiver = _FakeAsync + + def set(self, value): + self.value = value + self.async_watcher.send() + + def handle_error(self, context, exc_info): + self.context = context + self.exc_info = exc_info + self.async_watcher.send() + + # link protocol: + def successful(self): + return self.exception is None + + +def wrap_errors(errors, function, args, kwargs): + """ + .. deprecated:: 1.1a2 + Previously used by ThreadPool.apply_e. + """ + try: + return True, function(*args, **kwargs) + except errors as ex: + return False, ex + +try: + import concurrent.futures +except ImportError: + pass +else: + __all__.append("ThreadPoolExecutor") + + from gevent.timeout import Timeout as GTimeout + from gevent._util import Lazy + from concurrent.futures import _base as cfb + + def _wrap_error(future, fn): + def cbwrap(_): + del _ + # we're called with the async result, but + # be sure to pass in ourself. Also automatically + # unlink ourself so that we don't get called multiple + # times. + try: + fn(future) + except Exception: # pylint: disable=broad-except + future.hub.print_exception((fn, future), *sys.exc_info()) + cbwrap.auto_unlink = True + return cbwrap + + def _wrap(future, fn): + def f(_): + fn(future) + f.auto_unlink = True + return f + + class _FutureProxy(object): + def __init__(self, asyncresult): + self.asyncresult = asyncresult + + # Internal implementation details of a c.f.Future + + @Lazy + def _condition(self): + from gevent import monkey + if monkey.is_module_patched('threading') or self.done(): + import threading + return threading.Condition() + # We can only properly work with conditions + # when we've been monkey-patched. This is necessary + # for the wait/as_completed module functions. + raise AttributeError("_condition") + + @Lazy + def _waiters(self): + self.asyncresult.rawlink(self.__when_done) + return [] + + def __when_done(self, _): + # We should only be called when _waiters has + # already been accessed. + waiters = getattr(self, '_waiters') + for w in waiters: # pylint:disable=not-an-iterable + if self.successful(): + w.add_result(self) + else: + w.add_exception(self) + + __when_done.auto_unlink = True + + @property + def _state(self): + if self.done(): + return cfb.FINISHED + return cfb.RUNNING + + def set_running_or_notify_cancel(self): + # Does nothing, not even any consistency checks. It's + # meant to be internal to the executor and we don't use it. + return + + def result(self, timeout=None): + try: + return self.asyncresult.result(timeout=timeout) + except GTimeout: + # XXX: Theoretically this could be a completely + # unrelated timeout instance. Do we care about that? + raise concurrent.futures.TimeoutError() + + def exception(self, timeout=None): + try: + self.asyncresult.get(timeout=timeout) + return self.asyncresult.exception + except GTimeout: + raise concurrent.futures.TimeoutError() + + def add_done_callback(self, fn): + if self.done(): + fn(self) + else: + self.asyncresult.rawlink(_wrap_error(self, fn)) + + def rawlink(self, fn): + self.asyncresult.rawlink(_wrap(self, fn)) + + def __str__(self): + return str(self.asyncresult) + + def __getattr__(self, name): + return getattr(self.asyncresult, name) + + class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): + """ + A version of :class:`concurrent.futures.ThreadPoolExecutor` that + always uses native threads, even when threading is monkey-patched. + + The ``Future`` objects returned from this object can be used + with gevent waiting primitives like :func:`gevent.wait`. + + .. caution:: If threading is *not* monkey-patched, then the ``Future`` + objects returned by this object are not guaranteed to work with + :func:`~concurrent.futures.as_completed` and :func:`~concurrent.futures.wait`. + The individual blocking methods like :meth:`~concurrent.futures.Future.result` + and :meth:`~concurrent.futures.Future.exception` will always work. + + .. versionadded:: 1.2a1 + This is a provisional API. + """ + + def __init__(self, max_workers): + super(ThreadPoolExecutor, self).__init__(max_workers) + self._threadpool = ThreadPool(max_workers) + self._threadpool._destroy_worker_hub = True + + def submit(self, fn, *args, **kwargs): + with self._shutdown_lock: # pylint:disable=not-context-manager + if self._shutdown: + raise RuntimeError('cannot schedule new futures after shutdown') + + future = self._threadpool.spawn(fn, *args, **kwargs) + return _FutureProxy(future) + + def shutdown(self, wait=True): + super(ThreadPoolExecutor, self).shutdown(wait) + # XXX: We don't implement wait properly + kill = getattr(self._threadpool, 'kill', None) + if kill: # pylint:disable=using-constant-test + self._threadpool.kill() + self._threadpool = None + + kill = shutdown # greentest compat + + def _adjust_thread_count(self): + # Does nothing. We don't want to spawn any "threads", + # let the threadpool handle that. + pass diff --git a/src/gevent/time.py b/src/gevent/time.py new file mode 100644 index 0000000..34abf85 --- /dev/null +++ b/src/gevent/time.py @@ -0,0 +1,27 @@ +# Copyright (c) 2018 gevent. See LICENSE for details. +""" +The standard library :mod:`time` module, but :func:`sleep` is +gevent-aware. + +.. versionadded:: 1.3a2 +""" + +from __future__ import absolute_import + +__implements__ = [ + 'sleep', +] + +__all__ = __implements__ + +import time as __time__ + +from gevent._util import copy_globals + +__imports__ = copy_globals(__time__, globals(), + names_to_ignore=__implements__) + + + +from gevent.hub import sleep +sleep = sleep # pylint diff --git a/src/gevent/timeout.py b/src/gevent/timeout.py new file mode 100644 index 0000000..383d090 --- /dev/null +++ b/src/gevent/timeout.py @@ -0,0 +1,373 @@ +# Copyright (c) 2009-2010 Denis Bilenko. See LICENSE for details. +""" +Timeouts. + +Many functions in :mod:`gevent` have a *timeout* argument that allows +limiting the time the function will block. When that is not available, +the :class:`Timeout` class and :func:`with_timeout` function in this +module add timeouts to arbitrary code. + +.. warning:: + + Timeouts can only work when the greenlet switches to the hub. + If a blocking function is called or an intense calculation is ongoing during + which no switches occur, :class:`Timeout` is powerless. +""" +from __future__ import absolute_import, print_function, division + +from gevent._compat import string_types +from gevent._util import _NONE + +from greenlet import getcurrent +from gevent._hub_local import get_hub_noargs as get_hub + +__all__ = [ + 'Timeout', + 'with_timeout', +] + + +class _FakeTimer(object): + # An object that mimics the API of get_hub().loop.timer, but + # without allocating any native resources. This is useful for timeouts + # that will never expire. + # Also partially mimics the API of Timeout itself for use in _start_new_or_dummy + + # This object is used as a singleton, so it should be + # immutable. + __slots__ = () + + @property + def pending(self): + return False + + active = pending + + @property + def seconds(self): + "Always returns None" + + timer = exception = seconds + + def start(self, *args, **kwargs): + # pylint:disable=unused-argument + raise AssertionError("non-expiring timer cannot be started") + + def stop(self): + return + + cancel = stop + + stop = close = cancel + + def __enter__(self): + return self + + def __exit__(self, _t, _v, _tb): + return + +_FakeTimer = _FakeTimer() + + +class Timeout(BaseException): + """ + Timeout(seconds=None, exception=None, ref=True, priority=-1) + + Raise *exception* in the current greenlet after *seconds* + have elapsed:: + + timeout = Timeout(seconds, exception) + timeout.start() + try: + ... # exception will be raised here, after *seconds* passed since start() call + finally: + timeout.close() + + .. note:: + + If the code that the timeout was protecting finishes + executing before the timeout elapses, be sure to ``close`` the + timeout so it is not unexpectedly raised in the future. Even if it + is raised, it is a best practice to close it. This ``try/finally`` + construct or a ``with`` statement is a recommended pattern. (If + the timeout object will be started again, use ``cancel`` instead + of ``close``; this is rare.) + + When *exception* is omitted or ``None``, the ``Timeout`` instance + itself is raised:: + + >>> import gevent + >>> gevent.Timeout(0.1).start() + >>> gevent.sleep(0.2) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + Timeout: 0.1 seconds + + If the *seconds* argument is not given or is ``None`` (e.g., + ``Timeout()``), then the timeout will never expire and never raise + *exception*. This is convenient for creating functions which take + an optional timeout parameter of their own. (Note that this is **not** + the same thing as a *seconds* value of ``0``.) + + :: + + def function(args, timeout=None): + "A function with an optional timeout." + timer = Timeout(timeout) + with timer: + ... + + .. caution:: + + A *seconds* value less than ``0.0`` (e.g., ``-1``) is poorly defined. In the future, + support for negative values is likely to do the same thing as a value + of ``None`` or ``0`` + + A *seconds* value of ``0`` requests that the event loop spin and poll for I/O; + it will immediately expire as soon as control returns to the event loop. + + .. rubric:: Use As A Context Manager + + To simplify starting and canceling timeouts, the ``with`` + statement can be used:: + + with gevent.Timeout(seconds, exception) as timeout: + pass # ... code block ... + + This is equivalent to the try/finally block above with one + additional feature: if *exception* is the literal ``False``, the + timeout is still raised, but the context manager suppresses it, so + the code outside the with-block won't see it. + + This is handy for adding a timeout to the functions that don't + support a *timeout* parameter themselves:: + + data = None + with gevent.Timeout(5, False): + data = mysock.makefile().readline() + if data is None: + ... # 5 seconds passed without reading a line + else: + ... # a line was read within 5 seconds + + .. caution:: + + If ``readline()`` above catches and doesn't re-raise + :exc:`BaseException` (for example, with a bare ``except:``), then + your timeout will fail to function and control won't be returned + to you when you expect. + + .. rubric:: Catching Timeouts + + When catching timeouts, keep in mind that the one you catch may + not be the one you have set (a calling function may have set its + own timeout); if you going to silence a timeout, always check that + it's the instance you need:: + + timeout = Timeout(1) + timeout.start() + try: + ... + except Timeout as t: + if t is not timeout: + raise # not my timeout + finally: + timeout.close() + + + .. versionchanged:: 1.1b2 + + If *seconds* is not given or is ``None``, no longer allocate a + native timer object that will never be started. + + .. versionchanged:: 1.1 + + Add warning about negative *seconds* values. + + .. versionchanged:: 1.3a1 + + Timeout objects now have a :meth:`close` + method that must be called when the timeout will no longer be + used to properly clean up native resources. + The ``with`` statement does this automatically. + + """ + + # We inherit a __dict__ from BaseException, so __slots__ actually + # makes us larger. + + def __init__(self, seconds=None, exception=None, ref=True, priority=-1, + _one_shot=False): + BaseException.__init__(self) + self.seconds = seconds + self.exception = exception + self._one_shot = _one_shot + if seconds is None: + # Avoid going through the timer codepath if no timeout is + # desired; this avoids some CFFI interactions on PyPy that can lead to a + # RuntimeError if this implementation is used during an `import` statement. See + # https://bitbucket.org/pypy/pypy/issues/2089/crash-in-pypy-260-linux64-with-gevent-11b1 + # and https://github.com/gevent/gevent/issues/618. + # Plus, in general, it should be more efficient + + self.timer = _FakeTimer + else: + # XXX: A timer <= 0 could cause libuv to block the loop; we catch + # that case in libuv/loop.py + self.timer = get_hub().loop.timer(seconds or 0.0, ref=ref, priority=priority) + + def start(self): + """Schedule the timeout.""" + if self.pending: + raise AssertionError('%r is already started; to restart it, cancel it first' % self) + + if self.seconds is None: + # "fake" timeout (never expires) + return + + if self.exception is None or self.exception is False or isinstance(self.exception, string_types): + # timeout that raises self + throws = self + else: + # regular timeout with user-provided exception + throws = self.exception + + # Make sure the timer updates the current time so that we don't + # expire prematurely. + self.timer.start(getcurrent().throw, throws, update=True) + + @classmethod + def start_new(cls, timeout=None, exception=None, ref=True, _one_shot=False): + """Create a started :class:`Timeout`. + + This is a shortcut, the exact action depends on *timeout*'s type: + + * If *timeout* is a :class:`Timeout`, then call its :meth:`start` method + if it's not already begun. + * Otherwise, create a new :class:`Timeout` instance, passing (*timeout*, *exception*) as + arguments, then call its :meth:`start` method. + + Returns the :class:`Timeout` instance. + """ + if isinstance(timeout, Timeout): + if not timeout.pending: + timeout.start() + return timeout + timeout = cls(timeout, exception, ref=ref, _one_shot=_one_shot) + timeout.start() + return timeout + + @staticmethod + def _start_new_or_dummy(timeout, exception=None, ref=True): + # Internal use only in 1.1 + # Return an object with a 'cancel' method; if timeout is None, + # this will be a shared instance object that does nothing. Otherwise, + # return an actual Timeout. Because negative values are hard to reason about, + # and are often used as sentinels in Python APIs, in the future it's likely + # that a negative timeout will also return the shared instance. + # This saves the previously common idiom of 'timer = Timeout.start_new(t) if t is not None else None' + # followed by 'if timer is not None: timer.cancel()'. + # That idiom was used to avoid any object allocations. + # A staticmethod is slightly faster under CPython, compared to a classmethod; + # under PyPy in synthetic benchmarks it makes no difference. + if timeout is None: + return _FakeTimer + return Timeout.start_new(timeout, exception, ref, _one_shot=True) + + @property + def pending(self): + """True if the timeout is scheduled to be raised.""" + return self.timer.pending or self.timer.active + + def cancel(self): + """ + If the timeout is pending, cancel it. Otherwise, do nothing. + + The timeout object can be :meth:`started ` again. If + you will not start the timeout again, you should use + :meth:`close` instead. + """ + self.timer.stop() + if self._one_shot: + self.close() + + def close(self): + """ + Close the timeout and free resources. The timer cannot be started again + after this method has been used. + """ + self.timer.stop() + self.timer.close() + self.timer = _FakeTimer + + def __repr__(self): + classname = type(self).__name__ + if self.pending: + pending = ' pending' + else: + pending = '' + if self.exception is None: + exception = '' + else: + exception = ' exception=%r' % self.exception + return '<%s at %s seconds=%s%s%s>' % (classname, hex(id(self)), self.seconds, exception, pending) + + def __str__(self): + """ + >>> raise Timeout #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + Timeout + """ + if self.seconds is None: + return '' + + suffix = '' if self.seconds == 1 else 's' + + if self.exception is None: + return '%s second%s' % (self.seconds, suffix) + if self.exception is False: + return '%s second%s (silent)' % (self.seconds, suffix) + return '%s second%s: %s' % (self.seconds, suffix, self.exception) + + def __enter__(self): + """ + Start and return the timer. If the timer is already started, just return it. + """ + if not self.pending: + self.start() + return self + + def __exit__(self, typ, value, tb): + """ + Stop the timer. + + .. versionchanged:: 1.3a1 + The underlying native timer is also stopped. This object cannot be + used again. + """ + self.close() + if value is self and self.exception is False: + return True # Suppress the exception + + +def with_timeout(seconds, function, *args, **kwds): + """Wrap a call to *function* with a timeout; if the called + function fails to return before the timeout, cancel it and return a + flag value, provided by *timeout_value* keyword argument. + + If timeout expires but *timeout_value* is not provided, raise :class:`Timeout`. + + Keyword argument *timeout_value* is not passed to *function*. + """ + timeout_value = kwds.pop("timeout_value", _NONE) + timeout = Timeout.start_new(seconds, _one_shot=True) + try: + try: + return function(*args, **kwds) + except Timeout as ex: + if ex is timeout and timeout_value is not _NONE: + return timeout_value + raise + finally: + timeout.cancel() diff --git a/src/gevent/util.py b/src/gevent/util.py new file mode 100644 index 0000000..b24898c --- /dev/null +++ b/src/gevent/util.py @@ -0,0 +1,603 @@ +# Copyright (c) 2009 Denis Bilenko. See LICENSE for details. +""" +Low-level utilities. +""" + +from __future__ import absolute_import, print_function, division + +import functools +import pprint +import sys +import traceback + +from greenlet import getcurrent + +from gevent._compat import perf_counter +from gevent._compat import PYPY +from gevent._compat import thread_mod_name +from gevent._util import _NONE + +__all__ = [ + 'format_run_info', + 'print_run_info', + 'GreenletTree', + 'wrap_errors', + 'assert_switches', +] + +# PyPy is very slow at formatting stacks +# for some reason. +_STACK_LIMIT = 20 if PYPY else None + + +def _noop(): + return None + +def _ready(): + return False + +class wrap_errors(object): + """ + Helper to make function return an exception, rather than raise it. + + Because every exception that is unhandled by greenlet will be logged, + it is desirable to prevent non-error exceptions from leaving a greenlet. + This can done with a simple ``try/except`` construct:: + + def wrapped_func(*args, **kwargs): + try: + return func(*args, **kwargs) + except (TypeError, ValueError, AttributeError) as ex: + return ex + + This class provides a shortcut to write that in one line:: + + wrapped_func = wrap_errors((TypeError, ValueError, AttributeError), func) + + It also preserves ``__str__`` and ``__repr__`` of the original function. + """ + # QQQ could also support using wrap_errors as a decorator + + def __init__(self, errors, func): + """ + Calling this makes a new function from *func*, such that it catches *errors* (an + :exc:`BaseException` subclass, or a tuple of :exc:`BaseException` subclasses) and + return it as a value. + """ + self.__errors = errors + self.__func = func + # Set __doc__, __wrapped__, etc, especially useful on Python 3. + functools.update_wrapper(self, func) + + def __call__(self, *args, **kwargs): + func = self.__func + try: + return func(*args, **kwargs) + except self.__errors as ex: + return ex + + def __str__(self): + return str(self.__func) + + def __repr__(self): + return repr(self.__func) + + def __getattr__(self, name): + return getattr(self.__func, name) + + +def print_run_info(thread_stacks=True, greenlet_stacks=True, limit=_NONE, file=None): + """ + Call `format_run_info` and print the results to *file*. + + If *file* is not given, `sys.stderr` will be used. + + .. versionadded:: 1.3b1 + """ + lines = format_run_info(thread_stacks=thread_stacks, + greenlet_stacks=greenlet_stacks, + limit=limit) + file = sys.stderr if file is None else file + for l in lines: + print(l, file=file) + + +def format_run_info(thread_stacks=True, + greenlet_stacks=True, + limit=_NONE, + current_thread_ident=None): + """ + format_run_info(thread_stacks=True, greenlet_stacks=True, limit=None) -> [str] + + Request information about the running threads of the current process. + + This is a debugging utility. Its output has no guarantees other than being + intended for human consumption. + + :keyword bool thread_stacks: If true, then include the stacks for + running threads. + :keyword bool greenlet_stacks: If true, then include the stacks for + running greenlets. (Spawning stacks will always be printed.) + Setting this to False can reduce the output volume considerably + without reducing the overall information if *thread_stacks* is true + and you can associate a greenlet to a thread (using ``thread_ident`` + printed values). + :keyword int limit: If given, passed directly to `traceback.format_stack`. + If not given, this defaults to the whole stack under CPython, and a + smaller stack under PyPy. + + :return: A sequence of text lines detailing the stacks of running + threads and greenlets. (One greenlet will duplicate one thread, + the current thread and greenlet. If there are multiple running threads, + the stack for the current greenlet may be incorrectly duplicated in multiple + greenlets.) + Extra information about + :class:`gevent.Greenlet` object will also be returned. + + .. versionadded:: 1.3a1 + .. versionchanged:: 1.3a2 + Renamed from ``dump_stacks`` to reflect the fact that this + prints additional information about greenlets, including their + spawning stack, parent, locals, and any spawn tree locals. + .. versionchanged:: 1.3b1 + Added the *thread_stacks*, *greenlet_stacks*, and *limit* params. + """ + if current_thread_ident is None: + from gevent import monkey + current_thread_ident = monkey.get_original(thread_mod_name, 'get_ident')() + + lines = [] + + limit = _STACK_LIMIT if limit is _NONE else limit + _format_thread_info(lines, thread_stacks, limit, current_thread_ident) + _format_greenlet_info(lines, greenlet_stacks, limit) + return lines + + +def _format_thread_info(lines, thread_stacks, limit, current_thread_ident): + import threading + + threads = {th.ident: th for th in threading.enumerate()} + + lines.append('*' * 80) + lines.append('* Threads') + + thread = None + frame = None + for thread_ident, frame in sys._current_frames().items(): + lines.append("*" * 80) + thread = threads.get(thread_ident) + name = thread.name if thread else None + if getattr(thread, 'gevent_monitoring_thread', None): + name = repr(thread.gevent_monitoring_thread()) + if current_thread_ident == thread_ident: + name = '%s) (CURRENT' % (name,) + lines.append('Thread 0x%x (%s)\n' % (thread_ident, name)) + if thread_stacks: + lines.append(''.join(traceback.format_stack(frame, limit))) + else: + lines.append('\t...stack elided...') + + # We may have captured our own frame, creating a reference + # cycle, so clear it out. + del thread + del frame + del lines + del threads + +def _format_greenlet_info(lines, greenlet_stacks, limit): + # Use the gc module to inspect all objects to find the greenlets + # since there isn't a global registry + lines.append('*' * 80) + lines.append('* Greenlets') + lines.append('*' * 80) + for tree in GreenletTree.forest(): + lines.extend(tree.format_lines(details={ + 'running_stacks': greenlet_stacks, + 'running_stack_limit': limit, + })) + + del lines + +dump_stacks = format_run_info + +def _line(f): + @functools.wraps(f) + def w(self, *args, **kwargs): + r = f(self, *args, **kwargs) + self.lines.append(r) + + return w + +class _TreeFormatter(object): + UP_AND_RIGHT = '+' + HORIZONTAL = '-' + VERTICAL = '|' + VERTICAL_AND_RIGHT = '+' + DATA = ':' + + label_space = 1 + horiz_width = 3 + indent = 1 + + def __init__(self, details, depth=0): + self.lines = [] + self.depth = depth + self.details = details + if not details: + self.child_data = lambda *args, **kwargs: None + + def deeper(self): + return type(self)(self.details, self.depth + 1) + + @_line + def node_label(self, text): + return text + + @_line + def child_head(self, label, right=VERTICAL_AND_RIGHT): + return ( + ' ' * self.indent + + right + + self.HORIZONTAL * self.horiz_width + + ' ' * self.label_space + + label + ) + + def last_child_head(self, label): + return self.child_head(label, self.UP_AND_RIGHT) + + @_line + def child_tail(self, line, vertical=VERTICAL): + return ( + ' ' * self.indent + + vertical + + ' ' * self.horiz_width + + line + ) + + def last_child_tail(self, line): + return self.child_tail(line, vertical=' ' * len(self.VERTICAL)) + + @_line + def child_data(self, data, data_marker=DATA): # pylint:disable=method-hidden + return (( + ' ' * self.indent + + (data_marker if not self.depth else ' ') + + ' ' * self.horiz_width + + ' ' * self.label_space + + data + ),) + + def last_child_data(self, data): + return self.child_data(data, ' ') + + def child_multidata(self, data): + # Remove embedded newlines + for l in data.splitlines(): + self.child_data(l) + + +class GreenletTree(object): + """ + Represents a tree of greenlets. + + In gevent, the *parent* of a greenlet is usually the hub, so this + tree is primarily arganized along the *spawning_greenlet* dimension. + + This object has a small str form showing this hierarchy. The `format` + method can output more details. The exact output is unspecified but is + intended to be human readable. + + Use the `forest` method to get the root greenlet trees for + all threads, and the `current_tree` to get the root greenlet tree for + the current thread. + """ + + #: The greenlet this tree represents. + greenlet = None + + + def __init__(self, greenlet): + self.greenlet = greenlet + self.child_trees = [] + + def add_child(self, tree): + if tree is self: + return + self.child_trees.append(tree) + + @property + def root(self): + return self.greenlet.parent is None + + def __getattr__(self, name): + return getattr(self.greenlet, name) + + DEFAULT_DETAILS = { + 'running_stacks': True, + 'running_stack_limit': _STACK_LIMIT, + 'spawning_stacks': True, + 'locals': True, + } + + def format_lines(self, details=True): + """ + Return a sequence of lines for the greenlet tree. + + :keyword bool details: If true (the default), + then include more informative details in the output. + """ + if not isinstance(details, dict): + if not details: + details = {} + else: + details = self.DEFAULT_DETAILS.copy() + else: + params = details + details = self.DEFAULT_DETAILS.copy() + details.update(params) + tree = _TreeFormatter(details, depth=0) + lines = [l[0] if isinstance(l, tuple) else l + for l in self._render(tree)] + return lines + + def format(self, details=True): + """ + Like `format_lines` but returns a string. + """ + lines = self.format_lines(details) + return '\n'.join(lines) + + def __str__(self): + return self.format(False) + + @staticmethod + def __render_tb(tree, label, frame, limit): + tree.child_data(label) + tb = ''.join(traceback.format_stack(frame, limit)) + tree.child_multidata(tb) + + @staticmethod + def __spawning_parent(greenlet): + return (getattr(greenlet, 'spawning_greenlet', None) or _noop)() + + def __render_locals(self, tree): + # Defer the import to avoid cycles + from gevent.local import all_local_dicts_for_greenlet + + gr_locals = all_local_dicts_for_greenlet(self.greenlet) + if gr_locals: + tree.child_data("Greenlet Locals:") + for (kind, idl), vals in gr_locals: + if not vals: + continue # not set in this greenlet; ignore it. + tree.child_data(" Local %s at %s" % (kind, hex(idl))) + tree.child_multidata(" " + pprint.pformat(vals)) + + def _render(self, tree): + label = repr(self.greenlet) + if not self.greenlet: # Not running or dead + # raw greenlets do not have ready + if getattr(self.greenlet, 'ready', _ready)(): + label += '; finished' + if self.greenlet.value is not None: + label += ' with value ' + repr(self.greenlet.value)[:30] + elif getattr(self.greenlet, 'exception', None) is not None: + label += ' with exception ' + repr(self.greenlet.exception) + else: + label += '; not running' + tree.node_label(label) + + tree.child_data('Parent: ' + repr(self.greenlet.parent)) + + if getattr(self.greenlet, 'gevent_monitoring_thread', None) is not None: + tree.child_data('Monitoring Thread:' + repr(self.greenlet.gevent_monitoring_thread())) + + if self.greenlet and tree.details and tree.details['running_stacks']: + self.__render_tb(tree, 'Running:', self.greenlet.gr_frame, + tree.details['running_stack_limit']) + + + spawning_stack = getattr(self.greenlet, 'spawning_stack', None) + if spawning_stack and tree.details and tree.details['spawning_stacks']: + # We already placed a limit on the spawning stack when we captured it. + self.__render_tb(tree, 'Spawned at:', spawning_stack, None) + + spawning_parent = self.__spawning_parent(self.greenlet) + tree_locals = getattr(self.greenlet, 'spawn_tree_locals', None) + if tree_locals and tree_locals is not getattr(spawning_parent, 'spawn_tree_locals', None): + tree.child_data('Spawn Tree Locals') + tree.child_multidata(pprint.pformat(tree_locals)) + + self.__render_locals(tree) + self.__render_children(tree) + return tree.lines + + def __render_children(self, tree): + children = sorted(self.child_trees, + key=lambda c: ( + # raw greenlets first + getattr(c, 'minimal_ident', -1), + # running greenlets first + getattr(c, 'ready', _ready)(), + id(c.parent))) + for n, child in enumerate(children): + child_tree = child._render(tree.deeper()) + + head = tree.child_head + tail = tree.child_tail + data = tree.child_data + + if n == len(children) - 1: + # last child does not get the line drawn + head = tree.last_child_head + tail = tree.last_child_tail + data = tree.last_child_data + + head(child_tree.pop(0)) + for child_data in child_tree: + if isinstance(child_data, tuple): + data(child_data[0]) + else: + tail(child_data) + + return tree.lines + + + @staticmethod + def _root_greenlet(greenlet): + while greenlet.parent is not None and not getattr(greenlet, 'greenlet_tree_is_root', False): + greenlet = greenlet.parent + return greenlet + + @classmethod + def _forest(cls): + from gevent._greenlet_primitives import get_reachable_greenlets + main_greenlet = cls._root_greenlet(getcurrent()) + + trees = {} + roots = {} + current_tree = roots[main_greenlet] = trees[main_greenlet] = cls(main_greenlet) + + glets = get_reachable_greenlets() + + for ob in glets: + spawn_parent = cls.__spawning_parent(ob) + + if spawn_parent is None: + root = cls._root_greenlet(ob) + try: + tree = roots[root] + except KeyError: # pragma: no cover + tree = GreenletTree(root) + roots[root] = trees[root] = tree + else: + try: + tree = trees[spawn_parent] + except KeyError: # pragma: no cover + tree = trees[spawn_parent] = cls(spawn_parent) + + try: + child_tree = trees[ob] + except KeyError: + trees[ob] = child_tree = cls(ob) + tree.add_child(child_tree) + + return roots, current_tree + + @classmethod + def forest(cls): + """ + forest() -> sequence + + Return a sequence of `GreenletTree`, one for each running + native thread. + """ + + return list(cls._forest()[0].values()) + + @classmethod + def current_tree(cls): + """ + current_tree() -> GreenletTree + + Returns the `GreenletTree` for the current thread. + """ + return cls._forest()[1] + +class _FailedToSwitch(AssertionError): + pass + +class assert_switches(object): + """ + A context manager for ensuring a block of code switches greenlets. + + This performs a similar function as the :doc:`monitoring thread + `, but the scope is limited to the body of the with + statement. If the code within the body doesn't yield to the hub + (and doesn't raise an exception), then upon exiting the + context manager an :exc:`AssertionError` will be raised. + + This is useful in unit tests and for debugging purposes. + + :keyword float max_blocking_time: If given, the body is allowed + to block for up to this many fractional seconds before + an error is raised. + :keyword bool hub_only: If True, then *max_blocking_time* only + refers to the amount of time spent between switches into the + hub. If False, then it refers to the maximum time between + *any* switches. If *max_blocking_time* is not given, has no + effect. + + Example:: + + # This will always raise an exception: nothing switched + with assert_switches(): + pass + + # This will never raise an exception; nothing switched, + # but it happened very fast + with assert_switches(max_blocking_time=1.0): + pass + + .. versionadded:: 1.3 + + .. versionchanged:: 1.4 + If an exception is raised, it now includes information about + the duration of blocking and the parameters of this object. + """ + + hub = None + tracer = None + _entered = None + + + def __init__(self, max_blocking_time=None, hub_only=False): + self.max_blocking_time = max_blocking_time + self.hub_only = hub_only + + def __enter__(self): + from gevent import get_hub + from gevent import _tracer + + self.hub = hub = get_hub() + + # TODO: We could optimize this to use the GreenletTracer + # installed by the monitoring thread, if there is one. + # As it is, we will chain trace calls back to it. + if not self.max_blocking_time: + self.tracer = _tracer.GreenletTracer() + elif self.hub_only: + self.tracer = _tracer.HubSwitchTracer(hub, self.max_blocking_time) + else: + self.tracer = _tracer.MaxSwitchTracer(hub, self.max_blocking_time) + + self._entered = perf_counter() + self.tracer.monitor_current_greenlet_blocking() + return self + + def __exit__(self, t, v, tb): + self.tracer.kill() + hub = self.hub; self.hub = None + tracer = self.tracer; self.tracer = None + + # Only check if there was no exception raised, we + # don't want to hide anything + if t is not None: + return + + + did_block = tracer.did_block_hub(hub) + if did_block: + execution_time_s = perf_counter() - self._entered + active_greenlet = did_block[1] + report_lines = tracer.did_block_hub_report(hub, active_greenlet, {}) + + message = 'To the hub' if self.hub_only else 'To any greenlet' + message += ' in %.4f seconds' % (execution_time_s,) + max_block = self.max_blocking_time + message += ' (max allowed %.4f seconds)' % (max_block,) if max_block else '' + message += '\n' + message += '\n'.join(report_lines) + raise _FailedToSwitch(message) diff --git a/src/gevent/win32util.py b/src/gevent/win32util.py new file mode 100644 index 0000000..7158d69 --- /dev/null +++ b/src/gevent/win32util.py @@ -0,0 +1,98 @@ +# Copyright (c) 2001-2007 Twisted Matrix Laboratories. +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Error formatting function for Windows. + +The code is taken from twisted.python.win32 module. +""" + +from __future__ import absolute_import +import os + + +__all__ = ['formatError'] + + +class _ErrorFormatter(object): + """ + Formatter for Windows error messages. + + @ivar winError: A callable which takes one integer error number argument + and returns an L{exceptions.WindowsError} instance for that error (like + L{ctypes.WinError}). + + @ivar formatMessage: A callable which takes one integer error number + argument and returns a C{str} giving the message for that error (like + L{win32api.FormatMessage}). + + @ivar errorTab: A mapping from integer error numbers to C{str} messages + which correspond to those errors (like L{socket.errorTab}). + """ + def __init__(self, WinError, FormatMessage, errorTab): + self.winError = WinError + self.formatMessage = FormatMessage + self.errorTab = errorTab + + @classmethod + def fromEnvironment(cls): + """ + Get as many of the platform-specific error translation objects as + possible and return an instance of C{cls} created with them. + """ + try: + from ctypes import WinError + except ImportError: + WinError = None + try: + from win32api import FormatMessage + except ImportError: + FormatMessage = None + try: + from socket import errorTab + except ImportError: + errorTab = None + return cls(WinError, FormatMessage, errorTab) + + def formatError(self, errorcode): + """ + Returns the string associated with a Windows error message, such as the + ones found in socket.error. + + Attempts direct lookup against the win32 API via ctypes and then + pywin32 if available), then in the error table in the socket module, + then finally defaulting to C{os.strerror}. + + @param errorcode: the Windows error code + @type errorcode: C{int} + + @return: The error message string + @rtype: C{str} + """ + if self.winError is not None: + return str(self.winError(errorcode)) + if self.formatMessage is not None: + return self.formatMessage(errorcode) + if self.errorTab is not None: + result = self.errorTab.get(errorcode) + if result is not None: + return result + return os.strerror(errorcode) + +formatError = _ErrorFormatter.fromEnvironment().formatError diff --git a/src/greentest/2.7.8/badcert.pem b/src/greentest/2.7.8/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/greentest/2.7.8/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/greentest/2.7.8/badkey.pem b/src/greentest/2.7.8/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/greentest/2.7.8/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7.8/https_svn_python_org_root.pem b/src/greentest/2.7.8/https_svn_python_org_root.pem new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/2.7.8/https_svn_python_org_root.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/2.7.8/keycert.pem b/src/greentest/2.7.8/keycert.pem new file mode 100644 index 0000000..64318aa --- /dev/null +++ b/src/greentest/2.7.8/keycert.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/2.7.8/nokia.pem b/src/greentest/2.7.8/nokia.pem new file mode 100644 index 0000000..0d044df --- /dev/null +++ b/src/greentest/2.7.8/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7.8/nullbytecert.pem b/src/greentest/2.7.8/nullbytecert.pem new file mode 100644 index 0000000..447186c --- /dev/null +++ b/src/greentest/2.7.8/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7.8/nullcert.pem b/src/greentest/2.7.8/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/greentest/2.7.8/sha256.pem b/src/greentest/2.7.8/sha256.pem new file mode 100644 index 0000000..9475576 --- /dev/null +++ b/src/greentest/2.7.8/sha256.pem @@ -0,0 +1,128 @@ +# Certificate chain for https://sha256.tbs-internet.com + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=Certificats TBS X509/CN=ecom.tbs-x509.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business +-----BEGIN CERTIFICATE----- +MIIGTjCCBTagAwIBAgIQOh3d9dNDPq1cSdJmEiMpqDANBgkqhkiG9w0BAQUFADCB +yTELMAkGA1UEBhMCRlIxETAPBgNVBAgTCENhbHZhZG9zMQ0wCwYDVQQHEwRDYWVu +MRUwEwYDVQQKEwxUQlMgSU5URVJORVQxSDBGBgNVBAsTP1Rlcm1zIGFuZCBDb25k +aXRpb25zOiBodHRwOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0EvcmVwb3NpdG9y +eTEYMBYGA1UECxMPVEJTIElOVEVSTkVUIENBMR0wGwYDVQQDExRUQlMgWDUwOSBD +QSBidXNpbmVzczAeFw0xMTAxMjUwMDAwMDBaFw0xMzAyMDUyMzU5NTlaMIHHMQsw +CQYDVQQGEwJGUjEOMAwGA1UEERMFMTQwMDAxETAPBgNVBAgTCENhbHZhZG9zMQ0w +CwYDVQQHEwRDQUVOMRswGQYDVQQJExIyMiBydWUgZGUgQnJldGFnbmUxFTATBgNV +BAoTDFRCUyBJTlRFUk5FVDEXMBUGA1UECxMOMDAwMiA0NDA0NDM4MTAxHTAbBgNV +BAsTFENlcnRpZmljYXRzIFRCUyBYNTA5MRowGAYDVQQDExFlY29tLnRicy14NTA5 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRrlHUnJ++1lpcg +jtYco7cdmRe+EEfTmwPfCdfV3G1QfsTSvY6FfMpm/83pqHfT+4ANwr18wD9ZrAEN +G16mf9VdCGK12+TP7DmqeZyGIqlFFoahQnmb8EarvE43/1UeQ2CV9XmzwZvpqeli +LfXsFonawrY3H6ZnMwS64St61Z+9gdyuZ/RbsoZBbT5KUjDEG844QRU4OT1IGeEI +eY5NM5RNIh6ZNhVtqeeCxMS7afONkHQrOco73RdSTRck/Hj96Ofl3MHNHryr+AMK +DGFk1kLCZGpPdXtkxXvaDeQoiYDlil26CWc+YK6xyDPMdsWvoG14ZLyCpzMXA7/7 +4YAQRH0CAwEAAaOCAjAwggIsMB8GA1UdIwQYMBaAFBoJBMz5CY+7HqDO1KQUf0vV +I1jNMB0GA1UdDgQWBBQgOU8HsWzbmD4WZP5Wtdw7jca2WDAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw +TAYDVR0gBEUwQzBBBgsrBgEEAYDlNwIBATAyMDAGCCsGAQUFBwIBFiRodHRwczov +L3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL0NQUzEwdwYDVR0fBHAwbjA3oDWgM4Yx +aHR0cDovL2NybC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNy +bDAzoDGgL4YtaHR0cDovL2NybC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5l +c3MuY3JsMIGwBggrBgEFBQcBAQSBozCBoDA9BggrBgEFBQcwAoYxaHR0cDovL2Ny +dC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNydDA5BggrBgEF +BQcwAoYtaHR0cDovL2NydC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5lc3Mu +Y3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wMwYDVR0R +BCwwKoIRZWNvbS50YnMteDUwOS5jb22CFXd3dy5lY29tLnRicy14NTA5LmNvbTAN +BgkqhkiG9w0BAQUFAAOCAQEArT4NHfbY87bGAw8lPV4DmHlmuDuVp/y7ltO3Ynse +3Rz8RxW2AzuO0Oy2F0Cu4yWKtMyEyMXyHqWtae7ElRbdTu5w5GwVBLJHClCzC8S9 +SpgMMQTx3Rgn8vjkHuU9VZQlulZyiPK7yunjc7c310S9FRZ7XxOwf8Nnx4WnB+No +WrfApzhhQl31w+RyrNxZe58hCfDDHmevRvwLjQ785ZoQXJDj2j3qAD4aI2yB8lB5 +oaE1jlCJzC7Kmz/Y9jzfmv/zAs1LQTm9ktevv4BTUFaGjv9jxnQ1xnS862ZiouLW +zZYIlYPf4F6JjXGiIQgQRglILUfq3ftJd9/ok9W9ZF8h8w== +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFPzCCBCegAwIBAgIQDlBz/++iRSmLDeVRHT/hADANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDcwOTE4MTkyMlow +gckxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEdMBsGA1UEAxMUVEJTIFg1MDkg +Q0EgYnVzaW5lc3MwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDB1PAU +qudCcz3tmyGcf+u6EkZqonKKHrV4gZYbvVkIRojmmlhfi/jwvpHvo8bqSt/9Rj5S +jhCDW0pcbI+IPPtD1Jy+CHNSfnMqVDy6CKQ3p5maTzCMG6ZT+XjnvcND5v+FtaiB +xk1iCX6uvt0jeUtdZvYbyytsSDE6c3Y5//wRxOF8tM1JxibwO3pyER26jbbN2gQz +m/EkdGjLdJ4svPk23WDAvQ6G0/z2LcAaJB+XLfqRwfQpHQvfKa1uTi8PivC8qtip +rmNQMMPMjxSK2azX8cKjjTDJiUKaCb4VHlJDWKEsCFRpgJAoAuX8f7Yfs1M4esGo +sWb3PGspK3O22uIlAgMBAAGjggF6MIIBdjAdBgNVHQ4EFgQUGgkEzPkJj7seoM7U +pBR/S9UjWM0wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwGAYD +VR0gBBEwDzANBgsrBgEEAYDlNwIBATB7BgNVHR8EdDByMDigNqA0hjJodHRwOi8v +Y3JsLmNvbW9kb2NhLmNvbS9BZGRUcnVzdEV4dGVybmFsQ0FSb290LmNybDA2oDSg +MoYwaHR0cDovL2NybC5jb21vZG8ubmV0L0FkZFRydXN0RXh0ZXJuYWxDQVJvb3Qu +Y3JsMIGGBggrBgEFBQcBAQR6MHgwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29t +b2RvY2EuY29tL0FkZFRydXN0VVROU2VydmVyQ0EuY3J0MDkGCCsGAQUFBzAChi1o +dHRwOi8vY3J0LmNvbW9kby5uZXQvQWRkVHJ1c3RVVE5TZXJ2ZXJDQS5jcnQwEQYJ +YIZIAYb4QgEBBAQDAgIEMA0GCSqGSIb3DQEBBQUAA4IBAQA7mqrMgk/MrE6QnbNA +h4nRCn2ti4bg4w2C3lB6bSvRPnYwuNw9Jb8vuKkNFzRDxNJXqVDZdfFW5CVQJuyd +nfAx83+wk+spzvFaE1KhFYfN9G9pQfXUfvDRoIcJgPEKUXL1wRiOG+IjU3VVI8pg +IgqHkr7ylln5i5zCiFAPuIJmYUSFg/gxH5xkCNcjJqqrHrHatJr6Qrrke93joupw +oU1njfAcZtYp6fbiK6u2b1pJqwkVBE8RsfLnPhRj+SFbpvjv8Od7o/ieJhFIYQNU +k2jX2u8qZnAiNw93LZW9lpYjtuvMXq8QQppENNja5b53q7UwI+lU7ZGjZ7quuESp +J6/5 +-----END CERTIFICATE----- + 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware +-----BEGIN CERTIFICATE----- +MIIETzCCAzegAwIBAgIQHM5EYpUZep1jUvnyI6m2mDANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNMDUwNjA3MDgwOTEwWhcNMTkwNzA5MTgxOTIyWjBvMQswCQYD +VQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0 +IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5h +bCBDQSBSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/caM+by +AAQtOeBOW+0fvGwPzbX6I7bO3psRM5ekKUx9k5+9SryT7QMa44/P5W1QWtaXKZRa +gLBJetsulf24yr83OC0ePpFBrXBWx/BPP+gynnTKyJBU6cZfD3idmkA8Dqxhql4U +j56HoWpQ3NeaTq8Fs6ZxlJxxs1BgCscTnTgHhgKo6ahpJhiQq0ywTyOrOk+E2N/O +n+Fpb7vXQtdrROTHre5tQV9yWnEIN7N5ZaRZoJQ39wAvDcKSctrQOHLbFKhFxF0q +fbe01sTurM0TRLfJK91DACX6YblpalgjEbenM49WdVn1zSnXRrcKK2W200JvFbK4 +e/vv6V1T1TRaJwIDAQABo4G9MIG6MB8GA1UdIwQYMBaAFKFyXyYbKJhDlV0HN9WF +lp1L0sNFMB0GA1UdDgQWBBStvZh6NLQm9/rEJlTvA73gJMtUGjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhCAQEEBAMCAQIwRAYDVR0f +BD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VSRmly +c3QtSGFyZHdhcmUuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQByQhANOs4kClrwF8BW +onvUOGCSjRK52zYZgDXYNjDtmr5rJ6NyPFDNn+JxkLpjYetIFMTbSRe679Bt8m7a +gIAoQYFQtxMuyLnJegB2aEbQiIxh/tC21UcFF7ktdnDoTlA6w3pLuvunaI84Of3o +2YBrhzkTbCfaYk5JRlTpudW9DkUkHBsyx3nknPKnplkIGaK0jgn8E0n+SFabYaHk +I9LroYT/+JtLefh9lgBdAgVv0UPbzoGfuDsrk/Zh+UrgbLFpHoVnElhzbkh64Z0X +OGaJunQc68cCZu5HTn/aK7fBGMcVflRCXLVEQpU9PIAdGA8Ynvg684t8GMaKsRl1 +jIGZ +-----END CERTIFICATE----- + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware +-----BEGIN CERTIFICATE----- +MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgxOTIyWjCBlzELMAkG +A1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEe +MBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8v +d3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdh +cmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn +0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlIwrthdBKWHTxqctU8EGc6Oe0rE81m65UJ +M6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFdtqdt++BxF2uiiPsA3/4a +MXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8i4fDidNd +oI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqI +DsjfPe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9Ksy +oUhbAgMBAAGjgbkwgbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYD +VR0OBBYEFKFyXyYbKJhDlV0HN9WFlp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0 +dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNFUkZpcnN0LUhhcmR3YXJlLmNy +bDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUFBwMGBggrBgEF +BQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM +//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28Gpgoiskli +CE7/yMgUsogWXecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gE +CJChicsZUN/KHAG8HQQZexB2lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t +3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kniCrVWFCVH/A7HFe7fRQ5YiuayZSS +KqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67nfhmqA== +-----END CERTIFICATE----- diff --git a/src/greentest/2.7.8/test_ssl.py b/src/greentest/2.7.8/test_ssl.py new file mode 100644 index 0000000..f394dd0 --- /dev/null +++ b/src/greentest/2.7.8/test_ssl.py @@ -0,0 +1,1440 @@ +# Test the support for SSL and sockets + +import sys +import unittest +from test import test_support +import asyncore +import socket +import select +import time +import gc +import os +import errno +import pprint +import urllib, urlparse +import traceback +import weakref +import functools +import platform + +from BaseHTTPServer import HTTPServer +from SimpleHTTPServer import SimpleHTTPRequestHandler + +ssl = test_support.import_module("ssl") + +HOST = test_support.HOST +CERTFILE = None +SVN_PYTHON_ORG_ROOT_CERT = None +NULLBYTECERT = None + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + if test_support.verbose: + sys.stdout.write(prefix + exc_format) + + +class BasicTests(unittest.TestCase): + + def test_sslwrap_simple(self): + # A crude test for the legacy API + try: + ssl.sslwrap_simple(socket.socket(socket.AF_INET)) + except IOError, e: + if e.errno == 32: # broken pipe when ssl_sock.do_handshake(), this test doesn't care about that + pass + else: + raise + try: + ssl.sslwrap_simple(socket.socket(socket.AF_INET)._sock) + except IOError, e: + if e.errno == 32: # broken pipe when ssl_sock.do_handshake(), this test doesn't care about that + pass + else: + raise + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): + if hasattr(ssl, 'PROTOCOL_SSLv2'): + # We need to access the lower-level wrapper in order to create an + # implicit SSL context without trying to connect or listen. + try: + import _ssl + except ImportError: + # The returned function won't get executed, just ignore the error + pass + @functools.wraps(func) + def f(*args, **kwargs): + try: + s = socket.socket(socket.AF_INET) + _ssl.sslwrap(s._sock, 0, None, None, + ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None) + except ssl.SSLError as e: + if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and + platform.linux_distribution() == ('debian', 'squeeze/sid', '') + and 'Invalid SSL protocol variant specified' in str(e)): + raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") + return func(*args, **kwargs) + return f + else: + return func + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + #ssl.PROTOCOL_SSLv2 + ssl.PROTOCOL_SSLv23 + ssl.PROTOCOL_SSLv3 + ssl.PROTOCOL_TLSv1 + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + + def test_random(self): + v = ssl.RAND_status() + if test_support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + self.assertRaises(TypeError, ssl.RAND_egd, 1) + self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1) + ssl.RAND_add("this is a random string", 75.0) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + p = ssl._ssl._test_decode_cert(CERTFILE, False) + if test_support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subject'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if test_support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if test_support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl.OPENSSL_VERSION_INFO >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + + def test_DER_to_PEM(self): + with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, (int, long)) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 0.9 + self.assertGreaterEqual(n, 0x900000) + # < 2.0 + self.assertLess(n, 0x20000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 0) + self.assertLess(major, 2) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 26) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + # Version string as returned by OpenSSL, the format might change + self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)), + (s, t)) + + @test_support.requires_resource('network') + def test_ciphers(self): + remote = ("svn.python.org", 443) + with test_support.transient_internet(remote[0]): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") + s.connect(remote) + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") + s.connect(remote) + # Error checking occurs when connecting, because the SSL context + # isn't created before. + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + with self.assertRaisesRegexp(ssl.SSLError, "No cipher can be selected"): + s.connect(remote) + + @test_support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + wr = weakref.ref(ss) + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # The _delegate_methods in socket.py are correctly delegated to by an + # unconnected SSLSocket, so they will raise a socket.error rather than + # something unexpected like TypeError. + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + self.assertRaises(socket.error, ss.recv, 1) + self.assertRaises(socket.error, ss.recv_into, bytearray(b'x')) + self.assertRaises(socket.error, ss.recvfrom, 1) + self.assertRaises(socket.error, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(socket.error, ss.send, b'x') + self.assertRaises(socket.error, ss.sendto, b'x', ('0.0.0.0', 0)) + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + +class NetworkedTests(unittest.TestCase): + + def test_connect(self): + with test_support.transient_internet("svn.python.org"): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) + s.connect(("svn.python.org", 443)) + c = s.getpeercert() + if c: + self.fail("Peer cert %s shouldn't be here!") + s.close() + + # this should fail because we have no verification certs + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + try: + s.connect(("svn.python.org", 443)) + except ssl.SSLError: + pass + finally: + s.close() + + # this should succeed because we specify the root cert + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SVN_PYTHON_ORG_ROOT_CERT) + try: + s.connect(("svn.python.org", 443)) + finally: + s.close() + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + with test_support.transient_internet("svn.python.org"): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SVN_PYTHON_ORG_ROOT_CERT) + try: + self.assertEqual(0, s.connect_ex(("svn.python.org", 443))) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + with test_support.transient_internet("svn.python.org"): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SVN_PYTHON_ORG_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.setblocking(False) + rc = s.connect_ex(('svn.python.org', 443)) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLError as err: + if err.args[0] == ssl.SSL_ERROR_WANT_READ: + select.select([s], [], [], 5.0) + elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: + select.select([], [s], [], 5.0) + else: + raise + # SSL established + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with test_support.transient_internet("svn.python.org"): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SVN_PYTHON_ORG_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.settimeout(0.0000001) + rc = s.connect_ex(('svn.python.org', 443)) + if rc == 0: + self.skipTest("svn.python.org responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + s.close() + + def test_connect_ex_error(self): + with test_support.transient_internet("svn.python.org"): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SVN_PYTHON_ORG_ROOT_CERT) + try: + self.assertEqual(errno.ECONNREFUSED, + s.connect_ex(("svn.python.org", 444))) + finally: + s.close() + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + with test_support.transient_internet("svn.python.org"): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss.connect(("svn.python.org", 443)) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + with test_support.transient_internet("svn.python.org"): + s = socket.socket(socket.AF_INET) + s.connect(("svn.python.org", 443)) + s.setblocking(False) + s = ssl.wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLError, err: + if err.args[0] == ssl.SSL_ERROR_WANT_READ: + select.select([s], [], []) + elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: + select.select([], [s], []) + else: + raise + s.close() + if test_support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + with test_support.transient_internet("svn.python.org"): + pem = ssl.get_server_certificate(("svn.python.org", 443), + ssl.PROTOCOL_SSLv23) + if not pem: + self.fail("No server certificate on svn.python.org:443!") + + try: + pem = ssl.get_server_certificate(("svn.python.org", 443), + ssl.PROTOCOL_SSLv23, + ca_certs=CERTFILE) + except ssl.SSLError: + #should fail + pass + else: + self.fail("Got server certificate %s for svn.python.org!" % pem) + + pem = ssl.get_server_certificate(("svn.python.org", 443), + ssl.PROTOCOL_SSLv23, + ca_certs=SVN_PYTHON_ORG_ROOT_CERT) + if not pem: + self.fail("No server certificate on svn.python.org:443!") + if test_support.verbose: + sys.stdout.write("\nVerified certificate for svn.python.org:443 is\n%s\n" % pem) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + self.skipTest("remote host needs SNI, only available on Python 3.2+") + # NOTE: https://sha2.hboeck.de is another possible test host + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with test_support.transient_internet("sha256.tbs-internet.com"): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=sha256_cert,) + try: + s.connect(remote) + if test_support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() + + +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + + class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock): + self.server = server + self.running = False + self.sock = connsock + self.sock.setblocking(1) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def show_conn_details(self): + if self.server.certreqs == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if test_support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if test_support.verbose and self.server.chatty: + sys.stdout.write(" cert binary is " + str(len(cert_binary)) + " bytes\n") + cipher = self.sslconn.cipher() + if test_support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + + def wrap_conn(self): + try: + self.sslconn = ssl.wrap_socket(self.sock, server_side=True, + certfile=self.server.certificate, + ssl_version=self.server.protocol, + ca_certs=self.server.cacerts, + cert_reqs=self.server.certreqs, + ciphers=self.server.ciphers) + except ssl.SSLError as e: + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + self.server.conn_errors.append(e) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + + str(self.sock.getpeername()) + ":\n") + self.close() + self.running = False + self.server.stop() + return False + else: + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock._sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if isinstance(self.sock, ssl.SSLSocket): + self.sslconn = self.sock + elif not self.wrap_conn(): + return + self.show_conn_details() + while self.running: + try: + msg = self.read() + if not msg: + # eof, so quit this handler + self.running = False + self.close() + elif msg.strip() == 'over': + if test_support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif self.server.starttls_server and msg.strip() == 'STARTTLS': + if test_support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write("OK\n") + if not self.wrap_conn(): + return + elif self.server.starttls_server and self.sslconn and msg.strip() == 'ENDTLS': + if test_support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write("OK\n") + self.sslconn.unwrap() + self.sslconn = None + if test_support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + else: + if (test_support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %s (%s), sending back %s (%s)...\n" + % (repr(msg), ctype, repr(msg.lower()), ctype)) + self.write(msg.lower()) + except ssl.SSLError: + if self.server.chatty: + handle_error("Test server failure:\n") + self.close() + self.running = False + # normally, we'd just stop here, but for the test + # harness, we want to stop the server + self.server.stop() + + def __init__(self, certificate, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + wrap_accepting_socket=False, ciphers=None): + + if ssl_version is None: + ssl_version = ssl.PROTOCOL_TLSv1 + if certreqs is None: + certreqs = ssl.CERT_NONE + self.certificate = certificate + self.protocol = ssl_version + self.certreqs = certreqs + self.cacerts = cacerts + self.ciphers = ciphers + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.flag = None + if wrap_accepting_socket: + self.sock = ssl.wrap_socket(self.sock, server_side=True, + certfile=self.certificate, + cert_reqs = self.certreqs, + ca_certs = self.cacerts, + ssl_version = self.protocol, + ciphers = self.ciphers) + if test_support.verbose and self.chatty: + sys.stdout.write(' server: wrapped server socket as %s\n' % str(self.sock)) + self.port = test_support.bind_port(self.sock) + self.active = False + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.sock.settimeout(0.05) + self.sock.listen(5) + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if test_support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + str(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn) + handler.start() + handler.join() + except socket.timeout: + pass + except KeyboardInterrupt: + self.stop() + self.sock.close() + + def stop(self): + self.active = False + + class AsyncoreEchoServer(threading.Thread): + + class EchoServer(asyncore.dispatcher): + + class ConnectionHandler(asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + asyncore.dispatcher_with_send.__init__(self, conn) + self.socket = ssl.wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + self._ssl_accepting = True + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except ssl.SSLError, err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + elif err.args[0] == ssl.SSL_ERROR_EOF: + return self.handle_close() + raise + except socket.error, err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if data and data.strip() != 'over': + self.send(data.lower()) + + def handle_close(self): + self.close() + if test_support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + asyncore.dispatcher.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = test_support.bind_port(self.socket) + self.listen(5) + + def handle_accept(self): + sock_obj, addr = self.accept() + if test_support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if test_support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if test_support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if test_support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + asyncore.loop(0.05) + + def stop(self): + self.active = False + self.server.close() + + class SocketServerHTTPSServer(threading.Thread): + + class HTTPSServer(HTTPServer): + + def __init__(self, server_address, RequestHandlerClass, certfile): + HTTPServer.__init__(self, server_address, RequestHandlerClass) + # we assume the certfile contains both private key and certificate + self.certfile = certfile + self.allow_reuse_address = True + + def __str__(self): + return ('<%s %s:%s>' % + (self.__class__.__name__, + self.server_name, + self.server_port)) + + def get_request(self): + # override this to wrap socket with SSL + sock, addr = self.socket.accept() + sslconn = ssl.wrap_socket(sock, server_side=True, + certfile=self.certfile) + return sslconn, addr + + class RootedHTTPRequestHandler(SimpleHTTPRequestHandler): + # need to override translate_path to get a known root, + # instead of using os.curdir, since the test could be + # run from anywhere + + server_version = "TestHTTPS/1.0" + + root = None + + def translate_path(self, path): + """Translate a /-separated PATH to the local filename syntax. + + Components that mean special things to the local file system + (e.g. drive or directory names) are ignored. (XXX They should + probably be diagnosed.) + + """ + # abandon query parameters + path = urlparse.urlparse(path)[2] + path = os.path.normpath(urllib.unquote(path)) + words = path.split('/') + words = filter(None, words) + path = self.root + for word in words: + drive, word = os.path.splitdrive(word) + head, word = os.path.split(word) + if word in self.root: continue + path = os.path.join(path, word) + return path + + def log_message(self, format, *args): + + # we override this to suppress logging unless "verbose" + + if test_support.verbose: + sys.stdout.write(" server (%s:%d %s):\n [%s] %s\n" % + (self.server.server_address, + self.server.server_port, + self.request.cipher(), + self.log_date_time_string(), + format%args)) + + + def __init__(self, certfile): + self.flag = None + self.RootedHTTPRequestHandler.root = os.path.split(CERTFILE)[0] + self.server = self.HTTPSServer( + (HOST, 0), self.RootedHTTPRequestHandler, certfile) + self.port = self.server.server_port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + if self.flag: + self.flag.set() + self.server.serve_forever(0.05) + + def stop(self): + self.server.shutdown() + + + def bad_cert_test(certfile): + """ + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with the given client certificate fails. + """ + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_REQUIRED, + cacerts=CERTFILE, chatty=False) + with server: + try: + s = ssl.wrap_socket(socket.socket(), + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + except ssl.SSLError, x: + if test_support.verbose: + sys.stdout.write("\nSSLError is %s\n" % x[1]) + except socket.error, x: + if test_support.verbose: + sys.stdout.write("\nsocket.error is %s\n" % x[1]) + else: + raise AssertionError("Use of invalid cert should have failed!") + + def server_params_test(certfile, protocol, certreqs, cacertsfile, + client_certfile, client_protocol=None, indata="FOO\n", + ciphers=None, chatty=True, connectionchatty=False, + wrap_accepting_socket=False): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + server = ThreadedEchoServer(certfile, + certreqs=certreqs, + ssl_version=protocol, + cacerts=cacertsfile, + ciphers=ciphers, + chatty=chatty, + connectionchatty=connectionchatty, + wrap_accepting_socket=wrap_accepting_socket) + with server: + # try to connect + if client_protocol is None: + client_protocol = protocol + s = ssl.wrap_socket(socket.socket(), + certfile=client_certfile, + ca_certs=cacertsfile, + ciphers=ciphers, + cert_reqs=certreqs, + ssl_version=client_protocol) + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if test_support.verbose: + sys.stdout.write( + " client: sending %s...\n" % (repr(arg))) + s.write(arg) + outdata = s.read() + if connectionchatty: + if test_support.verbose: + sys.stdout.write(" client: read %s\n" % repr(outdata)) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%s>> (%d) received; expected <<%s>> (%d)\n" + % (outdata[:min(len(outdata),20)], len(outdata), + indata[:min(len(indata),20)].lower(), len(indata))) + s.write("over\n") + if connectionchatty: + if test_support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + + def try_protocol_combo(server_protocol, + client_protocol, + expect_success, + certsreqs=None): + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if test_support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + try: + # NOTE: we must enable "ALL" ciphers, otherwise an SSLv23 client + # will send an SSLv3 hello (rather than SSLv2) starting from + # OpenSSL 1.0.0 (see issue #8322). + server_params_test(CERTFILE, server_protocol, certsreqs, + CERTFILE, CERTFILE, client_protocol, + ciphers="ALL", chatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except socket.error as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + + + class ThreadedTests(unittest.TestCase): + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an IOError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = test_support.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen(5) + listener_ready.set() + s.accept() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + c = socket.socket() + c.connect((HOST, port)) + listener_gone.wait() + # XXX why is it necessary? + test_support.gc_collect() + try: + ssl_sock = ssl.wrap_socket(c) + except IOError: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + @skip_if_broken_ubuntu_ssl + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if test_support.verbose: + sys.stdout.write("\n") + server_params_test(CERTFILE, ssl.PROTOCOL_TLSv1, ssl.CERT_NONE, + CERTFILE, CERTFILE, ssl.PROTOCOL_TLSv1, + chatty=True, connectionchatty=True) + + def test_getpeercert(self): + if test_support.verbose: + sys.stdout.write("\n") + s2 = socket.socket() + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_SSLv23, + cacerts=CERTFILE, + chatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_REQUIRED, + ssl_version=ssl.PROTOCOL_SSLv23) + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if test_support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + s.close() + + def test_empty_cert(self): + """Connecting with an empty cert file""" + bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir, + "nullcert.pem")) + def test_malformed_cert(self): + """Connecting with a badly formatted certificate (syntax error)""" + bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir, + "badcert.pem")) + def test_nonexisting_cert(self): + """Connecting with a non-existing cert file""" + bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir, + "wrongcert.pem")) + def test_malformed_key(self): + """Connecting with a badly formatted key (syntax error)""" + bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir, + "badkey.pem")) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv2(self): + """Connecting to an SSLv2 server with various client options""" + if test_support.verbose: + sys.stdout.write("\n") + if not hasattr(ssl, 'PROTOCOL_SSLv2'): + self.skipTest("PROTOCOL_SSLv2 needed") + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv23(self): + """Connecting to an SSLv23 server with various client options""" + if test_support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if test_support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + + @skip_if_broken_ubuntu_ssl + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if test_support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = ("msg 1", "MSG 2", "STARTTLS", "MSG 3", "msg 4", "ENDTLS", "msg 5", "msg 6") + + server = ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(1) + s.connect((HOST, server.port)) + if test_support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if test_support.verbose: + sys.stdout.write( + " client: sending %s...\n" % repr(indata)) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + if (indata == "STARTTLS" and + outdata.strip().lower().startswith("ok")): + # STARTTLS ok, switch to secure mode + if test_support.verbose: + sys.stdout.write( + " client: read %s from server, starting TLS...\n" + % repr(outdata)) + conn = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + wrapped = True + elif (indata == "ENDTLS" and + outdata.strip().lower().startswith("ok")): + # ENDTLS ok, switch back to clear text + if test_support.verbose: + sys.stdout.write( + " client: read %s from server, ending TLS...\n" + % repr(outdata)) + s = conn.unwrap() + wrapped = False + else: + if test_support.verbose: + sys.stdout.write( + " client: read %s from server\n" % repr(outdata)) + if test_support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write("over\n") + else: + s.send("over\n") + s.close() + + def test_socketserver(self): + """Using a SocketServer to create and manage SSL connections.""" + server = SocketServerHTTPSServer(CERTFILE) + flag = threading.Event() + server.start(flag) + # wait for it to start + flag.wait() + # try to connect + try: + if test_support.verbose: + sys.stdout.write('\n') + with open(CERTFILE, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = 'https://127.0.0.1:%d/%s' % ( + server.port, os.path.split(CERTFILE)[1]) + with test_support.check_py3k_warnings(): + f = urllib.urlopen(url) + dlen = f.info().getheader("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if test_support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + f.close() + self.assertEqual(d1, d2) + finally: + server.stop() + server.join() + + def test_wrapped_accept(self): + """Check the accept() method on SSL sockets.""" + if test_support.verbose: + sys.stdout.write("\n") + server_params_test(CERTFILE, ssl.PROTOCOL_SSLv23, ssl.CERT_REQUIRED, + CERTFILE, CERTFILE, ssl.PROTOCOL_SSLv23, + chatty=True, connectionchatty=True, + wrap_accepting_socket=True) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + indata = "TEST MESSAGE of mixed case\n" + + if test_support.verbose: + sys.stdout.write("\n") + server = AsyncoreEchoServer(CERTFILE) + with server: + s = ssl.wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if test_support.verbose: + sys.stdout.write( + " client: sending %s...\n" % (repr(indata))) + s.write(indata) + outdata = s.read() + if test_support.verbose: + sys.stdout.write(" client: read %s\n" % repr(outdata)) + if outdata != indata.lower(): + self.fail( + "bad data <<%s>> (%d) received; expected <<%s>> (%d)\n" + % (outdata[:min(len(outdata),20)], len(outdata), + indata[:min(len(indata),20)].lower(), len(indata))) + s.write("over\n") + if test_support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if test_support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray("\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray("\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, whether to expect success, *args) + send_methods = [ + ('send', s.send, True, []), + ('sendto', s.sendto, False, ["some.address"]), + ('sendall', s.sendall, True, []), + ] + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = u"PREFIX_" + + for meth_name, send_meth, expect_success, args in send_methods: + indata = data_prefix + meth_name + try: + send_meth(indata.encode('ASCII', 'strict'), *args) + outdata = s.read() + outdata = outdata.decode('ASCII', 'strict') + if outdata != indata.lower(): + self.fail( + "While sending with <<%s>> bad data " + "<<%r>> (%d) received; " + "expected <<%r>> (%d)\n" % ( + meth_name, outdata[:20], len(outdata), + indata[:20], len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<%s>>; " + "expected to succeed.\n" % (meth_name,) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<%s>> failed with unexpected " + "exception message: %s\n" % ( + meth_name, e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = data_prefix + meth_name + try: + s.send(indata.encode('ASCII', 'strict')) + outdata = recv_meth(*args) + outdata = outdata.decode('ASCII', 'strict') + if outdata != indata.lower(): + self.fail( + "While receiving with <<%s>> bad data " + "<<%r>> (%d) received; " + "expected <<%r>> (%d)\n" % ( + meth_name, outdata[:20], len(outdata), + indata[:20], len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<%s>>; " + "expected to succeed.\n" % (meth_name,) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<%s>> failed with unexpected " + "exception message: %s\n" % ( + meth_name, e + ) + ) + # consume data + s.read() + + s.write("over\n".encode("ASCII", "strict")) + s.close() + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = test_support.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen(5) + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegexp(ssl.SSLError, "timed out", + ssl.wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c = ssl.wrap_socket(c) + # Will attempt handshake and time out + self.assertRaisesRegexp(ssl.SSLError, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_default_ciphers(self): + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_SSLv23, + chatty=False) as server: + sock = socket.socket() + try: + # Force a set of weak ciphers on our client socket + try: + s = ssl.wrap_socket(sock, + ssl_version=ssl.PROTOCOL_SSLv23, + ciphers="DES") + except ssl.SSLError: + self.skipTest("no DES cipher available") + with self.assertRaises((OSError, ssl.SSLError)): + s.connect((HOST, server.port)) + finally: + sock.close() + self.assertIn("no shared cipher", str(server.conn_errors[0])) + + +def test_main(verbose=False): + global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, NOKIACERT, NULLBYTECERT + CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, + "keycert.pem") + SVN_PYTHON_ORG_ROOT_CERT = os.path.join( + os.path.dirname(__file__) or os.curdir, + "https_svn_python_org_root.pem") + NOKIACERT = os.path.join(os.path.dirname(__file__) or os.curdir, + "nokia.pem") + NULLBYTECERT = os.path.join(os.path.dirname(__file__) or os.curdir, + "nullbytecert.pem") + + if (not os.path.exists(CERTFILE) or + not os.path.exists(SVN_PYTHON_ORG_ROOT_CERT) or + not os.path.exists(NOKIACERT) or + not os.path.exists(NULLBYTECERT)): + raise test_support.TestFailed("Can't read certificate files!") + + tests = [BasicTests, BasicSocketTests] + + if test_support.is_resource_enabled('network'): + tests.append(NetworkedTests) + + if _have_threads: + thread_info = test_support.threading_setup() + if thread_info and test_support.is_resource_enabled('network'): + tests.append(ThreadedTests) + + try: + test_support.run_unittest(*tests) + finally: + if _have_threads: + test_support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7.8/wrongcert.pem b/src/greentest/2.7.8/wrongcert.pem new file mode 100644 index 0000000..5f92f9b --- /dev/null +++ b/src/greentest/2.7.8/wrongcert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH +FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T +f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB +AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq +1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW +7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg +SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe +19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg +ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666 +lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs +ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv +frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk +2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6 ++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK +24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G +A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9 +OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ +fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E +usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+ +43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw +eSHj5jpC8iZKjCHBn+mAi4cQ514= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/badcert.pem b/src/greentest/2.7/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/greentest/2.7/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/badkey.pem b/src/greentest/2.7/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/greentest/2.7/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/capath/0e4015b9.0 b/src/greentest/2.7/capath/0e4015b9.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/2.7/capath/0e4015b9.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/capath/4e1295a3.0 b/src/greentest/2.7/capath/4e1295a3.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/2.7/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/capath/5ed36f99.0 b/src/greentest/2.7/capath/5ed36f99.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/2.7/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/capath/6e88d7b8.0 b/src/greentest/2.7/capath/6e88d7b8.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/2.7/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/capath/99d0fa06.0 b/src/greentest/2.7/capath/99d0fa06.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/2.7/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/capath/ce7b8643.0 b/src/greentest/2.7/capath/ce7b8643.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/2.7/capath/ce7b8643.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/dh1024.pem b/src/greentest/2.7/dh1024.pem new file mode 100644 index 0000000..a391176 --- /dev/null +++ b/src/greentest/2.7/dh1024.pem @@ -0,0 +1,7 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAIbzw1s9CT8SV5yv6L7esdAdZYZjPi3qWFs61CYTFFQnf2s/d09NYaJt +rrvJhIzWavqnue71qXCf83/J3nz3FEwUU/L0mGyheVbsSHiI64wUo3u50wK5Igo0 +RNs/LD0irs7m0icZ//hijafTU+JOBiuA8zMI+oZfU7BGuc9XrUprAgEC +-----END DH PARAMETERS----- + +Generated with: openssl dhparam -out dh1024.pem 1024 diff --git a/src/greentest/2.7/https_svn_python_org_root.pem b/src/greentest/2.7/https_svn_python_org_root.pem new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/2.7/https_svn_python_org_root.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/keycert.passwd.pem b/src/greentest/2.7/keycert.passwd.pem new file mode 100644 index 0000000..e905748 --- /dev/null +++ b/src/greentest/2.7/keycert.passwd.pem @@ -0,0 +1,33 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/keycert.pem b/src/greentest/2.7/keycert.pem new file mode 100644 index 0000000..64318aa --- /dev/null +++ b/src/greentest/2.7/keycert.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/keycert2.pem b/src/greentest/2.7/keycert2.pem new file mode 100644 index 0000000..c4a18bf --- /dev/null +++ b/src/greentest/2.7/keycert2.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBANcLaMB7T/Wi9DBc +PltGzgt8cxsv55m7PQPHMZvn6Ke8xmNqcmEzib8opRwKGrCV6TltKeFlNSg8dwQK +Tl4ktyTkGCVweRQJ37AkBayvEBml5s+QD4vlhqkJPsL/Nsd+fnqngOGc5+59+C6r +s3XpiLlF5ah/z8q92Mnw54nypw1JAgMBAAECgYBE3t2Mj7GbDLZB6rj5yKJioVfI +BD6bSJEQ7bGgqdQkLFwpKMU7BiN+ekjuwvmrRkesYZ7BFgXBPiQrwhU5J28Tpj5B +EOMYSIOHfzdalhxDGM1q2oK9LDFiCotTaSdEzMYadel5rmKXJ0zcK2Jho0PCuECf +tf/ghRxK+h1Hm0tKgQJBAO6MdGDSmGKYX6/5kPDje7we/lSLorSDkYmV0tmVShsc +JxgaGaapazceA/sHL3Myx7Eenkip+yPYDXEDFvAKNDECQQDmxsT9NOp6mo7ISvky +GFr2vVHsJ745BMWoma4rFjPBVnS8RkgK+b2EpDCdZSrQ9zw2r8sKTgrEyrDiGTEg +wJyZAkA8OOc0flYMJg2aHnYR6kwVjPmGHI5h5gk648EMPx0rROs1sXkiUwkHLCOz +HvhCq+Iv+9vX2lnVjbiu/CmxRdIxAkA1YEfzoKeTD+hyXxTgB04Sv5sRGegfXAEz +i8gC4zG5R/vcCA1lrHmvEiLEZL/QcT6WD3bQvVg0SAU9ZkI8pxARAkA7yqMSvP1l +gJXy44R+rzpLYb1/PtiLkIkaKG3x9TUfPnfD2jY09fPkZlfsRU3/uS09IkhSwimV +d5rWoljEfdou +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICXTCCAcagAwIBAgIJALVQzebTtrXFMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTAeFw0x +NDExMjMxNzAwMDdaFw0yNDExMjAxNzAwMDdaMGIxCzAJBgNVBAYTAlhZMRcwFQYD +VQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZv +dW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEA1wtowHtP9aL0MFw+W0bOC3xzGy/nmbs9A8cxm+fop7zGY2py +YTOJvyilHAoasJXpOW0p4WU1KDx3BApOXiS3JOQYJXB5FAnfsCQFrK8QGaXmz5AP +i+WGqQk+wv82x35+eqeA4Zzn7n34LquzdemIuUXlqH/Pyr3YyfDnifKnDUkCAwEA +AaMbMBkwFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBBQUAA4GB +AKuay3vDKfWzt5+ch/HHBsert84ISot4fUjzXDA/oOgTOEjVcSShHxqNShMOW1oA +QYBpBB/5Kx5RkD/w6imhucxt2WQPRgjX4x4bwMipVH/HvFDp03mG51/Cpi1TyZ74 +El7qa/Pd4lHhOLzMKBA6503fpeYSFUIBxZbGLqylqRK7 +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/keycert3.pem b/src/greentest/2.7/keycert3.pem new file mode 100644 index 0000000..5bfa62c --- /dev/null +++ b/src/greentest/2.7/keycert3.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP +jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM +9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ +aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe +yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j +y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ +AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW +5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL +9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 +1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT +DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh +1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m +JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 +RnJdHOMXWem7/w== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: + 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: + c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: + 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: + f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: + 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: + f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: + af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: + 21:82:a5:3c:88:e5:be:1b:b1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: + e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: + f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: + e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: + d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: + 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: + ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: + 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: + 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: + 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: + 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: + f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: + 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: + a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: + fc:a9:94:71 +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C +tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola +N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 +TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR +iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG +xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo +5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv +mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF +YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh +2EJ36/yplHE= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/keycert4.pem b/src/greentest/2.7/keycert4.pem new file mode 100644 index 0000000..53355c8 --- /dev/null +++ b/src/greentest/2.7/keycert4.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv +L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2 +NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1 +L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L +pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de +R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9 +myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT +drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS +Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx +i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK +Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu +JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN ++/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/ +e83Gq6ffLVfKNQ== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443282 (0xb09264b1f2da21d2) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14: + 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9: + cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a: + b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76: + 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7: + 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a: + d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79: + 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd: + 81:7e:bd:1b:ae:0b:5d:c6:39 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0: + 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f: + 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56: + 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15: + 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1: + 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb: + 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e: + e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18: + d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1: + af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53: + 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68: + 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da: + 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92: + 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa: + 49:12:1e:ce +-----BEGIN CERTIFICATE----- +MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z +dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU +aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0 +ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ +hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v +xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338 +Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP +XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0 +UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz +aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb +oF+6ufu6+kkSHs4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/lock_tests.py b/src/greentest/2.7/lock_tests.py new file mode 100644 index 0000000..f04fc20 --- /dev/null +++ b/src/greentest/2.7/lock_tests.py @@ -0,0 +1,560 @@ +""" +Various tests for synchronization primitives. +""" + +import sys +import time +from thread import start_new_thread, get_ident +import threading +import unittest + +from test import test_support as support + + +def _wait(): + # A crude wait/yield function not relying on synchronization primitives. + time.sleep(0.01) + +class Bunch(object): + """ + A bunch of threads. + """ + def __init__(self, f, n, wait_before_exit=False): + """ + Construct a bunch of `n` threads running the same function `f`. + If `wait_before_exit` is True, the threads won't terminate until + do_finish() is called. + """ + self.f = f + self.n = n + self.started = [] + self.finished = [] + self._can_exit = not wait_before_exit + def task(): + tid = get_ident() + self.started.append(tid) + try: + f() + finally: + self.finished.append(tid) + while not self._can_exit: + _wait() + try: + for i in range(n): + start_new_thread(task, ()) + except: + self._can_exit = True + raise + + def wait_for_started(self): + while len(self.started) < self.n: + _wait() + + def wait_for_finished(self): + while len(self.finished) < self.n: + _wait() + + def do_finish(self): + self._can_exit = True + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = support.threading_setup() + + def tearDown(self): + support.threading_cleanup(*self._threads) + support.reap_children() + + +class BaseLockTests(BaseTestCase): + """ + Tests for both recursive and non-recursive locks. + """ + + def test_constructor(self): + lock = self.locktype() + del lock + + def test_acquire_destroy(self): + lock = self.locktype() + lock.acquire() + del lock + + def test_acquire_release(self): + lock = self.locktype() + lock.acquire() + lock.release() + del lock + + def test_try_acquire(self): + lock = self.locktype() + self.assertTrue(lock.acquire(False)) + lock.release() + + def test_try_acquire_contended(self): + lock = self.locktype() + lock.acquire() + result = [] + def f(): + result.append(lock.acquire(False)) + Bunch(f, 1).wait_for_finished() + self.assertFalse(result[0]) + lock.release() + + def test_acquire_contended(self): + lock = self.locktype() + lock.acquire() + N = 5 + def f(): + lock.acquire() + lock.release() + + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(len(b.finished), 0) + lock.release() + b.wait_for_finished() + self.assertEqual(len(b.finished), N) + + def test_with(self): + lock = self.locktype() + def f(): + lock.acquire() + lock.release() + def _with(err=None): + with lock: + if err is not None: + raise err + _with() + # Check the lock is unacquired + Bunch(f, 1).wait_for_finished() + self.assertRaises(TypeError, _with, TypeError) + # Check the lock is unacquired + Bunch(f, 1).wait_for_finished() + + def test_thread_leak(self): + # The lock shouldn't leak a Thread instance when used from a foreign + # (non-threading) thread. + lock = self.locktype() + def f(): + lock.acquire() + lock.release() + n = len(threading.enumerate()) + # We run many threads in the hope that existing threads ids won't + # be recycled. + Bunch(f, 15).wait_for_finished() + self.assertEqual(n, len(threading.enumerate())) + + +class LockTests(BaseLockTests): + """ + Tests for non-recursive, weak locks + (which can be acquired and released from different threads). + """ + def test_reacquire(self): + # Lock needs to be released before re-acquiring. + lock = self.locktype() + phase = [] + def f(): + lock.acquire() + phase.append(None) + lock.acquire() + phase.append(None) + start_new_thread(f, ()) + while len(phase) == 0: + _wait() + _wait() + self.assertEqual(len(phase), 1) + lock.release() + while len(phase) == 1: + _wait() + self.assertEqual(len(phase), 2) + + def test_different_thread(self): + # Lock can be released from a different thread. + lock = self.locktype() + lock.acquire() + def f(): + lock.release() + b = Bunch(f, 1) + b.wait_for_finished() + lock.acquire() + lock.release() + + +class RLockTests(BaseLockTests): + """ + Tests for recursive locks. + """ + def test_reacquire(self): + lock = self.locktype() + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + + def test_release_unacquired(self): + # Cannot release an unacquired lock + lock = self.locktype() + self.assertRaises(RuntimeError, lock.release) + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + self.assertRaises(RuntimeError, lock.release) + + def test_different_thread(self): + # Cannot release from a different thread + lock = self.locktype() + def f(): + lock.acquire() + b = Bunch(f, 1, True) + try: + self.assertRaises(RuntimeError, lock.release) + finally: + b.do_finish() + + def test__is_owned(self): + lock = self.locktype() + self.assertFalse(lock._is_owned()) + lock.acquire() + self.assertTrue(lock._is_owned()) + lock.acquire() + self.assertTrue(lock._is_owned()) + result = [] + def f(): + result.append(lock._is_owned()) + Bunch(f, 1).wait_for_finished() + self.assertFalse(result[0]) + lock.release() + self.assertTrue(lock._is_owned()) + lock.release() + self.assertFalse(lock._is_owned()) + + +class EventTests(BaseTestCase): + """ + Tests for Event objects. + """ + + def test_is_set(self): + evt = self.eventtype() + self.assertFalse(evt.is_set()) + evt.set() + self.assertTrue(evt.is_set()) + evt.set() + self.assertTrue(evt.is_set()) + evt.clear() + self.assertFalse(evt.is_set()) + evt.clear() + self.assertFalse(evt.is_set()) + + def _check_notify(self, evt): + # All threads get notified + N = 5 + results1 = [] + results2 = [] + def f(): + results1.append(evt.wait()) + results2.append(evt.wait()) + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(len(results1), 0) + evt.set() + b.wait_for_finished() + self.assertEqual(results1, [True] * N) + self.assertEqual(results2, [True] * N) + + def test_notify(self): + evt = self.eventtype() + self._check_notify(evt) + # Another time, after an explicit clear() + evt.set() + evt.clear() + self._check_notify(evt) + + def test_timeout(self): + evt = self.eventtype() + results1 = [] + results2 = [] + N = 5 + def f(): + results1.append(evt.wait(0.0)) + t1 = time.time() + r = evt.wait(0.2) + t2 = time.time() + results2.append((r, t2 - t1)) + Bunch(f, N).wait_for_finished() + self.assertEqual(results1, [False] * N) + for r, dt in results2: + self.assertFalse(r) + self.assertTrue(dt >= 0.19, dt) # XXX: gevent: modified for libuv from 0.2 sometimes get 0.199865 + # The event is set + results1 = [] + results2 = [] + evt.set() + Bunch(f, N).wait_for_finished() + self.assertEqual(results1, [True] * N) + for r, dt in results2: + self.assertTrue(r) + + def test_reset_internal_locks(self): + evt = self.eventtype() + if not hasattr(evt, '_Event__cond') or sys.version_info[:3] <= (2, 7, 8): + self.skipTest("gevent: internal impl difference") + old_lock = evt._Event__cond._Condition__lock + evt._reset_internal_locks() + new_lock = evt._Event__cond._Condition__lock + self.assertIsNot(new_lock, old_lock) + self.assertIs(type(new_lock), type(old_lock)) + + +class ConditionTests(BaseTestCase): + """ + Tests for condition variables. + """ + + def test_acquire(self): + cond = self.condtype() + # Be default we have an RLock: the condition can be acquired multiple + # times. + cond.acquire() + cond.acquire() + cond.release() + cond.release() + lock = threading.Lock() + cond = self.condtype(lock) + cond.acquire() + self.assertFalse(lock.acquire(False)) + cond.release() + self.assertTrue(lock.acquire(False)) + self.assertFalse(cond.acquire(False)) + lock.release() + with cond: + self.assertFalse(lock.acquire(False)) + + def test_unacquired_wait(self): + cond = self.condtype() + self.assertRaises(RuntimeError, cond.wait) + + def test_unacquired_notify(self): + cond = self.condtype() + self.assertRaises(RuntimeError, cond.notify) + + def _check_notify(self, cond): + N = 5 + results1 = [] + results2 = [] + phase_num = 0 + def f(): + cond.acquire() + cond.wait() + cond.release() + results1.append(phase_num) + cond.acquire() + cond.wait() + cond.release() + results2.append(phase_num) + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(results1, []) + # Notify 3 threads at first + cond.acquire() + cond.notify(3) + _wait() + phase_num = 1 + cond.release() + while len(results1) < 3: + _wait() + self.assertEqual(results1, [1] * 3) + self.assertEqual(results2, []) + # Notify 5 threads: they might be in their first or second wait + cond.acquire() + cond.notify(5) + _wait() + phase_num = 2 + cond.release() + while len(results1) + len(results2) < 8: + _wait() + self.assertEqual(results1, [1] * 3 + [2] * 2) + self.assertEqual(results2, [2] * 3) + # Notify all threads: they are all in their second wait + cond.acquire() + cond.notify_all() + _wait() + phase_num = 3 + cond.release() + while len(results2) < 5: + _wait() + self.assertEqual(results1, [1] * 3 + [2] * 2) + self.assertEqual(results2, [2] * 3 + [3] * 2) + b.wait_for_finished() + + def test_notify(self): + cond = self.condtype() + self._check_notify(cond) + # A second time, to check internal state is still ok. + self._check_notify(cond) + + def test_timeout(self): + cond = self.condtype() + results = [] + N = 5 + def f(): + cond.acquire() + t1 = time.time() + cond.wait(0.2) + t2 = time.time() + cond.release() + results.append(t2 - t1) + Bunch(f, N).wait_for_finished() + self.assertEqual(len(results), 5) + for dt in results: + self.assertTrue(dt >= 0.19, dt) # XXX: gevent: gevent: modified from 0.2. sometimes get 0.199865 + + +class BaseSemaphoreTests(BaseTestCase): + """ + Common tests for {bounded, unbounded} semaphore objects. + """ + + def test_constructor(self): + self.assertRaises(ValueError, self.semtype, value = -1) + self.assertRaises(ValueError, self.semtype, value = -sys.maxint) + + def test_acquire(self): + sem = self.semtype(1) + sem.acquire() + sem.release() + sem = self.semtype(2) + sem.acquire() + sem.acquire() + sem.release() + sem.release() + + def test_acquire_destroy(self): + sem = self.semtype() + sem.acquire() + del sem + + def test_acquire_contended(self): + sem = self.semtype(7) + sem.acquire() + N = 10 + results1 = [] + results2 = [] + phase_num = 0 + def f(): + sem.acquire() + results1.append(phase_num) + sem.acquire() + results2.append(phase_num) + b = Bunch(f, 10) + b.wait_for_started() + while len(results1) + len(results2) < 6: + _wait() + self.assertEqual(results1 + results2, [0] * 6) + phase_num = 1 + for i in range(7): + sem.release() + while len(results1) + len(results2) < 13: + _wait() + self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) + phase_num = 2 + for i in range(6): + sem.release() + while len(results1) + len(results2) < 19: + _wait() + self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + # Final release, to let the last thread finish + sem.release() + b.wait_for_finished() + + def test_try_acquire(self): + sem = self.semtype(2) + self.assertTrue(sem.acquire(False)) + self.assertTrue(sem.acquire(False)) + self.assertFalse(sem.acquire(False)) + sem.release() + self.assertTrue(sem.acquire(False)) + + def test_try_acquire_contended(self): + sem = self.semtype(4) + sem.acquire() + results = [] + def f(): + results.append(sem.acquire(False)) + results.append(sem.acquire(False)) + Bunch(f, 5).wait_for_finished() + # There can be a thread switch between acquiring the semaphore and + # appending the result, therefore results will not necessarily be + # ordered. + self.assertEqual(sorted(results), [False] * 7 + [True] * 3 ) + + def test_default_value(self): + # The default initial value is 1. + sem = self.semtype() + sem.acquire() + def f(): + sem.acquire() + sem.release() + b = Bunch(f, 1) + b.wait_for_started() + _wait() + self.assertFalse(b.finished) + sem.release() + b.wait_for_finished() + + def test_with(self): + sem = self.semtype(2) + def _with(err=None): + with sem: + self.assertTrue(sem.acquire(False)) + sem.release() + with sem: + self.assertFalse(sem.acquire(False)) + if err: + raise err + _with() + self.assertTrue(sem.acquire(False)) + sem.release() + self.assertRaises(TypeError, _with, TypeError) + self.assertTrue(sem.acquire(False)) + sem.release() + +class SemaphoreTests(BaseSemaphoreTests): + """ + Tests for unbounded semaphores. + """ + + def test_release_unacquired(self): + # Unbounded releases are allowed and increment the semaphore's value + sem = self.semtype(1) + sem.release() + sem.acquire() + sem.acquire() + sem.release() + + +class BoundedSemaphoreTests(BaseSemaphoreTests): + """ + Tests for bounded semaphores. + """ + + def test_release_unacquired(self): + # Cannot go past the initial value + sem = self.semtype() + self.assertRaises(ValueError, sem.release) + sem.acquire() + sem.release() + self.assertRaises(ValueError, sem.release) diff --git a/src/greentest/2.7/nokia.pem b/src/greentest/2.7/nokia.pem new file mode 100644 index 0000000..0d044df --- /dev/null +++ b/src/greentest/2.7/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/nullbytecert.pem b/src/greentest/2.7/nullbytecert.pem new file mode 100644 index 0000000..447186c --- /dev/null +++ b/src/greentest/2.7/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/nullcert.pem b/src/greentest/2.7/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/greentest/2.7/pycacert.pem b/src/greentest/2.7/pycacert.pem new file mode 100644 index 0000000..09b1f3e --- /dev/null +++ b/src/greentest/2.7/pycacert.pem @@ -0,0 +1,78 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Jan 2 19:47:07 2023 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: + 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: + e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: + e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: + 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: + 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: + a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: + e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: + 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: + 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: + e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: + c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: + cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: + 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: + 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: + 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: + e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: + c5:4d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + X509v3 Authority Key Identifier: + keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: + 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: + a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: + 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: + 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: + 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: + fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: + 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: + 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: + 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: + ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: + 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: + b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: + 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: + 5e:58:c8:9e +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/revocation.crl b/src/greentest/2.7/revocation.crl new file mode 100644 index 0000000..6d89b08 --- /dev/null +++ b/src/greentest/2.7/revocation.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTEzMTEyMTE3MDg0N1oXDTIzMDkzMDE3MDg0N1qgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQCNJXC2mVKauEeN3LlQ3ZtM5gkH3ExH ++i4bmJjtJn497WwvvoIeUdrmVXgJQR93RtV37hZwN0SXMLlNmUZPH4rHhihayw4m +unCzVj/OhCCY7/TPjKuJ1O/0XhaLBpBVjQN7R/1ujoRKbSia/CD3vcn7Fqxzw7LK +fSRCKRGTj1CZiuxrphtFchwALXSiFDy9mr2ZKhImcyq1PydfgEzU78APpOkMQsIC +UNJ/cf3c9emzf+dUtcMEcejQ3mynBo4eIGg1EW42bz4q4hSjzQlKcBV0muw5qXhc +HOxH2iTFhQ7SrvVuK/dM14rYM4B5mSX3nRC1kNmXpS9j3wJDhuwmjHed +-----END X509 CRL----- diff --git a/src/greentest/2.7/selfsigned_pythontestdotnet.pem b/src/greentest/2.7/selfsigned_pythontestdotnet.pem new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/2.7/selfsigned_pythontestdotnet.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/sha256.pem b/src/greentest/2.7/sha256.pem new file mode 100644 index 0000000..d3db4b8 --- /dev/null +++ b/src/greentest/2.7/sha256.pem @@ -0,0 +1,128 @@ +# Certificate chain for https://sha256.tbs-internet.com + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC +-----BEGIN CERTIFICATE----- +MIIGXDCCBUSgAwIBAgIRAKpVmHgg9nfCodAVwcP4siwwDQYJKoZIhvcNAQELBQAw +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMB4XDTEyMDEwNDAwMDAwMFoXDTE0MDIxNzIzNTk1OVowgcsxCzAJBgNV +BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV +BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM +VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS +c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKQIX/zdJcyxty0m +PM1XQSoSSifueS3AVcgqMsaIKS/u+rYzsv4hQ/qA6vLn5m5/ewUcZDj7zdi6rBVf +PaVNXJ6YinLX0tkaW8TEjeVuZG5yksGZlhCt1CJ1Ho9XLiLaP4uJ7MCoNUntpJ+E +LfrOdgsIj91kPmwjDJeztVcQCvKzhjVJA/KxdInc0JvOATn7rpaSmQI5bvIjufgo +qVsTPwVFzuUYULXBk7KxRT7MiEqnd5HvviNh0285QC478zl3v0I0Fb5El4yD3p49 +IthcRnxzMKc0UhU5ogi0SbONyBfm/mzONVfSxpM+MlyvZmJqrbuuLoEDzJD+t8PU +xSuzgbcCAwEAAaOCAj4wggI6MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMB0GA1UdDgQWBBT/qTGYdaj+f61c2IRFL/B1eEsM8DAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG +CisGAQQBgjcKAwMGCWCGSAGG+EIEATBLBgNVHSAERDBCMEAGCisGAQQB5TcCBAEw +MjAwBggrBgEFBQcCARYkaHR0cHM6Ly93d3cudGJzLWludGVybmV0LmNvbS9DQS9D +UFM0MG0GA1UdHwRmMGQwMqAwoC6GLGh0dHA6Ly9jcmwudGJzLWludGVybmV0LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMC6gLKAqhihodHRwOi8vY3JsLnRicy14NTA5LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMIGmBggrBgEFBQcBAQSBmTCBljA4BggrBgEFBQcw +AoYsaHR0cDovL2NydC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQVNHQy5jcnQw +NAYIKwYBBQUHMAKGKGh0dHA6Ly9jcnQudGJzLXg1MDkuY29tL1RCU1g1MDlDQVNH +Qy5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLnRicy14NTA5LmNvbTA/BgNV +HREEODA2ghdzaGEyNTYudGJzLWludGVybmV0LmNvbYIbd3d3LnNoYTI1Ni50YnMt +aW50ZXJuZXQuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQA0pOuL8QvAa5yksTbGShzX +ABApagunUGoEydv4YJT1MXy9tTp7DrWaozZSlsqBxrYAXP1d9r2fuKbEniYHxaQ0 +UYaf1VSIlDo1yuC8wE7wxbHDIpQ/E5KAyxiaJ8obtDhFstWAPAH+UoGXq0kj2teN +21sFQ5dXgA95nldvVFsFhrRUNB6xXAcaj0VZFhttI0ZfQZmQwEI/P+N9Jr40OGun +aa+Dn0TMeUH4U20YntfLbu2nDcJcYfyurm+8/0Tr4HznLnedXu9pCPYj0TaddrgT +XO0oFiyy7qGaY6+qKh71yD64Y3ycCJ/HR9Wm39mjZYc9ezYwT4noP6r7Lk8YO7/q +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6 +rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0 +9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ +ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk +owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G +Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk +9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ +MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3 +AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk +ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k +by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw +cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV +VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B +ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN +AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232 +euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY +1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98 +RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz +8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV +v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E= +-----END CERTIFICATE----- + 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT +AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0 +ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB +IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05 +4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6 +2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh +alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv +u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW +xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p +XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd +tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB +BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX +BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov +L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN +AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO +rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd +FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM ++bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI +3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb ++M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g= +-----END CERTIFICATE----- + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG +EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD +VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu +dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 +E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ +D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK +4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq +lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW +bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB +o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT +MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js +LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr +BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB +AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft +Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj +j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH +KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv +2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 +mfnGV/TJVTl4uix5yaaIK/QI +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/ssl_cert.pem b/src/greentest/2.7/ssl_cert.pem new file mode 100644 index 0000000..47a7d7e --- /dev/null +++ b/src/greentest/2.7/ssl_cert.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/2.7/ssl_key.passwd.pem b/src/greentest/2.7/ssl_key.passwd.pem new file mode 100644 index 0000000..2524672 --- /dev/null +++ b/src/greentest/2.7/ssl_key.passwd.pem @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- diff --git a/src/greentest/2.7/ssl_key.pem b/src/greentest/2.7/ssl_key.pem new file mode 100644 index 0000000..3fd3bbd --- /dev/null +++ b/src/greentest/2.7/ssl_key.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- diff --git a/src/greentest/2.7/subprocessdata/sigchild_ignore.py b/src/greentest/2.7/subprocessdata/sigchild_ignore.py new file mode 100644 index 0000000..6072aec --- /dev/null +++ b/src/greentest/2.7/subprocessdata/sigchild_ignore.py @@ -0,0 +1,6 @@ +import signal, subprocess, sys +# On Linux this causes os.waitpid to fail with OSError as the OS has already +# reaped our child process. The wait() passing the OSError on to the caller +# and causing us to exit with an error is what we are testing against. +signal.signal(signal.SIGCHLD, signal.SIG_IGN) +subprocess.Popen([sys.executable, '-c', 'print("albatross")']).wait() diff --git a/src/greentest/2.7/test_asyncore.py b/src/greentest/2.7/test_asyncore.py new file mode 100644 index 0000000..20eceb6 --- /dev/null +++ b/src/greentest/2.7/test_asyncore.py @@ -0,0 +1,744 @@ +import asyncore +import unittest +import select +import os +import socket +import sys +import time +import warnings +import errno +import struct + +from test import test_support +from test.test_support import TESTFN, run_unittest, unlink, HOST +from StringIO import StringIO + +try: + import threading +except ImportError: + threading = None + + +class dummysocket: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def fileno(self): + return 42 + +class dummychannel: + def __init__(self): + self.socket = dummysocket() + + def close(self): + self.socket.close() + +class exitingdummy: + def __init__(self): + pass + + def handle_read_event(self): + raise asyncore.ExitNow() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + +class crashingdummy: + def __init__(self): + self.error_handled = False + + def handle_read_event(self): + raise Exception() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + + def handle_error(self): + self.error_handled = True + +# used when testing senders; just collects what it gets until newline is sent +def capture_server(evt, buf, serv): + try: + serv.listen(5) + conn, addr = serv.accept() + except socket.timeout: + pass + else: + n = 200 + while n > 0: + r, w, e = select.select([conn], [], []) + if r: + data = conn.recv(10) + # keep everything except for the newline terminator + buf.write(data.replace('\n', '')) + if '\n' in data: + break + n -= 1 + time.sleep(0.01) + + conn.close() + finally: + serv.close() + evt.set() + + +class HelperFunctionTests(unittest.TestCase): + def test_readwriteexc(self): + # Check exception handling behavior of read, write and _exception + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore read/write/_exception calls + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.read, tr1) + self.assertRaises(asyncore.ExitNow, asyncore.write, tr1) + self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + asyncore.read(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore.write(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore._exception(tr2) + self.assertEqual(tr2.error_handled, True) + + # asyncore.readwrite uses constants in the select module that + # are not present in Windows systems (see this thread: + # http://mail.python.org/pipermail/python-list/2001-October/109973.html) + # These constants should be present as long as poll is available + + @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') + def test_readwrite(self): + # Check that correct methods are called by readwrite() + + attributes = ('read', 'expt', 'write', 'closed', 'error_handled') + + expected = ( + (select.POLLIN, 'read'), + (select.POLLPRI, 'expt'), + (select.POLLOUT, 'write'), + (select.POLLERR, 'closed'), + (select.POLLHUP, 'closed'), + (select.POLLNVAL, 'closed'), + ) + + class testobj: + def __init__(self): + self.read = False + self.write = False + self.closed = False + self.expt = False + self.error_handled = False + + def handle_read_event(self): + self.read = True + + def handle_write_event(self): + self.write = True + + def handle_close(self): + self.closed = True + + def handle_expt_event(self): + self.expt = True + + def handle_error(self): + self.error_handled = True + + for flag, expectedattr in expected: + tobj = testobj() + self.assertEqual(getattr(tobj, expectedattr), False) + asyncore.readwrite(tobj, flag) + + # Only the attribute modified by the routine we expect to be + # called should be True. + for attr in attributes: + self.assertEqual(getattr(tobj, attr), attr==expectedattr) + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore readwrite call + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + self.assertEqual(tr2.error_handled, False) + asyncore.readwrite(tr2, flag) + self.assertEqual(tr2.error_handled, True) + + def test_closeall(self): + self.closeall_check(False) + + def test_closeall_default(self): + self.closeall_check(True) + + def closeall_check(self, usedefault): + # Check that close_all() closes everything in a given map + + l = [] + testmap = {} + for i in range(10): + c = dummychannel() + l.append(c) + self.assertEqual(c.socket.closed, False) + testmap[i] = c + + if usedefault: + socketmap = asyncore.socket_map + try: + asyncore.socket_map = testmap + asyncore.close_all() + finally: + testmap, asyncore.socket_map = asyncore.socket_map, socketmap + else: + asyncore.close_all(testmap) + + self.assertEqual(len(testmap), 0) + + for c in l: + self.assertEqual(c.socket.closed, True) + + def test_compact_traceback(self): + try: + raise Exception("I don't like spam!") + except: + real_t, real_v, real_tb = sys.exc_info() + r = asyncore.compact_traceback() + else: + self.fail("Expected exception") + + (f, function, line), t, v, info = r + self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py') + self.assertEqual(function, 'test_compact_traceback') + self.assertEqual(t, real_t) + self.assertEqual(v, real_v) + self.assertEqual(info, '[%s|%s|%s]' % (f, function, line)) + + +class DispatcherTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + def test_basic(self): + d = asyncore.dispatcher() + self.assertEqual(d.readable(), True) + self.assertEqual(d.writable(), True) + + def test_repr(self): + d = asyncore.dispatcher() + self.assertEqual(repr(d), '' % id(d)) + + def test_log(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log() (to stderr) + fp = StringIO() + stderr = sys.stderr + l1 = "Lovely spam! Wonderful spam!" + l2 = "I don't like spam!" + try: + sys.stderr = fp + d.log(l1) + d.log(l2) + finally: + sys.stderr = stderr + + lines = fp.getvalue().splitlines() + self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2]) + + def test_log_info(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log_info() (to stdout via print) + fp = StringIO() + stdout = sys.stdout + l1 = "Have you got anything without spam?" + l2 = "Why can't she have egg bacon spam and sausage?" + l3 = "THAT'S got spam in it!" + try: + sys.stdout = fp + d.log_info(l1, 'EGGS') + d.log_info(l2) + d.log_info(l3, 'SPAM') + finally: + sys.stdout = stdout + + lines = fp.getvalue().splitlines() + expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3] + + self.assertEqual(lines, expected) + + def test_unhandled(self): + d = asyncore.dispatcher() + d.ignore_log_types = () + + # capture output of dispatcher.log_info() (to stdout via print) + fp = StringIO() + stdout = sys.stdout + try: + sys.stdout = fp + d.handle_expt() + d.handle_read() + d.handle_write() + d.handle_connect() + d.handle_accept() + finally: + sys.stdout = stdout + + lines = fp.getvalue().splitlines() + expected = ['warning: unhandled incoming priority event', + 'warning: unhandled read event', + 'warning: unhandled write event', + 'warning: unhandled connect event', + 'warning: unhandled accept event'] + self.assertEqual(lines, expected) + + def test_issue_8594(self): + # XXX - this test is supposed to be removed in next major Python + # version + d = asyncore.dispatcher(socket.socket()) + # make sure the error message no longer refers to the socket + # object but the dispatcher instance instead + self.assertRaisesRegexp(AttributeError, 'dispatcher instance', + getattr, d, 'foo') + # cheap inheritance with the underlying socket is supposed + # to still work but a DeprecationWarning is expected + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + family = d.family + self.assertEqual(family, socket.AF_INET) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[0].category, DeprecationWarning)) + + def test_strerror(self): + # refers to bug #8573 + err = asyncore._strerror(errno.EPERM) + if hasattr(os, 'strerror'): + self.assertEqual(err, os.strerror(errno.EPERM)) + err = asyncore._strerror(-1) + self.assertTrue(err != "") + + +class dispatcherwithsend_noread(asyncore.dispatcher_with_send): + def readable(self): + return False + + def handle_connect(self): + pass + +class DispatcherWithSendTests(unittest.TestCase): + usepoll = False + + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + @unittest.skipUnless(threading, 'Threading required for this test.') + @test_support.reap_threads + def test_send(self): + evt = threading.Event() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + port = test_support.bind_port(sock) + + cap = StringIO() + args = (evt, cap, sock) + t = threading.Thread(target=capture_server, args=args) + t.start() + try: + # wait a little longer for the server to initialize (it sometimes + # refuses connections on slow machines without this wait) + time.sleep(0.2) + + data = "Suppose there isn't a 16-ton weight?" + d = dispatcherwithsend_noread() + d.create_socket(socket.AF_INET, socket.SOCK_STREAM) + d.connect((HOST, port)) + + # give time for socket to connect + time.sleep(0.1) + + d.send(data) + d.send(data) + d.send('\n') + + n = 1000 + while d.out_buffer and n > 0: + asyncore.poll() + n -= 1 + + evt.wait() + + self.assertEqual(cap.getvalue(), data*2) + finally: + t.join() + + +class DispatcherWithSendTests_UsePoll(DispatcherWithSendTests): + usepoll = True + +@unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), + 'asyncore.file_wrapper required') +class FileWrapperTest(unittest.TestCase): + def setUp(self): + self.d = "It's not dead, it's sleeping!" + with file(TESTFN, 'w') as h: + h.write(self.d) + + def tearDown(self): + unlink(TESTFN) + + def test_recv(self): + fd = os.open(TESTFN, os.O_RDONLY) + w = asyncore.file_wrapper(fd) + os.close(fd) + + self.assertNotEqual(w.fd, fd) + self.assertNotEqual(w.fileno(), fd) + self.assertEqual(w.recv(13), "It's not dead") + self.assertEqual(w.read(6), ", it's") + w.close() + self.assertRaises(OSError, w.read, 1) + + + def test_send(self): + d1 = "Come again?" + d2 = "I want to buy some cheese." + fd = os.open(TESTFN, os.O_WRONLY | os.O_APPEND) + w = asyncore.file_wrapper(fd) + os.close(fd) + + w.write(d1) + w.send(d2) + w.close() + self.assertEqual(file(TESTFN).read(), self.d + d1 + d2) + + @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'), + 'asyncore.file_dispatcher required') + def test_dispatcher(self): + fd = os.open(TESTFN, os.O_RDONLY) + data = [] + class FileDispatcher(asyncore.file_dispatcher): + def handle_read(self): + data.append(self.recv(29)) + s = FileDispatcher(fd) + os.close(fd) + asyncore.loop(timeout=0.01, use_poll=True, count=2) + self.assertEqual(b"".join(data), self.d) + + +class BaseTestHandler(asyncore.dispatcher): + + def __init__(self, sock=None): + asyncore.dispatcher.__init__(self, sock) + self.flag = False + + def handle_accept(self): + raise Exception("handle_accept not supposed to be called") + + def handle_connect(self): + raise Exception("handle_connect not supposed to be called") + + def handle_expt(self): + raise Exception("handle_expt not supposed to be called") + + def handle_close(self): + raise Exception("handle_close not supposed to be called") + + def handle_error(self): + raise + + +class TCPServer(asyncore.dispatcher): + """A server which listens on an address and dispatches the + connection to a handler. + """ + + def __init__(self, handler=BaseTestHandler, host=HOST, port=0): + asyncore.dispatcher.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.set_reuse_addr() + self.bind((host, port)) + self.listen(5) + self.handler = handler + + @property + def address(self): + return self.socket.getsockname()[:2] + + def handle_accept(self): + pair = self.accept() + if pair is not None: + self.handler(pair[0]) + + def handle_error(self): + raise + + +class BaseClient(BaseTestHandler): + + def __init__(self, address): + BaseTestHandler.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.connect(address) + + def handle_connect(self): + pass + + +class BaseTestAPI(unittest.TestCase): + + def tearDown(self): + asyncore.close_all() + + def loop_waiting_for_flag(self, instance, timeout=5): + timeout = float(timeout) / 100 + count = 100 + while asyncore.socket_map and count > 0: + asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) + if instance.flag: + return + count -= 1 + time.sleep(timeout) + self.fail("flag not set") + + def test_handle_connect(self): + # make sure handle_connect is called on connect() + + class TestClient(BaseClient): + def handle_connect(self): + self.flag = True + + server = TCPServer() + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_handle_accept(self): + # make sure handle_accept() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self): + BaseTestHandler.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.bind((HOST, 0)) + self.listen(5) + self.address = self.socket.getsockname()[:2] + + def handle_accept(self): + self.flag = True + + server = TestListener() + client = BaseClient(server.address) + self.loop_waiting_for_flag(server) + + def test_handle_read(self): + # make sure handle_read is called on data received + + class TestClient(BaseClient): + def handle_read(self): + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.send('x' * 1024) + + server = TCPServer(TestHandler) + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_handle_write(self): + # make sure handle_write is called + + class TestClient(BaseClient): + def handle_write(self): + self.flag = True + + server = TCPServer() + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close(self): + # make sure handle_close is called when the other end closes + # the connection + + class TestClient(BaseClient): + + def handle_read(self): + # in order to make handle_close be called we are supposed + # to make at least one recv() call + self.recv(1024) + + def handle_close(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.close() + + server = TCPServer(TestHandler) + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + @unittest.skipIf(sys.platform.startswith("sunos"), + "OOB support is broken on Solaris") + def test_handle_expt(self): + # Make sure handle_expt is called on OOB data received. + # Note: this might fail on some platforms as OOB data is + # tenuously supported and rarely used. + + class TestClient(BaseClient): + def handle_expt(self): + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.socket.send(chr(244), socket.MSG_OOB) + + server = TCPServer(TestHandler) + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_handle_error(self): + + class TestClient(BaseClient): + def handle_write(self): + 1.0 / 0 + def handle_error(self): + self.flag = True + try: + raise + except ZeroDivisionError: + pass + else: + raise Exception("exception not raised") + + server = TCPServer() + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_connection_attributes(self): + server = TCPServer() + client = BaseClient(server.address) + + # we start disconnected + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + # this can't be taken for granted across all platforms + #self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # execute some loops so that client connects to server + asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100) + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertTrue(client.connected) + self.assertFalse(client.accepting) + + # disconnect the client + client.close() + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # stop serving + server.close() + self.assertFalse(server.connected) + self.assertFalse(server.accepting) + + def test_create_socket(self): + s = asyncore.dispatcher() + s.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.assertEqual(s.socket.family, socket.AF_INET) + self.assertEqual(s.socket.type, socket.SOCK_STREAM) + + def test_bind(self): + s1 = asyncore.dispatcher() + s1.create_socket(socket.AF_INET, socket.SOCK_STREAM) + s1.bind((HOST, 0)) + s1.listen(5) + port = s1.socket.getsockname()[1] + + s2 = asyncore.dispatcher() + s2.create_socket(socket.AF_INET, socket.SOCK_STREAM) + # EADDRINUSE indicates the socket was correctly bound + self.assertRaises(socket.error, s2.bind, (HOST, port)) + + def test_set_reuse_addr(self): + sock = socket.socket() + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except socket.error: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket()) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.create_socket(socket.AF_INET, socket.SOCK_STREAM) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + finally: + sock.close() + + @unittest.skipUnless(threading, 'Threading required for this test.') + @test_support.reap_threads + def test_quick_connect(self): + # see: http://bugs.python.org/issue10340 + server = TCPServer() + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, count=500)) + t.start() + self.addCleanup(t.join) + + for x in xrange(20): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + try: + s.connect(server.address) + except socket.error: + pass + finally: + s.close() + + +class TestAPI_UseSelect(BaseTestAPI): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UsePoll(BaseTestAPI): + use_poll = True + + +def test_main(): + tests = [HelperFunctionTests, DispatcherTests, DispatcherWithSendTests, + DispatcherWithSendTests_UsePoll, TestAPI_UseSelect, + TestAPI_UsePoll, FileWrapperTest] + run_unittest(*tests) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_ftplib.py b/src/greentest/2.7/test_ftplib.py new file mode 100644 index 0000000..cc1c19b --- /dev/null +++ b/src/greentest/2.7/test_ftplib.py @@ -0,0 +1,865 @@ +"""Test script for ftplib module.""" + +# Modified by Giampaolo Rodola' to test FTP class, IPv6 and TLS +# environment + +import ftplib +import asyncore +import asynchat +import socket +import StringIO +import errno +import os +try: + import ssl +except ImportError: + ssl = None + +from unittest import TestCase, SkipTest, skipUnless +from test import test_support +from test.test_support import HOST, HOSTv6 +threading = test_support.import_module('threading') + +TIMEOUT = 3 +# the dummy data returned by server over the data channel when +# RETR, LIST and NLST commands are issued +RETR_DATA = 'abcde12345\r\n' * 1000 +LIST_DATA = 'foo\r\nbar\r\n' +NLST_DATA = 'foo\r\nbar\r\n' + + +class DummyDTPHandler(asynchat.async_chat): + dtp_conn_closed = False + + def __init__(self, conn, baseclass): + asynchat.async_chat.__init__(self, conn) + self.baseclass = baseclass + self.baseclass.last_received_data = '' + + def handle_read(self): + self.baseclass.last_received_data += self.recv(1024) + + def handle_close(self): + # XXX: this method can be called many times in a row for a single + # connection, including in clear-text (non-TLS) mode. + # (behaviour witnessed with test_data_connection) + if not self.dtp_conn_closed: + self.baseclass.push('226 transfer complete') + self.close() + self.dtp_conn_closed = True + + def handle_error(self): + raise + + +class DummyFTPHandler(asynchat.async_chat): + + dtp_handler = DummyDTPHandler + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + self.set_terminator("\r\n") + self.in_buffer = [] + self.dtp = None + self.last_received_cmd = None + self.last_received_data = '' + self.next_response = '' + self.rest = None + self.next_retr_data = RETR_DATA + self.push('220 welcome') + + def collect_incoming_data(self, data): + self.in_buffer.append(data) + + def found_terminator(self): + line = ''.join(self.in_buffer) + self.in_buffer = [] + if self.next_response: + self.push(self.next_response) + self.next_response = '' + cmd = line.split(' ')[0].lower() + self.last_received_cmd = cmd + space = line.find(' ') + if space != -1: + arg = line[space + 1:] + else: + arg = "" + if hasattr(self, 'cmd_' + cmd): + method = getattr(self, 'cmd_' + cmd) + method(arg) + else: + self.push('550 command "%s" not understood.' %cmd) + + def handle_error(self): + raise + + def push(self, data): + asynchat.async_chat.push(self, data + '\r\n') + + def cmd_port(self, arg): + addr = map(int, arg.split(',')) + ip = '%d.%d.%d.%d' %tuple(addr[:4]) + port = (addr[4] * 256) + addr[5] + s = socket.create_connection((ip, port), timeout=10) + self.dtp = self.dtp_handler(s, baseclass=self) + self.push('200 active data connection established') + + def cmd_pasv(self, arg): + sock = socket.socket() + sock.bind((self.socket.getsockname()[0], 0)) + sock.listen(5) + sock.settimeout(10) + ip, port = sock.getsockname()[:2] + ip = ip.replace('.', ',') + p1, p2 = divmod(port, 256) + self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2)) + conn, addr = sock.accept() + self.dtp = self.dtp_handler(conn, baseclass=self) + + def cmd_eprt(self, arg): + af, ip, port = arg.split(arg[0])[1:-1] + port = int(port) + s = socket.create_connection((ip, port), timeout=10) + self.dtp = self.dtp_handler(s, baseclass=self) + self.push('200 active data connection established') + + def cmd_epsv(self, arg): + sock = socket.socket(socket.AF_INET6) + sock.bind((self.socket.getsockname()[0], 0)) + sock.listen(5) + sock.settimeout(10) + port = sock.getsockname()[1] + self.push('229 entering extended passive mode (|||%d|)' %port) + conn, addr = sock.accept() + self.dtp = self.dtp_handler(conn, baseclass=self) + + def cmd_echo(self, arg): + # sends back the received string (used by the test suite) + self.push(arg) + + def cmd_user(self, arg): + self.push('331 username ok') + + def cmd_pass(self, arg): + self.push('230 password ok') + + def cmd_acct(self, arg): + self.push('230 acct ok') + + def cmd_rnfr(self, arg): + self.push('350 rnfr ok') + + def cmd_rnto(self, arg): + self.push('250 rnto ok') + + def cmd_dele(self, arg): + self.push('250 dele ok') + + def cmd_cwd(self, arg): + self.push('250 cwd ok') + + def cmd_size(self, arg): + self.push('250 1000') + + def cmd_mkd(self, arg): + self.push('257 "%s"' %arg) + + def cmd_rmd(self, arg): + self.push('250 rmd ok') + + def cmd_pwd(self, arg): + self.push('257 "pwd ok"') + + def cmd_type(self, arg): + self.push('200 type ok') + + def cmd_quit(self, arg): + self.push('221 quit ok') + self.close() + + def cmd_stor(self, arg): + self.push('125 stor ok') + + def cmd_rest(self, arg): + self.rest = arg + self.push('350 rest ok') + + def cmd_retr(self, arg): + self.push('125 retr ok') + if self.rest is not None: + offset = int(self.rest) + else: + offset = 0 + self.dtp.push(self.next_retr_data[offset:]) + self.dtp.close_when_done() + self.rest = None + + def cmd_list(self, arg): + self.push('125 list ok') + self.dtp.push(LIST_DATA) + self.dtp.close_when_done() + + def cmd_nlst(self, arg): + self.push('125 nlst ok') + self.dtp.push(NLST_DATA) + self.dtp.close_when_done() + + def cmd_setlongretr(self, arg): + # For testing. Next RETR will return long line. + self.next_retr_data = 'x' * int(arg) + self.push('125 setlongretr ok') + + +class DummyFTPServer(asyncore.dispatcher, threading.Thread): + + handler = DummyFTPHandler + + def __init__(self, address, af=socket.AF_INET): + threading.Thread.__init__(self) + asyncore.dispatcher.__init__(self) + self.create_socket(af, socket.SOCK_STREAM) + self.bind(address) + self.listen(5) + self.active = False + self.active_lock = threading.Lock() + self.host, self.port = self.socket.getsockname()[:2] + self.handler_instance = None + + def start(self): + assert not self.active + self.__flag = threading.Event() + threading.Thread.start(self) + self.__flag.wait() + + def run(self): + self.active = True + self.__flag.set() + while self.active and asyncore.socket_map: + self.active_lock.acquire() + asyncore.loop(timeout=0.1, count=1) + self.active_lock.release() + asyncore.close_all(ignore_all=True) + + def stop(self): + assert self.active + self.active = False + self.join() + + def handle_accept(self): + conn, addr = self.accept() + self.handler_instance = self.handler(conn) + + def handle_connect(self): + self.close() + handle_read = handle_connect + + def writable(self): + return 0 + + def handle_error(self): + raise + + +if ssl is not None: + + CERTFILE = os.path.join(os.path.dirname(__file__), "keycert3.pem") + CAFILE = os.path.join(os.path.dirname(__file__), "pycacert.pem") + + class SSLConnection(object, asyncore.dispatcher): + """An asyncore.dispatcher subclass supporting TLS/SSL.""" + + _ssl_accepting = False + _ssl_closing = False + + def secure_connection(self): + socket = ssl.wrap_socket(self.socket, suppress_ragged_eofs=False, + certfile=CERTFILE, server_side=True, + do_handshake_on_connect=False, + ssl_version=ssl.PROTOCOL_SSLv23) + self.del_channel() + self.set_socket(socket) + self._ssl_accepting = True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + elif err.args[0] == ssl.SSL_ERROR_EOF: + return self.handle_close() + raise + except socket.error as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def _do_ssl_shutdown(self): + self._ssl_closing = True + try: + self.socket = self.socket.unwrap() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + except socket.error as err: + # Any "socket error" corresponds to a SSL_ERROR_SYSCALL return + # from OpenSSL's SSL_shutdown(), corresponding to a + # closed socket condition. See also: + # http://www.mail-archive.com/openssl-users@openssl.org/msg60710.html + pass + self._ssl_closing = False + if getattr(self, '_ccc', False) is False: + super(SSLConnection, self).close() + else: + pass + + def handle_read_event(self): + if self._ssl_accepting: + self._do_ssl_handshake() + elif self._ssl_closing: + self._do_ssl_shutdown() + else: + super(SSLConnection, self).handle_read_event() + + def handle_write_event(self): + if self._ssl_accepting: + self._do_ssl_handshake() + elif self._ssl_closing: + self._do_ssl_shutdown() + else: + super(SSLConnection, self).handle_write_event() + + def send(self, data): + try: + return super(SSLConnection, self).send(data) + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN, + ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return 0 + raise + + def recv(self, buffer_size): + try: + return super(SSLConnection, self).recv(buffer_size) + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return b'' + if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN): + self.handle_close() + return b'' + raise + + def handle_error(self): + raise + + def close(self): + if (isinstance(self.socket, ssl.SSLSocket) and + self.socket._sslobj is not None): + self._do_ssl_shutdown() + else: + super(SSLConnection, self).close() + + + class DummyTLS_DTPHandler(SSLConnection, DummyDTPHandler): + """A DummyDTPHandler subclass supporting TLS/SSL.""" + + def __init__(self, conn, baseclass): + DummyDTPHandler.__init__(self, conn, baseclass) + if self.baseclass.secure_data_channel: + self.secure_connection() + + + class DummyTLS_FTPHandler(SSLConnection, DummyFTPHandler): + """A DummyFTPHandler subclass supporting TLS/SSL.""" + + dtp_handler = DummyTLS_DTPHandler + + def __init__(self, conn): + DummyFTPHandler.__init__(self, conn) + self.secure_data_channel = False + + def cmd_auth(self, line): + """Set up secure control channel.""" + self.push('234 AUTH TLS successful') + self.secure_connection() + + def cmd_pbsz(self, line): + """Negotiate size of buffer for secure data transfer. + For TLS/SSL the only valid value for the parameter is '0'. + Any other value is accepted but ignored. + """ + self.push('200 PBSZ=0 successful.') + + def cmd_prot(self, line): + """Setup un/secure data channel.""" + arg = line.upper() + if arg == 'C': + self.push('200 Protection set to Clear') + self.secure_data_channel = False + elif arg == 'P': + self.push('200 Protection set to Private') + self.secure_data_channel = True + else: + self.push("502 Unrecognized PROT type (use C or P).") + + + class DummyTLS_FTPServer(DummyFTPServer): + handler = DummyTLS_FTPHandler + + +class TestFTPClass(TestCase): + + def setUp(self): + self.server = DummyFTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP(timeout=10) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + + def test_getwelcome(self): + self.assertEqual(self.client.getwelcome(), '220 welcome') + + def test_sanitize(self): + self.assertEqual(self.client.sanitize('foo'), repr('foo')) + self.assertEqual(self.client.sanitize('pass 12345'), repr('pass *****')) + self.assertEqual(self.client.sanitize('PASS 12345'), repr('PASS *****')) + + def test_exceptions(self): + self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 400') + self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 499') + self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 500') + self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 599') + self.assertRaises(ftplib.error_proto, self.client.sendcmd, 'echo 999') + + def test_all_errors(self): + exceptions = (ftplib.error_reply, ftplib.error_temp, ftplib.error_perm, + ftplib.error_proto, ftplib.Error, IOError, EOFError) + for x in exceptions: + try: + raise x('exception not included in all_errors set') + except ftplib.all_errors: + pass + + def test_set_pasv(self): + # passive mode is supposed to be enabled by default + self.assertTrue(self.client.passiveserver) + self.client.set_pasv(True) + self.assertTrue(self.client.passiveserver) + self.client.set_pasv(False) + self.assertFalse(self.client.passiveserver) + + def test_voidcmd(self): + self.client.voidcmd('echo 200') + self.client.voidcmd('echo 299') + self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') + self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') + + def test_login(self): + self.client.login() + + def test_acct(self): + self.client.acct('passwd') + + def test_rename(self): + self.client.rename('a', 'b') + self.server.handler_instance.next_response = '200' + self.assertRaises(ftplib.error_reply, self.client.rename, 'a', 'b') + + def test_delete(self): + self.client.delete('foo') + self.server.handler_instance.next_response = '199' + self.assertRaises(ftplib.error_reply, self.client.delete, 'foo') + + def test_size(self): + self.client.size('foo') + + def test_mkd(self): + dir = self.client.mkd('/foo') + self.assertEqual(dir, '/foo') + + def test_rmd(self): + self.client.rmd('foo') + + def test_cwd(self): + dir = self.client.cwd('/foo') + self.assertEqual(dir, '250 cwd ok') + + def test_pwd(self): + dir = self.client.pwd() + self.assertEqual(dir, 'pwd ok') + + def test_quit(self): + self.assertEqual(self.client.quit(), '221 quit ok') + # Ensure the connection gets closed; sock attribute should be None + self.assertEqual(self.client.sock, None) + + def test_retrbinary(self): + received = [] + self.client.retrbinary('retr', received.append) + self.assertEqual(''.join(received), RETR_DATA) + + def test_retrbinary_rest(self): + for rest in (0, 10, 20): + received = [] + self.client.retrbinary('retr', received.append, rest=rest) + self.assertEqual(''.join(received), RETR_DATA[rest:], + msg='rest test case %d %d %d' % (rest, + len(''.join(received)), + len(RETR_DATA[rest:]))) + + def test_retrlines(self): + received = [] + self.client.retrlines('retr', received.append) + self.assertEqual(''.join(received), RETR_DATA.replace('\r\n', '')) + + def test_storbinary(self): + f = StringIO.StringIO(RETR_DATA) + self.client.storbinary('stor', f) + self.assertEqual(self.server.handler_instance.last_received_data, RETR_DATA) + # test new callback arg + flag = [] + f.seek(0) + self.client.storbinary('stor', f, callback=lambda x: flag.append(None)) + self.assertTrue(flag) + + def test_storbinary_rest(self): + f = StringIO.StringIO(RETR_DATA) + for r in (30, '30'): + f.seek(0) + self.client.storbinary('stor', f, rest=r) + self.assertEqual(self.server.handler_instance.rest, str(r)) + + def test_storlines(self): + f = StringIO.StringIO(RETR_DATA.replace('\r\n', '\n')) + self.client.storlines('stor', f) + self.assertEqual(self.server.handler_instance.last_received_data, RETR_DATA) + # test new callback arg + flag = [] + f.seek(0) + self.client.storlines('stor foo', f, callback=lambda x: flag.append(None)) + self.assertTrue(flag) + + def test_nlst(self): + self.client.nlst() + self.assertEqual(self.client.nlst(), NLST_DATA.split('\r\n')[:-1]) + + def test_dir(self): + l = [] + self.client.dir(lambda x: l.append(x)) + self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) + + def test_makeport(self): + self.client.makeport() + # IPv4 is in use, just make sure send_eprt has not been used + self.assertEqual(self.server.handler_instance.last_received_cmd, 'port') + + def test_makepasv(self): + host, port = self.client.makepasv() + conn = socket.create_connection((host, port), 10) + conn.close() + # IPv4 is in use, just make sure send_epsv has not been used + self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv') + + def test_line_too_long(self): + self.assertRaises(ftplib.Error, self.client.sendcmd, + 'x' * self.client.maxline * 2) + + def test_retrlines_too_long(self): + self.client.sendcmd('SETLONGRETR %d' % (self.client.maxline * 2)) + received = [] + self.assertRaises(ftplib.Error, + self.client.retrlines, 'retr', received.append) + + def test_storlines_too_long(self): + f = StringIO.StringIO('x' * self.client.maxline * 2) + self.assertRaises(ftplib.Error, self.client.storlines, 'stor', f) + + +@skipUnless(socket.has_ipv6, "IPv6 not enabled") +class TestIPv6Environment(TestCase): + + @classmethod + def setUpClass(cls): + try: + DummyFTPServer((HOST, 0), af=socket.AF_INET6) + except socket.error: + raise SkipTest("IPv6 not enabled") + + def setUp(self): + self.server = DummyFTPServer((HOSTv6, 0), af=socket.AF_INET6) + self.server.start() + self.client = ftplib.FTP() + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + + def test_af(self): + self.assertEqual(self.client.af, socket.AF_INET6) + + def test_makeport(self): + self.client.makeport() + self.assertEqual(self.server.handler_instance.last_received_cmd, 'eprt') + + def test_makepasv(self): + host, port = self.client.makepasv() + conn = socket.create_connection((host, port), 10) + conn.close() + self.assertEqual(self.server.handler_instance.last_received_cmd, 'epsv') + + def test_transfer(self): + def retr(): + received = [] + self.client.retrbinary('retr', received.append) + self.assertEqual(''.join(received), RETR_DATA) + self.client.set_pasv(True) + retr() + self.client.set_pasv(False) + retr() + + +@skipUnless(ssl, "SSL not available") +class TestTLS_FTPClassMixin(TestFTPClass): + """Repeat TestFTPClass tests starting the TLS layer for both control + and data connections first. + """ + + def setUp(self): + self.server = DummyTLS_FTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP_TLS(timeout=10) + self.client.connect(self.server.host, self.server.port) + # enable TLS + self.client.auth() + self.client.prot_p() + + +@skipUnless(ssl, "SSL not available") +class TestTLS_FTPClass(TestCase): + """Specific TLS_FTP class tests.""" + + def setUp(self): + self.server = DummyTLS_FTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP_TLS(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + + def test_control_connection(self): + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.auth() + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + + def test_data_connection(self): + # clear text + sock = self.client.transfercmd('list') + self.assertNotIsInstance(sock, ssl.SSLSocket) + sock.close() + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + # secured, after PROT P + self.client.prot_p() + sock = self.client.transfercmd('list') + self.assertIsInstance(sock, ssl.SSLSocket) + sock.close() + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + # PROT C is issued, the connection must be in cleartext again + self.client.prot_c() + sock = self.client.transfercmd('list') + self.assertNotIsInstance(sock, ssl.SSLSocket) + sock.close() + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + def test_login(self): + # login() is supposed to implicitly secure the control connection + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.login() + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + # make sure that AUTH TLS doesn't get issued again + self.client.login() + + def test_auth_issued_twice(self): + self.client.auth() + self.assertRaises(ValueError, self.client.auth) + + def test_auth_ssl(self): + try: + self.client.ssl_version = ssl.PROTOCOL_SSLv23 + self.client.auth() + self.assertRaises(ValueError, self.client.auth) + finally: + self.client.ssl_version = ssl.PROTOCOL_TLSv1 + + def test_context(self): + self.client.quit() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE, + context=ctx) + self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, + context=ctx) + self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, + keyfile=CERTFILE, context=ctx) + + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.auth() + self.assertIs(self.client.sock.context, ctx) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + + self.client.prot_p() + sock = self.client.transfercmd('list') + try: + self.assertIs(sock.context, ctx) + self.assertIsInstance(sock, ssl.SSLSocket) + finally: + sock.close() + + def test_check_hostname(self): + self.client.quit() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.check_hostname = True + ctx.load_verify_locations(CAFILE) + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) + + # 127.0.0.1 doesn't match SAN + self.client.connect(self.server.host, self.server.port) + with self.assertRaises(ssl.CertificateError): + self.client.auth() + # exception quits connection + + self.client.connect(self.server.host, self.server.port) + self.client.prot_p() + with self.assertRaises(ssl.CertificateError): + self.client.transfercmd("list").close() + self.client.quit() + + self.client.connect("localhost", self.server.port) + self.client.auth() + self.client.quit() + + self.client.connect("localhost", self.server.port) + self.client.prot_p() + self.client.transfercmd("list").close() + + +class TestTimeouts(TestCase): + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(10) + self.port = test_support.bind_port(self.sock) + threading.Thread(target=self.server, args=(self.evt,self.sock)).start() + # Wait for the server to be ready. + self.evt.wait() + self.evt.clear() + ftplib.FTP.port = self.port + + def tearDown(self): + self.evt.wait() + + def server(self, evt, serv): + # This method sets the evt 3 times: + # 1) when the connection is ready to be accepted. + # 2) when it is safe for the caller to close the connection + # 3) when we have closed the socket + serv.listen(5) + # (1) Signal the caller that we are ready to accept the connection. + evt.set() + try: + conn, addr = serv.accept() + except socket.timeout: + pass + else: + conn.send("1 Hola mundo\n") + # (2) Signal the caller that it is safe to close the socket. + evt.set() + conn.close() + finally: + serv.close() + # (3) Signal the caller that we are done. + evt.set() + + def testTimeoutDefault(self): + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + ftp = ftplib.FTP(HOST) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutNone(self): + # no timeout -- do not use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + ftp = ftplib.FTP(HOST, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(ftp.sock.gettimeout()) + self.evt.wait() + ftp.close() + + def testTimeoutValue(self): + # a value + ftp = ftplib.FTP(HOST, timeout=30) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutConnect(self): + ftp = ftplib.FTP() + ftp.connect(HOST, timeout=30) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutDifferentOrder(self): + ftp = ftplib.FTP(timeout=30) + ftp.connect(HOST) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutDirectAccess(self): + ftp = ftplib.FTP() + ftp.timeout = 30 + ftp.connect(HOST) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + +def test_main(): + tests = [TestFTPClass, TestTimeouts, + TestIPv6Environment, + TestTLS_FTPClassMixin, TestTLS_FTPClass] + + thread_info = test_support.threading_setup() + try: + test_support.run_unittest(*tests) + finally: + test_support.threading_cleanup(*thread_info) + + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7/test_httplib.py b/src/greentest/2.7/test_httplib.py new file mode 100644 index 0000000..b81f1e6 --- /dev/null +++ b/src/greentest/2.7/test_httplib.py @@ -0,0 +1,880 @@ +import httplib +import itertools +import array +import StringIO +import socket +import errno +import os +import tempfile + +import unittest +TestCase = unittest.TestCase + +from test import test_support + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') +# Self-signed cert file for self-signed.pythontest.net +CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') + +HOST = test_support.HOST + +class FakeSocket: + def __init__(self, text, fileclass=StringIO.StringIO, host=None, port=None): + self.text = text + self.fileclass = fileclass + self.data = '' + self.file_closed = False + self.host = host + self.port = port + + def sendall(self, data): + self.data += ''.join(data) + + def makefile(self, mode, bufsize=None): + if mode != 'r' and mode != 'rb': + raise httplib.UnimplementedFileMode() + # keep the file around so we can check how much was read from it + self.file = self.fileclass(self.text) + self.file.close = self.file_close #nerf close () + return self.file + + def file_close(self): + self.file_closed = True + + def close(self): + pass + +class EPipeSocket(FakeSocket): + + def __init__(self, text, pipe_trigger): + # When sendall() is called with pipe_trigger, raise EPIPE. + FakeSocket.__init__(self, text) + self.pipe_trigger = pipe_trigger + + def sendall(self, data): + if self.pipe_trigger in data: + raise socket.error(errno.EPIPE, "gotcha") + self.data += data + + def close(self): + pass + +class NoEOFStringIO(StringIO.StringIO): + """Like StringIO, but raises AssertionError on EOF. + + This is used below to test that httplib doesn't try to read + more from the underlying file than it should. + """ + def read(self, n=-1): + data = StringIO.StringIO.read(self, n) + if data == '': + raise AssertionError('caller tried to read past EOF') + return data + + def readline(self, length=None): + data = StringIO.StringIO.readline(self, length) + if data == '': + raise AssertionError('caller tried to read past EOF') + return data + + +class HeaderTests(TestCase): + def test_auto_headers(self): + # Some headers are added automatically, but should not be added by + # .request() if they are explicitly set. + + class HeaderCountingBuffer(list): + def __init__(self): + self.count = {} + def append(self, item): + kv = item.split(':') + if len(kv) > 1: + # item is a 'Key: Value' header string + lcKey = kv[0].lower() + self.count.setdefault(lcKey, 0) + self.count[lcKey] += 1 + list.append(self, item) + + for explicit_header in True, False: + for header in 'Content-length', 'Host', 'Accept-encoding': + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket('blahblahblah') + conn._buffer = HeaderCountingBuffer() + + body = 'spamspamspam' + headers = {} + if explicit_header: + headers[header] = str(len(body)) + conn.request('POST', '/', body, headers) + self.assertEqual(conn._buffer.count[header.lower()], 1) + + def test_content_length_0(self): + + class ContentLengthChecker(list): + def __init__(self): + list.__init__(self) + self.content_length = None + def append(self, item): + kv = item.split(':', 1) + if len(kv) > 1 and kv[0].lower() == 'content-length': + self.content_length = kv[1].strip() + list.append(self, item) + + # Here, we're testing that methods expecting a body get a + # content-length set to zero if the body is empty (either None or '') + bodies = (None, '') + methods_with_body = ('PUT', 'POST', 'PATCH') + for method, body in itertools.product(methods_with_body, bodies): + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', body) + self.assertEqual( + conn._buffer.content_length, '0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # For these methods, we make sure that content-length is not set when + # the body is None because it might cause unexpected behaviour on the + # server. + methods_without_body = ( + 'GET', 'CONNECT', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', + ) + for method in methods_without_body: + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', None) + self.assertEqual( + conn._buffer.content_length, None, + 'Header Content-Length set for empty body on {}'.format(method) + ) + + # If the body is set to '', that's considered to be "present but + # empty" rather than "missing", so content length would be set, even + # for methods that don't expect a body. + for method in methods_without_body: + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', '') + self.assertEqual( + conn._buffer.content_length, '0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # If the body is set, make sure Content-Length is set. + for method in itertools.chain(methods_without_body, methods_with_body): + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', ' ') + self.assertEqual( + conn._buffer.content_length, '1', + 'Header Content-Length incorrect on {}'.format(method) + ) + + def test_putheader(self): + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.putrequest('GET','/') + conn.putheader('Content-length',42) + self.assertIn('Content-length: 42', conn._buffer) + + conn.putheader('Foo', ' bar ') + self.assertIn(b'Foo: bar ', conn._buffer) + conn.putheader('Bar', '\tbaz\t') + self.assertIn(b'Bar: \tbaz\t', conn._buffer) + conn.putheader('Authorization', 'Bearer mytoken') + self.assertIn(b'Authorization: Bearer mytoken', conn._buffer) + conn.putheader('IterHeader', 'IterA', 'IterB') + self.assertIn(b'IterHeader: IterA\r\n\tIterB', conn._buffer) + conn.putheader('LatinHeader', b'\xFF') + self.assertIn(b'LatinHeader: \xFF', conn._buffer) + conn.putheader('Utf8Header', b'\xc3\x80') + self.assertIn(b'Utf8Header: \xc3\x80', conn._buffer) + conn.putheader('C1-Control', b'next\x85line') + self.assertIn(b'C1-Control: next\x85line', conn._buffer) + conn.putheader('Embedded-Fold-Space', 'is\r\n allowed') + self.assertIn(b'Embedded-Fold-Space: is\r\n allowed', conn._buffer) + conn.putheader('Embedded-Fold-Tab', 'is\r\n\tallowed') + self.assertIn(b'Embedded-Fold-Tab: is\r\n\tallowed', conn._buffer) + conn.putheader('Key Space', 'value') + self.assertIn(b'Key Space: value', conn._buffer) + conn.putheader('KeySpace ', 'value') + self.assertIn(b'KeySpace : value', conn._buffer) + conn.putheader(b'Nonbreak\xa0Space', 'value') + self.assertIn(b'Nonbreak\xa0Space: value', conn._buffer) + conn.putheader(b'\xa0NonbreakSpace', 'value') + self.assertIn(b'\xa0NonbreakSpace: value', conn._buffer) + + def test_ipv6host_header(self): + # Default host header on IPv6 transaction should wrapped by [] if + # its actual IPv6 address + expected = 'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \ + 'Accept-Encoding: identity\r\n\r\n' + conn = httplib.HTTPConnection('[2001::]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + expected = 'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ + 'Accept-Encoding: identity\r\n\r\n' + conn = httplib.HTTPConnection('[2001:102A::]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + def test_malformed_headers_coped_with(self): + # Issue 19996 + body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + + self.assertEqual(resp.getheader('First'), 'val') + self.assertEqual(resp.getheader('Second'), 'val') + + def test_invalid_headers(self): + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/') + + # http://tools.ietf.org/html/rfc7230#section-3.2.4, whitespace is no + # longer allowed in header names + cases = ( + (b'Invalid\r\nName', b'ValidValue'), + (b'Invalid\rName', b'ValidValue'), + (b'Invalid\nName', b'ValidValue'), + (b'\r\nInvalidName', b'ValidValue'), + (b'\rInvalidName', b'ValidValue'), + (b'\nInvalidName', b'ValidValue'), + (b' InvalidName', b'ValidValue'), + (b'\tInvalidName', b'ValidValue'), + (b'Invalid:Name', b'ValidValue'), + (b':InvalidName', b'ValidValue'), + (b'ValidName', b'Invalid\r\nValue'), + (b'ValidName', b'Invalid\rValue'), + (b'ValidName', b'Invalid\nValue'), + (b'ValidName', b'InvalidValue\r\n'), + (b'ValidName', b'InvalidValue\r'), + (b'ValidName', b'InvalidValue\n'), + ) + for name, value in cases: + with self.assertRaisesRegexp(ValueError, 'Invalid header'): + conn.putheader(name, value) + + +class BasicTest(TestCase): + def test_status_lines(self): + # Test HTTP status lines + + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(0), '') # Issue #20007 + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(), 'Text') + self.assertTrue(resp.isclosed()) + + body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + self.assertRaises(httplib.BadStatusLine, resp.begin) + + def test_bad_status_repr(self): + exc = httplib.BadStatusLine('') + self.assertEqual(repr(exc), '''BadStatusLine("\'\'",)''') + + def test_partial_reads(self): + # if we have a length, the system knows when to close itself + # same behaviour than when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), 'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), 'xt') + self.assertTrue(resp.isclosed()) + + def test_partial_reads_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), 'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), 'xt') + self.assertEqual(resp.read(1), '') + self.assertTrue(resp.isclosed()) + + def test_partial_reads_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), 'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), 'xt') + self.assertEqual(resp.read(1), '') + self.assertTrue(resp.isclosed()) + + def test_host_port(self): + # Check invalid host_port + + # Note that httplib does not accept user:password@ in the host-port. + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(httplib.InvalidURL, httplib.HTTP, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", "fe80::207:e9ff:fe9b", + 8000), + ("www.python.org:80", "www.python.org", 80), + ("www.python.org", "www.python.org", 80), + ("www.python.org:", "www.python.org", 80), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80)): + http = httplib.HTTP(hp) + c = http._conn + if h != c.host: + self.fail("Host incorrectly parsed: %s != %s" % (h, c.host)) + if p != c.port: + self.fail("Port incorrectly parsed: %s != %s" % (p, c.host)) + + def test_response_headers(self): + # test response with multiple message headers with the same field name. + text = ('HTTP/1.1 200 OK\r\n' + 'Set-Cookie: Customer="WILE_E_COYOTE";' + ' Version="1"; Path="/acme"\r\n' + 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' + ' Path="/acme"\r\n' + '\r\n' + 'No body\r\n') + hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"' + ', ' + 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"') + s = FakeSocket(text) + r = httplib.HTTPResponse(s) + r.begin() + cookies = r.getheader("Set-Cookie") + if cookies != hdr: + self.fail("multiple headers not combined properly") + + def test_read_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFStringIO) + resp = httplib.HTTPResponse(sock, method="HEAD") + resp.begin() + if resp.read() != "": + self.fail("Did not expect response from HEAD request") + + def test_too_many_headers(self): + headers = '\r\n'.join('Header%d: foo' % i for i in xrange(200)) + '\r\n' + text = ('HTTP/1.1 200 OK\r\n' + headers) + s = FakeSocket(text) + r = httplib.HTTPResponse(s) + self.assertRaises(httplib.HTTPException, r.begin) + + def test_send_file(self): + expected = 'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + 'Accept-Encoding: identity\r\nContent-Length:' + + body = open(__file__, 'rb') + conn = httplib.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertTrue(sock.data.startswith(expected)) + self.assertIn('def test_send_file', sock.data) + + def test_send_tempfile(self): + expected = ('GET /foo HTTP/1.1\r\nHost: example.com\r\n' + 'Accept-Encoding: identity\r\nContent-Length: 9\r\n\r\n' + 'fake\ndata') + + with tempfile.TemporaryFile() as body: + body.write('fake\ndata') + body.seek(0) + + conn = httplib.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertEqual(sock.data, expected) + + def test_send(self): + expected = 'this is a test this is only a test' + conn = httplib.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + conn.send(expected) + self.assertEqual(expected, sock.data) + sock.data = '' + conn.send(array.array('c', expected)) + self.assertEqual(expected, sock.data) + sock.data = '' + conn.send(StringIO.StringIO(expected)) + self.assertEqual(expected, sock.data) + + def test_chunked(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + '0\r\n') + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), 'hello world') + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except httplib.IncompleteRead, i: + self.assertEqual(i.partial, 'hello world') + self.assertEqual(repr(i),'IncompleteRead(11 bytes read)') + self.assertEqual(str(i),'IncompleteRead(11 bytes read)') + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + '0\r\n') + resp = httplib.HTTPResponse(sock, method="HEAD") + resp.begin() + self.assertEqual(resp.read(), '') + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + + def test_negative_content_length(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\n' + 'Content-Length: -1\r\n\r\nHello\r\n') + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), 'Hello\r\n') + self.assertTrue(resp.isclosed()) + + def test_incomplete_read(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except httplib.IncompleteRead as i: + self.assertEqual(i.partial, 'Hello\r\n') + self.assertEqual(repr(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertEqual(str(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertTrue(resp.isclosed()) + else: + self.fail('IncompleteRead expected') + + def test_epipe(self): + sock = EPipeSocket( + "HTTP/1.0 401 Authorization Required\r\n" + "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"example\"\r\n", + b"Content-Length") + conn = httplib.HTTPConnection("example.com") + conn.sock = sock + self.assertRaises(socket.error, + lambda: conn.request("PUT", "/url", "body")) + resp = conn.getresponse() + self.assertEqual(401, resp.status) + self.assertEqual("Basic realm=\"example\"", + resp.getheader("www-authenticate")) + + def test_filenoattr(self): + # Just test the fileno attribute in the HTTPResponse Object. + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + self.assertTrue(hasattr(resp,'fileno'), + 'HTTPResponse should expose a fileno attribute') + + # Test lines overflowing the max line size (_MAXLINE in http.client) + + def test_overflowing_status_line(self): + self.skipTest("disabled for HTTP 0.9 support") + body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n" + resp = httplib.HTTPResponse(FakeSocket(body)) + self.assertRaises((httplib.LineTooLong, httplib.BadStatusLine), resp.begin) + + def test_overflowing_header_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(body)) + self.assertRaises(httplib.LineTooLong, resp.begin) + + def test_overflowing_chunked_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + + '0' * 65536 + 'a\r\n' + 'hello world\r\n' + '0\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertRaises(httplib.LineTooLong, resp.read) + + def test_early_eof(self): + # Test httpresponse with no \r\n termination, + body = "HTTP/1.1 200 Ok" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(), '') + self.assertTrue(resp.isclosed()) + + def test_error_leak(self): + # Test that the socket is not leaked if getresponse() fails + conn = httplib.HTTPConnection('example.com') + response = [] + class Response(httplib.HTTPResponse): + def __init__(self, *pos, **kw): + response.append(self) # Avoid garbage collector closing the socket + httplib.HTTPResponse.__init__(self, *pos, **kw) + conn.response_class = Response + conn.sock = FakeSocket('') # Emulate server dropping connection + conn.request('GET', '/') + self.assertRaises(httplib.BadStatusLine, conn.getresponse) + self.assertTrue(response) + #self.assertTrue(response[0].closed) + self.assertTrue(conn.sock.file_closed) + + def test_proxy_tunnel_without_status_line(self): + # Issue 17849: If a proxy tunnel is created that does not return + # a status code, fail. + body = 'hello world' + conn = httplib.HTTPConnection('example.com', strict=False) + conn.set_tunnel('foo') + conn.sock = FakeSocket(body) + with self.assertRaisesRegexp(socket.error, "Invalid response"): + conn._tunnel() + +class OfflineTest(TestCase): + def test_responses(self): + self.assertEqual(httplib.responses[httplib.NOT_FOUND], "Not Found") + + +class TestServerMixin: + """A limited socket server mixin. + + This is used by test cases for testing http connection end points. + """ + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = test_support.bind_port(self.serv) + self.source_port = test_support.find_unused_port() + self.serv.listen(5) + self.conn = None + + def tearDown(self): + if self.conn: + self.conn.close() + self.conn = None + self.serv.close() + self.serv = None + +class SourceAddressTest(TestServerMixin, TestCase): + def testHTTPConnectionSourceAddress(self): + self.conn = httplib.HTTPConnection(HOST, self.port, + source_address=('', self.source_port)) + self.conn.connect() + self.assertEqual(self.conn.sock.getsockname()[1], self.source_port) + + @unittest.skipIf(not hasattr(httplib, 'HTTPSConnection'), + 'httplib.HTTPSConnection not defined') + def testHTTPSConnectionSourceAddress(self): + self.conn = httplib.HTTPSConnection(HOST, self.port, + source_address=('', self.source_port)) + # We don't test anything here other the constructor not barfing as + # this code doesn't deal with setting up an active running SSL server + # for an ssl_wrapped connect() to actually return from. + + +class HTTPTest(TestServerMixin, TestCase): + def testHTTPConnection(self): + self.conn = httplib.HTTP(host=HOST, port=self.port, strict=None) + self.conn.connect() + self.assertEqual(self.conn._conn.host, HOST) + self.assertEqual(self.conn._conn.port, self.port) + + def testHTTPWithConnectHostPort(self): + testhost = 'unreachable.test.domain' + testport = '80' + self.conn = httplib.HTTP(host=testhost, port=testport) + self.conn.connect(host=HOST, port=self.port) + self.assertNotEqual(self.conn._conn.host, testhost) + self.assertNotEqual(self.conn._conn.port, testport) + self.assertEqual(self.conn._conn.host, HOST) + self.assertEqual(self.conn._conn.port, self.port) + + +class TimeoutTest(TestCase): + PORT = None + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + TimeoutTest.PORT = test_support.bind_port(self.serv) + self.serv.listen(5) + + def tearDown(self): + self.serv.close() + self.serv = None + + def testTimeoutAttribute(self): + '''This will prove that the timeout gets through + HTTPConnection and into the socket. + ''' + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = httplib.HTTPConnection(HOST, TimeoutTest.PORT) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + # no timeout -- do not use global socket default + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = httplib.HTTPConnection(HOST, TimeoutTest.PORT, + timeout=None) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), None) + httpConn.close() + + # a value + httpConn = httplib.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30) + httpConn.connect() + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + +class HTTPSTest(TestCase): + + def setUp(self): + if not hasattr(httplib, 'HTTPSConnection'): + self.skipTest('ssl support required') + + def make_server(self, certfile): + from test.ssl_servers import make_https_server + return make_https_server(self, certfile=certfile) + + def test_attributes(self): + # simple test to check it's storing the timeout + h = httplib.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) + self.assertEqual(h.timeout, 30) + + def test_networked(self): + # Default settings: requires a valid cert from a trusted CA + import ssl + test_support.requires('network') + with test_support.transient_internet('self-signed.pythontest.net'): + h = httplib.HTTPSConnection('self-signed.pythontest.net', 443) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_networked_noverification(self): + # Switch off cert verification + import ssl + test_support.requires('network') + with test_support.transient_internet('self-signed.pythontest.net'): + context = ssl._create_stdlib_context() + h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, + context=context) + h.request('GET', '/') + resp = h.getresponse() + self.assertIn('nginx', resp.getheader('server')) + + if hasattr(test_support, 'system_must_validate_cert'): # gevent: run on 2.7.8 + @test_support.system_must_validate_cert + def test_networked_trusted_by_default_cert(self): + # Default settings: requires a valid cert from a trusted CA + test_support.requires('network') + with test_support.transient_internet('www.python.org'): + h = httplib.HTTPSConnection('www.python.org', 443) + h.request('GET', '/') + resp = h.getresponse() + content_type = resp.getheader('content-type') + self.assertIn('text/html', content_type) + + def test_networked_good_cert(self): + # We feed the server's cert as a validating cert + import ssl + test_support.requires('network') + with test_support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_selfsigned_pythontestdotnet) + h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h.request('GET', '/') + resp = h.getresponse() + server_string = resp.getheader('server') + self.assertIn('nginx', server_string) + + def test_networked_bad_cert(self): + # We feed a "CA" cert that is unrelated to the server's cert + import ssl + test_support.requires('network') + with test_support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_unknown_cert(self): + # The custom cert isn't known to the default trust bundle + import ssl + server = self.make_server(CERT_localhost) + h = httplib.HTTPSConnection('localhost', server.port) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_good_hostname(self): + # The (valid) cert validates the HTTP hostname + import ssl + server = self.make_server(CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = httplib.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + + def test_local_bad_hostname(self): + # The (valid) cert doesn't validate the HTTP hostname + import ssl + server = self.make_server(CERT_fakehostname) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(CERT_fakehostname) + h = httplib.HTTPSConnection('localhost', server.port, context=context) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + h.close() + # With context.check_hostname=False, the mismatching is ignored + context.check_hostname = False + h = httplib.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + h.close() + + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(httplib.InvalidURL, httplib.HTTPSConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:443", "www.python.org", 443), + ("www.python.org:", "www.python.org", 443), + ("www.python.org", "www.python.org", 443), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 443), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", + 443)): + c = httplib.HTTPSConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + +class TunnelTests(TestCase): + def test_connect(self): + response_text = ( + 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n' # Reply to HEAD + 'Content-Length: 42\r\n\r\n' + ) + + def create_connection(address, timeout=None, source_address=None): + return FakeSocket(response_text, host=address[0], port=address[1]) + + conn = httplib.HTTPConnection('proxy.com') + conn._create_connection = create_connection + + # Once connected, we should not be able to tunnel anymore + conn.connect() + self.assertRaises(RuntimeError, conn.set_tunnel, 'destination.com') + + # But if close the connection, we are good. + conn.close() + conn.set_tunnel('destination.com') + conn.request('HEAD', '/', '') + + self.assertEqual(conn.sock.host, 'proxy.com') + self.assertEqual(conn.sock.port, 80) + self.assertIn('CONNECT destination.com', conn.sock.data) + # issue22095 + self.assertNotIn('Host: destination.com:None', conn.sock.data) + self.assertIn('Host: destination.com', conn.sock.data) + + self.assertNotIn('Host: proxy.com', conn.sock.data) + + conn.close() + + conn.request('PUT', '/', '') + self.assertEqual(conn.sock.host, 'proxy.com') + self.assertEqual(conn.sock.port, 80) + self.assertTrue('CONNECT destination.com' in conn.sock.data) + self.assertTrue('Host: destination.com' in conn.sock.data) + + +@test_support.reap_threads +def test_main(verbose=None): + test_support.run_unittest(HeaderTests, OfflineTest, BasicTest, TimeoutTest, + HTTPTest, HTTPSTest, SourceAddressTest, + TunnelTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7/test_httpservers.py b/src/greentest/2.7/test_httpservers.py new file mode 100644 index 0000000..11f0d5d --- /dev/null +++ b/src/greentest/2.7/test_httpservers.py @@ -0,0 +1,679 @@ +"""Unittests for the various HTTPServer modules. + +Written by Cody A.W. Somerville , +Josip Dzolonga, and Michael Otteneder for the 2007/08 GHOP contest. +""" + +import os +import sys +import re +import base64 +import ntpath +import shutil +import urllib +import httplib +import tempfile +import unittest +import CGIHTTPServer + + +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +from SimpleHTTPServer import SimpleHTTPRequestHandler +from CGIHTTPServer import CGIHTTPRequestHandler +from StringIO import StringIO +from test import test_support + + +threading = test_support.import_module('threading') + + +class NoLogRequestHandler: + def log_message(self, *args): + # don't write log messages to stderr + pass + +class SocketlessRequestHandler(SimpleHTTPRequestHandler): + def __init__(self): + self.get_called = False + self.protocol_version = "HTTP/1.1" + + def do_GET(self): + self.get_called = True + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(b'Data\r\n') + + def log_message(self, fmt, *args): + pass + + +class TestServerThread(threading.Thread): + def __init__(self, test_object, request_handler): + threading.Thread.__init__(self) + self.request_handler = request_handler + self.test_object = test_object + + def run(self): + self.server = HTTPServer(('', 0), self.request_handler) + self.test_object.PORT = self.server.socket.getsockname()[1] + self.test_object.server_started.set() + self.test_object = None + try: + self.server.serve_forever(0.05) + finally: + self.server.server_close() + + def stop(self): + self.server.shutdown() + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test_support.threading_setup() + os.environ = test_support.EnvironmentVarGuard() + self.server_started = threading.Event() + self.thread = TestServerThread(self, self.request_handler) + self.thread.start() + self.server_started.wait() + + def tearDown(self): + self.thread.stop() + os.environ.__exit__() + test_support.threading_cleanup(*self._threads) + + def request(self, uri, method='GET', body=None, headers={}): + self.connection = httplib.HTTPConnection('localhost', self.PORT) + self.connection.request(method, uri, body, headers) + return self.connection.getresponse() + +class BaseHTTPRequestHandlerTestCase(unittest.TestCase): + """Test the functionality of the BaseHTTPServer focussing on + BaseHTTPRequestHandler. + """ + + HTTPResponseMatch = re.compile('HTTP/1.[0-9]+ 200 OK') + + def setUp (self): + self.handler = SocketlessRequestHandler() + + def send_typical_request(self, message): + input_msg = StringIO(message) + output = StringIO() + self.handler.rfile = input_msg + self.handler.wfile = output + self.handler.handle_one_request() + output.seek(0) + return output.readlines() + + def verify_get_called(self): + self.assertTrue(self.handler.get_called) + + def verify_expected_headers(self, headers): + for fieldName in 'Server: ', 'Date: ', 'Content-Type: ': + self.assertEqual(sum(h.startswith(fieldName) for h in headers), 1) + + def verify_http_server_response(self, response): + match = self.HTTPResponseMatch.search(response) + self.assertIsNotNone(match) + + def test_http_1_1(self): + result = self.send_typical_request('GET / HTTP/1.1\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], 'Data\r\n') + + def test_http_1_0(self): + result = self.send_typical_request('GET / HTTP/1.0\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], 'Data\r\n') + + def test_http_0_9(self): + result = self.send_typical_request('GET / HTTP/0.9\r\n\r\n') + self.assertEqual(len(result), 1) + self.assertEqual(result[0], 'Data\r\n') + self.verify_get_called() + + def test_with_continue_1_0(self): + result = self.send_typical_request('GET / HTTP/1.0\r\nExpect: 100-continue\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], 'Data\r\n') + + def test_request_length(self): + # Issue #10714: huge request lines are discarded, to avoid Denial + # of Service attacks. + result = self.send_typical_request(b'GET ' + b'x' * 65537) + self.assertEqual(result[0], b'HTTP/1.1 414 Request-URI Too Long\r\n') + self.assertFalse(self.handler.get_called) + + +class BaseHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + default_request_version = 'HTTP/1.1' + + def do_TEST(self): + self.send_response(204) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'close') + self.end_headers() + + def do_KEEP(self): + self.send_response(204) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'keep-alive') + self.end_headers() + + def do_KEYERROR(self): + self.send_error(999) + + def do_CUSTOM(self): + self.send_response(999) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'close') + self.end_headers() + + def do_SEND_ERROR(self): + self.send_error(int(self.path[1:])) + + def do_HEAD(self): + self.send_error(int(self.path[1:])) + + def setUp(self): + BaseTestCase.setUp(self) + self.con = httplib.HTTPConnection('localhost', self.PORT) + self.con.connect() + + def test_command(self): + self.con.request('GET', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_request_line_trimming(self): + self.con._http_vsn_str = 'HTTP/1.1\n' + self.con.putrequest('XYZBOGUS', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_version_bogus(self): + self.con._http_vsn_str = 'FUBAR' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 400) + + def test_version_digits(self): + self.con._http_vsn_str = 'HTTP/9.9.9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 400) + + def test_version_none_get(self): + self.con._http_vsn_str = '' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_version_none(self): + # Test that a valid method is rejected when not HTTP/1.x + self.con._http_vsn_str = '' + self.con.putrequest('CUSTOM', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 400) + + def test_version_invalid(self): + self.con._http_vsn = 99 + self.con._http_vsn_str = 'HTTP/9.9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 505) + + def test_send_blank(self): + self.con._http_vsn_str = '' + self.con.putrequest('', '') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 400) + + def test_header_close(self): + self.con.putrequest('GET', '/') + self.con.putheader('Connection', 'close') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_head_keep_alive(self): + self.con._http_vsn_str = 'HTTP/1.1' + self.con.putrequest('GET', '/') + self.con.putheader('Connection', 'keep-alive') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_handler(self): + self.con.request('TEST', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 204) + + def test_return_header_keep_alive(self): + self.con.request('KEEP', '/') + res = self.con.getresponse() + self.assertEqual(res.getheader('Connection'), 'keep-alive') + self.con.request('TEST', '/') + self.addCleanup(self.con.close) + + def test_internal_key_error(self): + self.con.request('KEYERROR', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 999) + + def test_return_custom_status(self): + self.con.request('CUSTOM', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 999) + + def test_send_error(self): + allow_transfer_encoding_codes = (205, 304) + for code in (101, 102, 204, 205, 304): + self.con.request('SEND_ERROR', '/{}'.format(code)) + res = self.con.getresponse() + self.assertEqual(code, res.status) + self.assertEqual(None, res.getheader('Content-Length')) + self.assertEqual(None, res.getheader('Content-Type')) + if code not in allow_transfer_encoding_codes: + self.assertEqual(None, res.getheader('Transfer-Encoding')) + + data = res.read() + self.assertEqual(b'', data) + + def test_head_via_send_error(self): + allow_transfer_encoding_codes = (205, 304) + for code in (101, 200, 204, 205, 304): + self.con.request('HEAD', '/{}'.format(code)) + res = self.con.getresponse() + self.assertEqual(code, res.status) + if code == 200: + self.assertEqual(None, res.getheader('Content-Length')) + self.assertIn('text/html', res.getheader('Content-Type')) + else: + self.assertEqual(None, res.getheader('Content-Length')) + self.assertEqual(None, res.getheader('Content-Type')) + if code not in allow_transfer_encoding_codes: + self.assertEqual(None, res.getheader('Transfer-Encoding')) + + data = res.read() + self.assertEqual(b'', data) + + +class SimpleHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, SimpleHTTPRequestHandler): + pass + + def setUp(self): + BaseTestCase.setUp(self) + self.cwd = os.getcwd() + basetempdir = tempfile.gettempdir() + os.chdir(basetempdir) + self.data = 'We are the knights who say Ni!' + self.tempdir = tempfile.mkdtemp(dir=basetempdir) + self.tempdir_name = os.path.basename(self.tempdir) + self.base_url = '/' + self.tempdir_name + temp = open(os.path.join(self.tempdir, 'test'), 'wb') + temp.write(self.data) + temp.close() + + def tearDown(self): + try: + os.chdir(self.cwd) + try: + shutil.rmtree(self.tempdir) + except OSError: + pass + finally: + BaseTestCase.tearDown(self) + + def check_status_and_reason(self, response, status, data=None): + body = response.read() + self.assertTrue(response) + self.assertEqual(response.status, status) + self.assertIsNotNone(response.reason) + if data: + self.assertEqual(data, body) + + def test_get(self): + #constructs the path relative to the root directory of the HTTPServer + response = self.request(self.base_url + '/test') + self.check_status_and_reason(response, 200, data=self.data) + # check for trailing "/" which should return 404. See Issue17324 + response = self.request(self.base_url + '/test/') + self.check_status_and_reason(response, 404) + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, 200) + response = self.request(self.base_url) + self.check_status_and_reason(response, 301) + response = self.request(self.base_url + '/?hi=2') + self.check_status_and_reason(response, 200) + response = self.request(self.base_url + '?hi=1') + self.check_status_and_reason(response, 301) + self.assertEqual(response.getheader("Location"), + self.base_url + "/?hi=1") + response = self.request('/ThisDoesNotExist') + self.check_status_and_reason(response, 404) + response = self.request('/' + 'ThisDoesNotExist' + '/') + self.check_status_and_reason(response, 404) + with open(os.path.join(self.tempdir_name, 'index.html'), 'w') as fp: + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, 200) + # chmod() doesn't work as expected on Windows, and filesystem + # permissions are ignored by root on Unix. + if os.name == 'posix' and os.geteuid() != 0: + os.chmod(self.tempdir, 0) + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, 404) + os.chmod(self.tempdir, 0755) + + def test_head(self): + response = self.request( + self.base_url + '/test', method='HEAD') + self.check_status_and_reason(response, 200) + self.assertEqual(response.getheader('content-length'), + str(len(self.data))) + self.assertEqual(response.getheader('content-type'), + 'application/octet-stream') + + def test_invalid_requests(self): + response = self.request('/', method='FOO') + self.check_status_and_reason(response, 501) + # requests must be case sensitive,so this should fail too + response = self.request('/', method='custom') + self.check_status_and_reason(response, 501) + response = self.request('/', method='GETs') + self.check_status_and_reason(response, 501) + + def test_path_without_leading_slash(self): + response = self.request(self.tempdir_name + '/test') + self.check_status_and_reason(response, 200, data=self.data) + response = self.request(self.tempdir_name + '/test/') + self.check_status_and_reason(response, 404) + response = self.request(self.tempdir_name + '/') + self.check_status_and_reason(response, 200) + response = self.request(self.tempdir_name) + self.check_status_and_reason(response, 301) + response = self.request(self.tempdir_name + '/?hi=2') + self.check_status_and_reason(response, 200) + response = self.request(self.tempdir_name + '?hi=1') + self.check_status_and_reason(response, 301) + self.assertEqual(response.getheader("Location"), + self.tempdir_name + "/?hi=1") + + +cgi_file1 = """\ +#!%s + +print "Content-type: text/html" +print +print "Hello World" +""" + +cgi_file2 = """\ +#!%s +import cgi + +print "Content-type: text/html" +print + +form = cgi.FieldStorage() +print "%%s, %%s, %%s" %% (form.getfirst("spam"), form.getfirst("eggs"), + form.getfirst("bacon")) +""" + +cgi_file4 = """\ +#!%s +import os + +print("Content-type: text/html") +print("") + +print(os.environ["%s"]) +""" + + +@unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, + "This test can't be run reliably as root (issue #13308).") +class CGIHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): + pass + + def setUp(self): + BaseTestCase.setUp(self) + self.parent_dir = tempfile.mkdtemp() + self.cgi_dir = os.path.join(self.parent_dir, 'cgi-bin') + self.cgi_child_dir = os.path.join(self.cgi_dir, 'child-dir') + os.mkdir(self.cgi_dir) + os.mkdir(self.cgi_child_dir) + + # The shebang line should be pure ASCII: use symlink if possible. + # See issue #7668. + if hasattr(os, 'symlink'): + self.pythonexe = os.path.join(self.parent_dir, 'python') + os.symlink(sys.executable, self.pythonexe) + else: + self.pythonexe = sys.executable + + self.nocgi_path = os.path.join(self.parent_dir, 'nocgi.py') + with open(self.nocgi_path, 'w') as fp: + fp.write(cgi_file1 % self.pythonexe) + os.chmod(self.nocgi_path, 0777) + + self.file1_path = os.path.join(self.cgi_dir, 'file1.py') + with open(self.file1_path, 'w') as file1: + file1.write(cgi_file1 % self.pythonexe) + os.chmod(self.file1_path, 0777) + + self.file2_path = os.path.join(self.cgi_dir, 'file2.py') + with open(self.file2_path, 'w') as file2: + file2.write(cgi_file2 % self.pythonexe) + os.chmod(self.file2_path, 0777) + + self.file3_path = os.path.join(self.cgi_child_dir, 'file3.py') + with open(self.file3_path, 'w') as file3: + file3.write(cgi_file1 % self.pythonexe) + os.chmod(self.file3_path, 0777) + + self.file4_path = os.path.join(self.cgi_dir, 'file4.py') + with open(self.file4_path, 'w') as file4: + file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING')) + os.chmod(self.file4_path, 0o777) + + self.cwd = os.getcwd() + os.chdir(self.parent_dir) + + def tearDown(self): + try: + os.chdir(self.cwd) + if self.pythonexe != sys.executable: + os.remove(self.pythonexe) + os.remove(self.nocgi_path) + os.remove(self.file1_path) + os.remove(self.file2_path) + os.remove(self.file3_path) + os.remove(self.file4_path) + os.rmdir(self.cgi_child_dir) + os.rmdir(self.cgi_dir) + os.rmdir(self.parent_dir) + finally: + BaseTestCase.tearDown(self) + + def test_url_collapse_path(self): + # verify tail is the last portion and head is the rest on proper urls + test_vectors = { + '': '//', + '..': IndexError, + '/.//..': IndexError, + '/': '//', + '//': '//', + '/\\': '//\\', + '/.//': '//', + 'cgi-bin/file1.py': '/cgi-bin/file1.py', + '/cgi-bin/file1.py': '/cgi-bin/file1.py', + 'a': '//a', + '/a': '//a', + '//a': '//a', + './a': '//a', + './C:/': '/C:/', + '/a/b': '/a/b', + '/a/b/': '/a/b/', + '/a/b/.': '/a/b/', + '/a/b/c/..': '/a/b/', + '/a/b/c/../d': '/a/b/d', + '/a/b/c/../d/e/../f': '/a/b/d/f', + '/a/b/c/../d/e/../../f': '/a/b/f', + '/a/b/c/../d/e/.././././..//f': '/a/b/f', + '../a/b/c/../d/e/.././././..//f': IndexError, + '/a/b/c/../d/e/../../../f': '/a/f', + '/a/b/c/../d/e/../../../../f': '//f', + '/a/b/c/../d/e/../../../../../f': IndexError, + '/a/b/c/../d/e/../../../../f/..': '//', + '/a/b/c/../d/e/../../../../f/../.': '//', + } + for path, expected in test_vectors.iteritems(): + if isinstance(expected, type) and issubclass(expected, Exception): + self.assertRaises(expected, + CGIHTTPServer._url_collapse_path, path) + else: + actual = CGIHTTPServer._url_collapse_path(path) + self.assertEqual(expected, actual, + msg='path = %r\nGot: %r\nWanted: %r' % + (path, actual, expected)) + + def test_headers_and_content(self): + res = self.request('/cgi-bin/file1.py') + self.assertEqual(('Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_issue19435(self): + res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh') + self.assertEqual(res.status, 404) + + def test_post(self): + params = urllib.urlencode({'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) + headers = {'Content-type' : 'application/x-www-form-urlencoded'} + res = self.request('/cgi-bin/file2.py', 'POST', params, headers) + + self.assertEqual(res.read(), '1, python, 123456\n') + + def test_invaliduri(self): + res = self.request('/cgi-bin/invalid') + res.read() + self.assertEqual(res.status, 404) + + def test_authorization(self): + headers = {'Authorization' : 'Basic %s' % + base64.b64encode('username:pass')} + res = self.request('/cgi-bin/file1.py', 'GET', headers=headers) + self.assertEqual(('Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_no_leading_slash(self): + # http://bugs.python.org/issue2254 + res = self.request('cgi-bin/file1.py') + self.assertEqual(('Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_os_environ_is_not_altered(self): + signature = "Test CGI Server" + os.environ['SERVER_SOFTWARE'] = signature + res = self.request('/cgi-bin/file1.py') + self.assertEqual((b'Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) + + def test_urlquote_decoding_in_cgi_check(self): + res = self.request('/cgi-bin%2ffile1.py') + self.assertEqual((b'Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_nested_cgi_path_issue21323(self): + res = self.request('/cgi-bin/child-dir/file3.py') + self.assertEqual((b'Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_query_with_multiple_question_mark(self): + res = self.request('/cgi-bin/file4.py?a=b?c=d') + self.assertEqual( + (b'a=b?c=d\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_query_with_continuous_slashes(self): + res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//') + self.assertEqual( + (b'k=aa%2F%2Fbb&//q//p//=//a//b//\n', + 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + +class SimpleHTTPRequestHandlerTestCase(unittest.TestCase): + """ Test url parsing """ + def setUp(self): + self.translated = os.getcwd() + self.translated = os.path.join(self.translated, 'filename') + self.handler = SocketlessRequestHandler() + + def test_query_arguments(self): + path = self.handler.translate_path('/filename') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('/filename?foo=bar') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('/filename?a=b&spam=eggs#zot') + self.assertEqual(path, self.translated) + + def test_start_with_double_slash(self): + path = self.handler.translate_path('//filename') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('//filename?foo=bar') + self.assertEqual(path, self.translated) + + def test_windows_colon(self): + import SimpleHTTPServer + with test_support.swap_attr(SimpleHTTPServer.os, 'path', ntpath): + path = self.handler.translate_path('c:c:c:foo/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('\\c:../filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('c:\\c:..\\foo/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('c:c:foo\\c:c:bar/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + +def test_main(verbose=None): + try: + cwd = os.getcwd() + test_support.run_unittest(BaseHTTPRequestHandlerTestCase, + SimpleHTTPRequestHandlerTestCase, + BaseHTTPServerTestCase, + SimpleHTTPServerTestCase, + CGIHTTPServerTestCase + ) + finally: + os.chdir(cwd) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7/test_queue.py b/src/greentest/2.7/test_queue.py new file mode 100644 index 0000000..34a4aef --- /dev/null +++ b/src/greentest/2.7/test_queue.py @@ -0,0 +1,325 @@ +# Some simple queue module tests, plus some failure conditions +# to ensure the Queue locks remain stable. +import Queue +import time +import unittest +from test import test_support +threading = test_support.import_module('threading') + +QUEUE_SIZE = 5 + +# A thread to run a function that unclogs a blocked Queue. +class _TriggerThread(threading.Thread): + def __init__(self, fn, args): + self.fn = fn + self.args = args + self.startedEvent = threading.Event() + threading.Thread.__init__(self) + + def run(self): + # The sleep isn't necessary, but is intended to give the blocking + # function in the main thread a chance at actually blocking before + # we unclog it. But if the sleep is longer than the timeout-based + # tests wait in their blocking functions, those tests will fail. + # So we give them much longer timeout values compared to the + # sleep here (I aimed at 10 seconds for blocking functions -- + # they should never actually wait that long - they should make + # progress as soon as we call self.fn()). + time.sleep(0.1) + self.startedEvent.set() + self.fn(*self.args) + + +# Execute a function that blocks, and in a separate thread, a function that +# triggers the release. Returns the result of the blocking function. Caution: +# block_func must guarantee to block until trigger_func is called, and +# trigger_func must guarantee to change queue state so that block_func can make +# enough progress to return. In particular, a block_func that just raises an +# exception regardless of whether trigger_func is called will lead to +# timing-dependent sporadic failures, and one of those went rarely seen but +# undiagnosed for years. Now block_func must be unexceptional. If block_func +# is supposed to raise an exception, call do_exceptional_blocking_test() +# instead. + +class BlockingTestMixin: + + def tearDown(self): + self.t = None + + def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): + self.t = _TriggerThread(trigger_func, trigger_args) + self.t.start() + self.result = block_func(*block_args) + # If block_func returned before our thread made the call, we failed! + if not self.t.startedEvent.is_set(): + self.fail("blocking function '%r' appeared not to block" % + block_func) + self.t.join(10) # make sure the thread terminates + if self.t.is_alive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) + return self.result + + # Call this instead if block_func is supposed to raise an exception. + def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, + trigger_args, expected_exception_class): + self.t = _TriggerThread(trigger_func, trigger_args) + self.t.start() + try: + try: + block_func(*block_args) + except expected_exception_class: + raise + else: + self.fail("expected exception of kind %r" % + expected_exception_class) + finally: + self.t.join(10) # make sure the thread terminates + if self.t.is_alive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) + if not self.t.startedEvent.is_set(): + self.fail("trigger thread ended but event never set") + + +class BaseQueueTest(BlockingTestMixin): + def setUp(self): + self.cum = 0 + self.cumlock = threading.Lock() + + def simple_queue_test(self, q): + if not q.empty(): + raise RuntimeError, "Call this function with an empty queue" + # I guess we better check things actually queue correctly a little :) + q.put(111) + q.put(333) + q.put(222) + target_order = dict(Queue = [111, 333, 222], + LifoQueue = [222, 333, 111], + PriorityQueue = [111, 222, 333]) + actual_order = [q.get(), q.get(), q.get()] + self.assertEqual(actual_order, target_order[q.__class__.__name__], + "Didn't seem to queue the correct data!") + for i in range(QUEUE_SIZE-1): + q.put(i) + self.assertTrue(not q.empty(), "Queue should not be empty") + self.assertTrue(not q.full(), "Queue should not be full") + last = 2 * QUEUE_SIZE + full = 3 * 2 * QUEUE_SIZE + q.put(last) + self.assertTrue(q.full(), "Queue should be full") + try: + q.put(full, block=0) + self.fail("Didn't appear to block with a full queue") + except Queue.Full: + pass + try: + q.put(full, timeout=0.01) + self.fail("Didn't appear to time-out with a full queue") + except Queue.Full: + pass + # Test a blocking put + self.do_blocking_test(q.put, (full,), q.get, ()) + self.do_blocking_test(q.put, (full, True, 10), q.get, ()) + # Empty it + for i in range(QUEUE_SIZE): + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + try: + q.get(block=0) + self.fail("Didn't appear to block with an empty queue") + except Queue.Empty: + pass + try: + q.get(timeout=0.01) + self.fail("Didn't appear to time-out with an empty queue") + except Queue.Empty: + pass + # Test a blocking get + self.do_blocking_test(q.get, (), q.put, ('empty',)) + self.do_blocking_test(q.get, (True, 10), q.put, ('empty',)) + + + def worker(self, q): + while True: + x = q.get() + if x is None: + q.task_done() + return + with self.cumlock: + self.cum += x + q.task_done() + + def queue_join_test(self, q): + self.cum = 0 + for i in (0,1): + threading.Thread(target=self.worker, args=(q,)).start() + for i in xrange(100): + q.put(i) + q.join() + self.assertEqual(self.cum, sum(range(100)), + "q.join() did not block until all tasks were done") + for i in (0,1): + q.put(None) # instruct the threads to close + q.join() # verify that you can join twice + + def test_queue_task_done(self): + # Test to make sure a queue task completed successfully. + q = self.type2test() + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + + def test_queue_join(self): + # Test that a queue join()s successfully, and before anything else + # (done twice for insurance). + q = self.type2test() + self.queue_join_test(q) + self.queue_join_test(q) + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + + def test_simple_queue(self): + # Do it a couple of times on the same queue. + # Done twice to make sure works with same instance reused. + q = self.type2test(QUEUE_SIZE) + self.simple_queue_test(q) + self.simple_queue_test(q) + + +class QueueTest(BaseQueueTest, unittest.TestCase): + type2test = Queue.Queue + +class LifoQueueTest(BaseQueueTest, unittest.TestCase): + type2test = Queue.LifoQueue + +class PriorityQueueTest(BaseQueueTest, unittest.TestCase): + type2test = Queue.PriorityQueue + + + +# A Queue subclass that can provoke failure at a moment's notice :) +class FailingQueueException(Exception): + pass + +class FailingQueue(Queue.Queue): + def __init__(self, *args): + self.fail_next_put = False + self.fail_next_get = False + Queue.Queue.__init__(self, *args) + def _put(self, item): + if self.fail_next_put: + self.fail_next_put = False + raise FailingQueueException, "You Lose" + return Queue.Queue._put(self, item) + def _get(self): + if self.fail_next_get: + self.fail_next_get = False + raise FailingQueueException, "You Lose" + return Queue.Queue._get(self) + +class FailingQueueTest(BlockingTestMixin, unittest.TestCase): + + def failing_queue_test(self, q): + if not q.empty(): + raise RuntimeError, "Call this function with an empty queue" + for i in range(QUEUE_SIZE-1): + q.put(i) + # Test a failing non-blocking put. + q.fail_next_put = True + try: + q.put("oops", block=0) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + q.fail_next_put = True + try: + q.put("oops", timeout=0.1) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + q.put("last") + self.assertTrue(q.full(), "Queue should be full") + # Test a failing blocking put + q.fail_next_put = True + try: + self.do_blocking_test(q.put, ("full",), q.get, ()) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # Check the Queue isn't damaged. + # put failed, but get succeeded - re-add + q.put("last") + # Test a failing timeout put + q.fail_next_put = True + try: + self.do_exceptional_blocking_test(q.put, ("full", True, 10), q.get, (), + FailingQueueException) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # Check the Queue isn't damaged. + # put failed, but get succeeded - re-add + q.put("last") + self.assertTrue(q.full(), "Queue should be full") + q.get() + self.assertTrue(not q.full(), "Queue should not be full") + q.put("last") + self.assertTrue(q.full(), "Queue should be full") + # Test a blocking put + self.do_blocking_test(q.put, ("full",), q.get, ()) + # Empty it + for i in range(QUEUE_SIZE): + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + q.put("first") + q.fail_next_get = True + try: + q.get() + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + self.assertTrue(not q.empty(), "Queue should not be empty") + q.fail_next_get = True + try: + q.get(timeout=0.1) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + self.assertTrue(not q.empty(), "Queue should not be empty") + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + q.fail_next_get = True + try: + self.do_exceptional_blocking_test(q.get, (), q.put, ('empty',), + FailingQueueException) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # put succeeded, but get failed. + self.assertTrue(not q.empty(), "Queue should not be empty") + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + + def test_failing_queue(self): + # Test to make sure a queue is functioning correctly. + # Done twice to the same instance. + q = FailingQueue(QUEUE_SIZE) + self.failing_queue_test(q) + self.failing_queue_test(q) + + +def test_main(): + test_support.run_unittest(QueueTest, LifoQueueTest, PriorityQueueTest, + FailingQueueTest) + + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_select.py b/src/greentest/2.7/test_select.py new file mode 100644 index 0000000..175bbda --- /dev/null +++ b/src/greentest/2.7/test_select.py @@ -0,0 +1,67 @@ +from test import test_support +import unittest +import select +import os +import sys + +@unittest.skipIf(sys.platform[:3] in ('win', 'os2', 'riscos'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + def test_select(self): + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if test_support.verbose: + print 'timeout =', tout + rfd, wfd, xfd = select.select([p], [], [], tout) + if (rfd, wfd, xfd) == ([], [], []): + continue + if (rfd, wfd, xfd) == ([p], [], []): + line = p.readline() + if test_support.verbose: + print repr(line) + if not line: + if test_support.verbose: + print 'EOF' + break + continue + self.fail('Unexpected return values from select():', rfd, wfd, xfd) + p.close() + + # Issue 16230: Crash on select resized list + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + self.assertEqual(select.select([], a, []), ([], a[:5], [])) + +def test_main(): + test_support.run_unittest(SelectTestCase) + test_support.reap_children() + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_signal.py b/src/greentest/2.7/test_signal.py new file mode 100644 index 0000000..7483f64 --- /dev/null +++ b/src/greentest/2.7/test_signal.py @@ -0,0 +1,501 @@ +import unittest +from test import test_support +from contextlib import closing +import gc +import pickle +import select +import signal +import subprocess +import traceback +import sys, os, time, errno + +if sys.platform in ('os2', 'riscos'): + raise unittest.SkipTest("Can't test signal on %s" % sys.platform) + + +class HandlerBCalled(Exception): + pass + + +def exit_subprocess(): + """Use os._exit(0) to exit the current subprocess. + + Otherwise, the test catches the SystemExit and continues executing + in parallel with the original test, so you wind up with an + exponential number of tests running concurrently. + """ + os._exit(0) + + +def ignoring_eintr(__func, *args, **kwargs): + try: + return __func(*args, **kwargs) + except EnvironmentError as e: + if e.errno != errno.EINTR: + raise + return None + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class InterProcessSignalTests(unittest.TestCase): + MAX_DURATION = 20 # Entire test should last at most 20 sec. + + def setUp(self): + self.using_gc = gc.isenabled() + gc.disable() + + def tearDown(self): + if self.using_gc: + gc.enable() + + def format_frame(self, frame, limit=None): + return ''.join(traceback.format_stack(frame, limit=limit)) + + def handlerA(self, signum, frame): + self.a_called = True + if test_support.verbose: + print "handlerA invoked from signal %s at:\n%s" % ( + signum, self.format_frame(frame, limit=1)) + + def handlerB(self, signum, frame): + self.b_called = True + if test_support.verbose: + print "handlerB invoked from signal %s at:\n%s" % ( + signum, self.format_frame(frame, limit=1)) + raise HandlerBCalled(signum, self.format_frame(frame)) + + def wait(self, child): + """Wait for child to finish, ignoring EINTR.""" + while True: + try: + child.wait() + return + except OSError as e: + if e.errno != errno.EINTR: + raise + + def run_test(self): + # Install handlers. This function runs in a sub-process, so we + # don't worry about re-setting the default handlers. + signal.signal(signal.SIGHUP, self.handlerA) + signal.signal(signal.SIGUSR1, self.handlerB) + signal.signal(signal.SIGUSR2, signal.SIG_IGN) + signal.signal(signal.SIGALRM, signal.default_int_handler) + + # Variables the signals will modify: + self.a_called = False + self.b_called = False + + # Let the sub-processes know who to send signals to. + pid = os.getpid() + if test_support.verbose: + print "test runner's pid is", pid + + child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)]) + if child: + self.wait(child) + if not self.a_called: + time.sleep(1) # Give the signal time to be delivered. + self.assertTrue(self.a_called) + self.assertFalse(self.b_called) + self.a_called = False + + # Make sure the signal isn't delivered while the previous + # Popen object is being destroyed, because __del__ swallows + # exceptions. + del child + try: + child = subprocess.Popen(['kill', '-USR1', str(pid)]) + # This wait should be interrupted by the signal's exception. + self.wait(child) + time.sleep(1) # Give the signal time to be delivered. + self.fail('HandlerBCalled exception not raised') + except HandlerBCalled: + self.assertTrue(self.b_called) + self.assertFalse(self.a_called) + if test_support.verbose: + print "HandlerBCalled exception caught" + + child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)]) + if child: + self.wait(child) # Nothing should happen. + + try: + signal.alarm(1) + # The race condition in pause doesn't matter in this case, + # since alarm is going to raise a KeyboardException, which + # will skip the call. + signal.pause() + # But if another signal arrives before the alarm, pause + # may return early. + time.sleep(1) + except KeyboardInterrupt: + if test_support.verbose: + print "KeyboardInterrupt (the alarm() went off)" + except: + self.fail("Some other exception woke us from pause: %s" % + traceback.format_exc()) + else: + self.fail("pause returned of its own accord, and the signal" + " didn't arrive after another second.") + + # Issue 3864. Unknown if this affects earlier versions of freebsd also. + @unittest.skipIf(sys.platform=='freebsd6', + 'inter process signals not reliable (do not mix well with threading) ' + 'on freebsd6') + def test_main(self): + # This function spawns a child process to insulate the main + # test-running process from all the signals. It then + # communicates with that child process over a pipe and + # re-raises information about any exceptions the child + # raises. The real work happens in self.run_test(). + os_done_r, os_done_w = os.pipe() + with closing(os.fdopen(os_done_r)) as done_r, \ + closing(os.fdopen(os_done_w, 'w')) as done_w: + child = os.fork() + if child == 0: + # In the child process; run the test and report results + # through the pipe. + try: + done_r.close() + # Have to close done_w again here because + # exit_subprocess() will skip the enclosing with block. + with closing(done_w): + try: + self.run_test() + except: + pickle.dump(traceback.format_exc(), done_w) + else: + pickle.dump(None, done_w) + except: + print 'Uh oh, raised from pickle.' + traceback.print_exc() + finally: + exit_subprocess() + + done_w.close() + # Block for up to MAX_DURATION seconds for the test to finish. + r, w, x = select.select([done_r], [], [], self.MAX_DURATION) + if done_r in r: + tb = pickle.load(done_r) + if tb: + self.fail(tb) + else: + os.kill(child, signal.SIGKILL) + self.fail('Test deadlocked after %d seconds.' % + self.MAX_DURATION) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class BasicSignalTests(unittest.TestCase): + def trivial_signal_handler(self, *args): + pass + + def test_out_of_range_signal_number_raises_error(self): + self.assertRaises(ValueError, signal.getsignal, 4242) + + self.assertRaises(ValueError, signal.signal, 4242, + self.trivial_signal_handler) + + def test_setting_signal_handler_to_none_raises_error(self): + self.assertRaises(TypeError, signal.signal, + signal.SIGUSR1, None) + + def test_getsignal(self): + hup = signal.signal(signal.SIGHUP, self.trivial_signal_handler) + self.assertEqual(signal.getsignal(signal.SIGHUP), + self.trivial_signal_handler) + signal.signal(signal.SIGHUP, hup) + self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + + +@unittest.skipUnless(sys.platform == "win32", "Windows specific") +class WindowsSignalTests(unittest.TestCase): + def test_issue9324(self): + # Updated for issue #10003, adding SIGBREAK + handler = lambda x, y: None + for sig in (signal.SIGABRT, signal.SIGBREAK, signal.SIGFPE, + signal.SIGILL, signal.SIGINT, signal.SIGSEGV, + signal.SIGTERM): + # Set and then reset a handler for signals that work on windows + signal.signal(sig, signal.signal(sig, handler)) + + with self.assertRaises(ValueError): + signal.signal(-1, handler) + + with self.assertRaises(ValueError): + signal.signal(7, handler) + + +class WakeupFDTests(unittest.TestCase): + + def test_invalid_fd(self): + fd = test_support.make_bad_fd() + self.assertRaises(ValueError, signal.set_wakeup_fd, fd) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class WakeupSignalTests(unittest.TestCase): + TIMEOUT_FULL = 10 + TIMEOUT_HALF = 5 + + def test_wakeup_fd_early(self): + import select + + signal.alarm(1) + before_time = time.time() + # We attempt to get a signal during the sleep, + # before select is called + time.sleep(self.TIMEOUT_FULL) + mid_time = time.time() + self.assertTrue(mid_time - before_time < self.TIMEOUT_HALF) + select.select([self.read], [], [], self.TIMEOUT_FULL) + after_time = time.time() + self.assertTrue(after_time - mid_time < self.TIMEOUT_HALF) + + def test_wakeup_fd_during(self): + import select + + signal.alarm(1) + before_time = time.time() + # We attempt to get a signal during the select call + self.assertRaises(select.error, select.select, + [self.read], [], [], self.TIMEOUT_FULL) + after_time = time.time() + self.assertTrue(after_time - before_time < self.TIMEOUT_HALF) + + def setUp(self): + import fcntl + + self.alrm = signal.signal(signal.SIGALRM, lambda x,y:None) + self.read, self.write = os.pipe() + flags = fcntl.fcntl(self.write, fcntl.F_GETFL, 0) + flags = flags | os.O_NONBLOCK + fcntl.fcntl(self.write, fcntl.F_SETFL, flags) + self.old_wakeup = signal.set_wakeup_fd(self.write) + + def tearDown(self): + signal.set_wakeup_fd(self.old_wakeup) + os.close(self.read) + os.close(self.write) + signal.signal(signal.SIGALRM, self.alrm) + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class SiginterruptTest(unittest.TestCase): + + def setUp(self): + """Install a no-op signal handler that can be set to allow + interrupts or not, and arrange for the original signal handler to be + re-installed when the test is finished. + """ + self.signum = signal.SIGUSR1 + oldhandler = signal.signal(self.signum, lambda x,y: None) + self.addCleanup(signal.signal, self.signum, oldhandler) + + def readpipe_interrupted(self): + """Perform a read during which a signal will arrive. Return True if the + read is interrupted by the signal and raises an exception. Return False + if it returns normally. + """ + # Create a pipe that can be used for the read. Also clean it up + # when the test is over, since nothing else will (but see below for + # the write end). + r, w = os.pipe() + self.addCleanup(os.close, r) + + # Create another process which can send a signal to this one to try + # to interrupt the read. + ppid = os.getpid() + pid = os.fork() + + if pid == 0: + # Child code: sleep to give the parent enough time to enter the + # read() call (there's a race here, but it's really tricky to + # eliminate it); then signal the parent process. Also, sleep + # again to make it likely that the signal is delivered to the + # parent process before the child exits. If the child exits + # first, the write end of the pipe will be closed and the test + # is invalid. + try: + time.sleep(0.2) + os.kill(ppid, self.signum) + time.sleep(0.2) + finally: + # No matter what, just exit as fast as possible now. + exit_subprocess() + else: + # Parent code. + # Make sure the child is eventually reaped, else it'll be a + # zombie for the rest of the test suite run. + self.addCleanup(os.waitpid, pid, 0) + + # Close the write end of the pipe. The child has a copy, so + # it's not really closed until the child exits. We need it to + # close when the child exits so that in the non-interrupt case + # the read eventually completes, otherwise we could just close + # it *after* the test. + os.close(w) + + # Try the read and report whether it is interrupted or not to + # the caller. + try: + d = os.read(r, 1) + return False + except OSError, err: + if err.errno != errno.EINTR: + raise + return True + + def test_without_siginterrupt(self): + """If a signal handler is installed and siginterrupt is not called + at all, when that signal arrives, it interrupts a syscall that's in + progress. + """ + i = self.readpipe_interrupted() + self.assertTrue(i) + # Arrival of the signal shouldn't have changed anything. + i = self.readpipe_interrupted() + self.assertTrue(i) + + def test_siginterrupt_on(self): + """If a signal handler is installed and siginterrupt is called with + a true value for the second argument, when that signal arrives, it + interrupts a syscall that's in progress. + """ + signal.siginterrupt(self.signum, 1) + i = self.readpipe_interrupted() + self.assertTrue(i) + # Arrival of the signal shouldn't have changed anything. + i = self.readpipe_interrupted() + self.assertTrue(i) + + def test_siginterrupt_off(self): + """If a signal handler is installed and siginterrupt is called with + a false value for the second argument, when that signal arrives, it + does not interrupt a syscall that's in progress. + """ + signal.siginterrupt(self.signum, 0) + i = self.readpipe_interrupted() + self.assertFalse(i) + # Arrival of the signal shouldn't have changed anything. + i = self.readpipe_interrupted() + self.assertFalse(i) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class ItimerTest(unittest.TestCase): + def setUp(self): + self.hndl_called = False + self.hndl_count = 0 + self.itimer = None + self.old_alarm = signal.signal(signal.SIGALRM, self.sig_alrm) + + def tearDown(self): + signal.signal(signal.SIGALRM, self.old_alarm) + if self.itimer is not None: # test_itimer_exc doesn't change this attr + # just ensure that itimer is stopped + signal.setitimer(self.itimer, 0) + + def sig_alrm(self, *args): + self.hndl_called = True + if test_support.verbose: + print("SIGALRM handler invoked", args) + + def sig_vtalrm(self, *args): + self.hndl_called = True + + if self.hndl_count > 3: + # it shouldn't be here, because it should have been disabled. + raise signal.ItimerError("setitimer didn't disable ITIMER_VIRTUAL " + "timer.") + elif self.hndl_count == 3: + # disable ITIMER_VIRTUAL, this function shouldn't be called anymore + signal.setitimer(signal.ITIMER_VIRTUAL, 0) + if test_support.verbose: + print("last SIGVTALRM handler call") + + self.hndl_count += 1 + + if test_support.verbose: + print("SIGVTALRM handler invoked", args) + + def sig_prof(self, *args): + self.hndl_called = True + signal.setitimer(signal.ITIMER_PROF, 0) + + if test_support.verbose: + print("SIGPROF handler invoked", args) + + def test_itimer_exc(self): + # XXX I'm assuming -1 is an invalid itimer, but maybe some platform + # defines it ? + self.assertRaises(signal.ItimerError, signal.setitimer, -1, 0) + # Negative times are treated as zero on some platforms. + if 0: + self.assertRaises(signal.ItimerError, + signal.setitimer, signal.ITIMER_REAL, -1) + + def test_itimer_real(self): + self.itimer = signal.ITIMER_REAL + signal.setitimer(self.itimer, 1.0) + if test_support.verbose: + print("\ncall pause()...") + signal.pause() + + self.assertEqual(self.hndl_called, True) + + # Issue 3864. Unknown if this affects earlier versions of freebsd also. + @unittest.skipIf(sys.platform in ('freebsd6', 'netbsd5'), + 'itimer not reliable (does not mix well with threading) on some BSDs.') + def test_itimer_virtual(self): + self.itimer = signal.ITIMER_VIRTUAL + signal.signal(signal.SIGVTALRM, self.sig_vtalrm) + signal.setitimer(self.itimer, 0.3, 0.2) + + start_time = time.time() + while time.time() - start_time < 60.0: + # use up some virtual time by doing real work + _ = pow(12345, 67890, 10000019) + if signal.getitimer(self.itimer) == (0.0, 0.0): + break # sig_vtalrm handler stopped this itimer + else: # Issue 8424 + self.skipTest("timeout: likely cause: machine too slow or load too " + "high") + + # virtual itimer should be (0.0, 0.0) now + self.assertEqual(signal.getitimer(self.itimer), (0.0, 0.0)) + # and the handler should have been called + self.assertEqual(self.hndl_called, True) + + # Issue 3864. Unknown if this affects earlier versions of freebsd also. + @unittest.skipIf(sys.platform=='freebsd6', + 'itimer not reliable (does not mix well with threading) on freebsd6') + def test_itimer_prof(self): + self.itimer = signal.ITIMER_PROF + signal.signal(signal.SIGPROF, self.sig_prof) + signal.setitimer(self.itimer, 0.2, 0.2) + + start_time = time.time() + while time.time() - start_time < 60.0: + # do some work + _ = pow(12345, 67890, 10000019) + if signal.getitimer(self.itimer) == (0.0, 0.0): + break # sig_prof handler stopped this itimer + else: # Issue 8424 + self.skipTest("timeout: likely cause: machine too slow or load too " + "high") + + # profiling itimer should be (0.0, 0.0) now + self.assertEqual(signal.getitimer(self.itimer), (0.0, 0.0)) + # and the handler should have been called + self.assertEqual(self.hndl_called, True) + +def test_main(): + test_support.run_unittest(BasicSignalTests, InterProcessSignalTests, + WakeupFDTests, WakeupSignalTests, + SiginterruptTest, ItimerTest, + WindowsSignalTests) + + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_smtplib.py b/src/greentest/2.7/test_smtplib.py new file mode 100644 index 0000000..a5cdfaa --- /dev/null +++ b/src/greentest/2.7/test_smtplib.py @@ -0,0 +1,561 @@ +import asyncore +import email.utils +import socket +import smtpd +import smtplib +import StringIO +import sys +import time +import select + +import unittest +from test import test_support + +try: + import threading +except ImportError: + threading = None + +HOST = test_support.HOST + +def server(evt, buf, serv): + serv.listen(5) + evt.set() + try: + conn, addr = serv.accept() + except socket.timeout: + pass + else: + n = 500 + while buf and n > 0: + r, w, e = select.select([], [conn], []) + if w: + sent = conn.send(buf) + buf = buf[sent:] + + n -= 1 + + conn.close() + finally: + serv.close() + evt.set() + +@unittest.skipUnless(threading, 'Threading required for this test.') +class GeneralTests(unittest.TestCase): + + def setUp(self): + self._threads = test_support.threading_setup() + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = test_support.bind_port(self.sock) + servargs = (self.evt, "220 Hola mundo\n", self.sock) + self.thread = threading.Thread(target=server, args=servargs) + self.thread.start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + self.thread.join() + test_support.threading_cleanup(*self._threads) + + def testBasic1(self): + # connects + smtp = smtplib.SMTP(HOST, self.port) + smtp.close() + + def testBasic2(self): + # connects, include port in host name + smtp = smtplib.SMTP("%s:%s" % (HOST, self.port)) + smtp.close() + + def testLocalHostName(self): + # check that supplied local_hostname is used + smtp = smtplib.SMTP(HOST, self.port, local_hostname="testhost") + self.assertEqual(smtp.local_hostname, "testhost") + smtp.close() + + def testTimeoutDefault(self): + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + smtp = smtplib.SMTP(HOST, self.port) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(smtp.sock.gettimeout(), 30) + smtp.close() + + def testTimeoutNone(self): + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + smtp = smtplib.SMTP(HOST, self.port, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(smtp.sock.gettimeout()) + smtp.close() + + def testTimeoutValue(self): + smtp = smtplib.SMTP(HOST, self.port, timeout=30) + self.assertEqual(smtp.sock.gettimeout(), 30) + smtp.close() + + +# Test server thread using the specified SMTP server class +def debugging_server(serv, serv_evt, client_evt): + serv_evt.set() + + try: + if hasattr(select, 'poll'): + poll_fun = asyncore.poll2 + else: + poll_fun = asyncore.poll + + n = 1000 + while asyncore.socket_map and n > 0: + poll_fun(0.01, asyncore.socket_map) + + # when the client conversation is finished, it will + # set client_evt, and it's then ok to kill the server + if client_evt.is_set(): + serv.close() + break + + n -= 1 + + except socket.timeout: + pass + finally: + if not client_evt.is_set(): + # allow some time for the client to read the result + time.sleep(0.5) + serv.close() + asyncore.close_all() + serv_evt.set() + +MSG_BEGIN = '---------- MESSAGE FOLLOWS ----------\n' +MSG_END = '------------ END MESSAGE ------------\n' + +# NOTE: Some SMTP objects in the tests below are created with a non-default +# local_hostname argument to the constructor, since (on some systems) the FQDN +# lookup caused by the default local_hostname sometimes takes so long that the +# test server times out, causing the test to fail. + +# Test behavior of smtpd.DebuggingServer +@unittest.skipUnless(threading, 'Threading required for this test.') +class DebuggingServerTests(unittest.TestCase): + + def setUp(self): + # temporarily replace sys.stdout to capture DebuggingServer output + self.old_stdout = sys.stdout + self.output = StringIO.StringIO() + sys.stdout = self.output + + self._threads = test_support.threading_setup() + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = smtpd.DebuggingServer((HOST, 0), ('nowhere', -1)) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + self.thread.join() + test_support.threading_cleanup(*self._threads) + # restore sys.stdout + sys.stdout = self.old_stdout + + def testBasic(self): + # connect + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.quit() + + def testNOOP(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (250, 'Ok') + self.assertEqual(smtp.noop(), expected) + smtp.quit() + + def testRSET(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (250, 'Ok') + self.assertEqual(smtp.rset(), expected) + smtp.quit() + + def testNotImplemented(self): + # EHLO isn't implemented in DebuggingServer + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (502, 'Error: command "EHLO" not implemented') + self.assertEqual(smtp.ehlo(), expected) + smtp.quit() + + def testVRFY(self): + # VRFY isn't implemented in DebuggingServer + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (502, 'Error: command "VRFY" not implemented') + self.assertEqual(smtp.vrfy('nobody@nowhere.com'), expected) + self.assertEqual(smtp.verify('nobody@nowhere.com'), expected) + smtp.quit() + + def testSecondHELO(self): + # check that a second HELO returns a message that it's a duplicate + # (this behavior is specific to smtpd.SMTPChannel) + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.helo() + expected = (503, 'Duplicate HELO/EHLO') + self.assertEqual(smtp.helo(), expected) + smtp.quit() + + def testHELP(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + self.assertEqual(smtp.help(), 'Error: command "HELP" not implemented') + smtp.quit() + + def testSend(self): + # connect and send mail + m = 'A test message' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.sendmail('John', 'Sally', m) + # XXX(nnorwitz): this test is flaky and dies with a bad file descriptor + # in asyncore. This sleep might help, but should really be fixed + # properly by using an Event variable. + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + +class NonConnectingTests(unittest.TestCase): + + def testNotConnected(self): + # Test various operations on an unconnected SMTP object that + # should raise exceptions (at present the attempt in SMTP.send + # to reference the nonexistent 'sock' attribute of the SMTP object + # causes an AttributeError) + smtp = smtplib.SMTP() + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.ehlo) + self.assertRaises(smtplib.SMTPServerDisconnected, + smtp.send, 'test msg') + + def testNonnumericPort(self): + # check that non-numeric port raises socket.error + self.assertRaises(socket.error, smtplib.SMTP, + "localhost", "bogus") + self.assertRaises(socket.error, smtplib.SMTP, + "localhost:bogus") + + +# test response of client to a non-successful HELO message +@unittest.skipUnless(threading, 'Threading required for this test.') +class BadHELOServerTests(unittest.TestCase): + + def setUp(self): + self.old_stdout = sys.stdout + self.output = StringIO.StringIO() + sys.stdout = self.output + + self._threads = test_support.threading_setup() + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = test_support.bind_port(self.sock) + servargs = (self.evt, "199 no hello for you!\n", self.sock) + self.thread = threading.Thread(target=server, args=servargs) + self.thread.start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + self.thread.join() + test_support.threading_cleanup(*self._threads) + sys.stdout = self.old_stdout + + def testFailingHELO(self): + self.assertRaises(smtplib.SMTPConnectError, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class TooLongLineTests(unittest.TestCase): + # gevent: run on 2.7.8 + respdata = '250 OK' + ('.' * getattr(smtplib, '_MAXLINE', 1) * 2) + '\n' + + def setUp(self): + self.old_stdout = sys.stdout + self.output = StringIO.StringIO() + sys.stdout = self.output + + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = test_support.bind_port(self.sock) + servargs = (self.evt, self.respdata, self.sock) + threading.Thread(target=server, args=servargs).start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + sys.stdout = self.old_stdout + + def testLineTooLong(self): + self.assertRaises(smtplib.SMTPResponseException, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + +sim_users = {'Mr.A@somewhere.com':'John A', + 'Ms.B@somewhere.com':'Sally B', + 'Mrs.C@somewhereesle.com':'Ruth C', + } + +sim_auth = ('Mr.A@somewhere.com', 'somepassword') +sim_cram_md5_challenge = ('PENCeUxFREJoU0NnbmhNWitOMjNGNn' + 'dAZWx3b29kLmlubm9zb2Z0LmNvbT4=') +sim_auth_credentials = { + 'login': 'TXIuQUBzb21ld2hlcmUuY29t', + 'plain': 'AE1yLkFAc29tZXdoZXJlLmNvbQBzb21lcGFzc3dvcmQ=', + 'cram-md5': ('TXIUQUBZB21LD2HLCMUUY29TIDG4OWQ0MJ' + 'KWZGQ4ODNMNDA4NTGXMDRLZWMYZJDMODG1'), + } +sim_auth_login_password = 'C29TZXBHC3N3B3JK' + +sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'], + 'list-2':['Ms.B@somewhere.com',], + } + +# Simulated SMTP channel & server +class SimSMTPChannel(smtpd.SMTPChannel): + + def __init__(self, extra_features, *args, **kw): + self._extrafeatures = ''.join( + [ "250-{0}\r\n".format(x) for x in extra_features ]) + smtpd.SMTPChannel.__init__(self, *args, **kw) + + def smtp_EHLO(self, arg): + resp = ('250-testhost\r\n' + '250-EXPN\r\n' + '250-SIZE 20000000\r\n' + '250-STARTTLS\r\n' + '250-DELIVERBY\r\n') + resp = resp + self._extrafeatures + '250 HELP' + self.push(resp) + + def smtp_VRFY(self, arg): + # For max compatibility smtplib should be sending the raw address. + if arg in sim_users: + self.push('250 %s %s' % (sim_users[arg], smtplib.quoteaddr(arg))) + else: + self.push('550 No such user: %s' % arg) + + def smtp_EXPN(self, arg): + list_name = arg.lower() + if list_name in sim_lists: + user_list = sim_lists[list_name] + for n, user_email in enumerate(user_list): + quoted_addr = smtplib.quoteaddr(user_email) + if n < len(user_list) - 1: + self.push('250-%s %s' % (sim_users[user_email], quoted_addr)) + else: + self.push('250 %s %s' % (sim_users[user_email], quoted_addr)) + else: + self.push('550 No access for you!') + + def smtp_AUTH(self, arg): + if arg.strip().lower()=='cram-md5': + self.push('334 {0}'.format(sim_cram_md5_challenge)) + return + mech, auth = arg.split() + mech = mech.lower() + if mech not in sim_auth_credentials: + self.push('504 auth type unimplemented') + return + if mech == 'plain' and auth==sim_auth_credentials['plain']: + self.push('235 plain auth ok') + elif mech=='login' and auth==sim_auth_credentials['login']: + self.push('334 Password:') + else: + self.push('550 No access for you!') + + def handle_error(self): + raise + + +class SimSMTPServer(smtpd.SMTPServer): + + def __init__(self, *args, **kw): + self._extra_features = [] + smtpd.SMTPServer.__init__(self, *args, **kw) + + def handle_accept(self): + conn, addr = self.accept() + self._SMTPchannel = SimSMTPChannel(self._extra_features, + self, conn, addr) + + def process_message(self, peer, mailfrom, rcpttos, data): + pass + + def add_feature(self, feature): + self._extra_features.append(feature) + + def handle_error(self): + raise + + +# Test various SMTP & ESMTP commands/behaviors that require a simulated server +# (i.e., something with more features than DebuggingServer) +@unittest.skipUnless(threading, 'Threading required for this test.') +class SMTPSimTests(unittest.TestCase): + + def setUp(self): + self._threads = test_support.threading_setup() + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = SimSMTPServer((HOST, 0), ('nowhere', -1)) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + self.thread.join() + test_support.threading_cleanup(*self._threads) + + def testBasic(self): + # smoke test + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.quit() + + def testEHLO(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + # no features should be present before the EHLO + self.assertEqual(smtp.esmtp_features, {}) + + # features expected from the test server + expected_features = {'expn':'', + 'size': '20000000', + 'starttls': '', + 'deliverby': '', + 'help': '', + } + + smtp.ehlo() + self.assertEqual(smtp.esmtp_features, expected_features) + for k in expected_features: + self.assertTrue(smtp.has_extn(k)) + self.assertFalse(smtp.has_extn('unsupported-feature')) + smtp.quit() + + def testVRFY(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + for email, name in sim_users.items(): + expected_known = (250, '%s %s' % (name, smtplib.quoteaddr(email))) + self.assertEqual(smtp.vrfy(email), expected_known) + + u = 'nobody@nowhere.com' + expected_unknown = (550, 'No such user: %s' % u) + self.assertEqual(smtp.vrfy(u), expected_unknown) + smtp.quit() + + def testEXPN(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + for listname, members in sim_lists.items(): + users = [] + for m in members: + users.append('%s %s' % (sim_users[m], smtplib.quoteaddr(m))) + expected_known = (250, '\n'.join(users)) + self.assertEqual(smtp.expn(listname), expected_known) + + u = 'PSU-Members-List' + expected_unknown = (550, 'No access for you!') + self.assertEqual(smtp.expn(u), expected_unknown) + smtp.quit() + + def testAUTH_PLAIN(self): + self.serv.add_feature("AUTH PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + expected_auth_ok = (235, b'plain auth ok') + self.assertEqual(smtp.login(sim_auth[0], sim_auth[1]), expected_auth_ok) + + # SimSMTPChannel doesn't fully support LOGIN or CRAM-MD5 auth because they + # require a synchronous read to obtain the credentials...so instead smtpd + # sees the credential sent by smtplib's login method as an unknown command, + # which results in smtplib raising an auth error. Fortunately the error + # message contains the encoded credential, so we can partially check that it + # was generated correctly (partially, because the 'word' is uppercased in + # the error message). + + def testAUTH_LOGIN(self): + self.serv.add_feature("AUTH LOGIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + try: smtp.login(sim_auth[0], sim_auth[1]) + except smtplib.SMTPAuthenticationError as err: + if sim_auth_login_password not in str(err): + raise "expected encoded password not found in error message" + + def testAUTH_CRAM_MD5(self): + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + try: smtp.login(sim_auth[0], sim_auth[1]) + except smtplib.SMTPAuthenticationError as err: + if sim_auth_credentials['cram-md5'] not in str(err): + raise "expected encoded credentials not found in error message" + + #TODO: add tests for correct AUTH method fallback now that the + #test infrastructure can support it. + + def test_quit_resets_greeting(self): + smtp = smtplib.SMTP(HOST, self.port, + local_hostname='localhost', + timeout=15) + code, message = smtp.ehlo() + self.assertEqual(code, 250) + self.assertIn('size', smtp.esmtp_features) + smtp.quit() + self.assertNotIn('size', smtp.esmtp_features) + smtp.connect(HOST, self.port) + self.assertNotIn('size', smtp.esmtp_features) + smtp.ehlo_or_helo_if_needed() + self.assertIn('size', smtp.esmtp_features) + smtp.quit() + + +def test_main(verbose=None): + test_support.run_unittest(GeneralTests, DebuggingServerTests, + NonConnectingTests, + BadHELOServerTests, SMTPSimTests, + TooLongLineTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7/test_socket.py b/src/greentest/2.7/test_socket.py new file mode 100644 index 0000000..8c594a1 --- /dev/null +++ b/src/greentest/2.7/test_socket.py @@ -0,0 +1,1808 @@ +import unittest +from test import test_support + +import errno +import itertools +import socket +import select +import time +import traceback +import Queue +import sys +import os +import array +import contextlib +import signal +import math +import weakref +try: + import _socket +except ImportError: + _socket = None + + +def try_address(host, port=0, family=socket.AF_INET): + """Try to bind a socket on the given host:port and return True + if that has been possible.""" + try: + sock = socket.socket(family, socket.SOCK_STREAM) + sock.bind((host, port)) + except (socket.error, socket.gaierror): + return False + else: + sock.close() + return True + +HOST = test_support.HOST +MSG = b'Michael Gilfix was here\n' +SUPPORTS_IPV6 = socket.has_ipv6 and try_address('::1', family=socket.AF_INET6) + +try: + import thread + import threading +except ImportError: + thread = None + threading = None + +HOST = test_support.HOST +MSG = 'Michael Gilfix was here\n' + +class SocketTCPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = test_support.bind_port(self.serv) + self.serv.listen(1) + + def tearDown(self): + self.serv.close() + self.serv = None + +class SocketUDPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.port = test_support.bind_port(self.serv) + + def tearDown(self): + self.serv.close() + self.serv = None + +class ThreadableTest: + """Threadable Test class + + The ThreadableTest class makes it easy to create a threaded + client/server pair from an existing unit test. To create a + new threaded class from an existing unit test, use multiple + inheritance: + + class NewClass (OldClass, ThreadableTest): + pass + + This class defines two new fixture functions with obvious + purposes for overriding: + + clientSetUp () + clientTearDown () + + Any new test functions within the class must then define + tests in pairs, where the test name is preceeded with a + '_' to indicate the client portion of the test. Ex: + + def testFoo(self): + # Server portion + + def _testFoo(self): + # Client portion + + Any exceptions raised by the clients during their tests + are caught and transferred to the main thread to alert + the testing framework. + + Note, the server setup function cannot call any blocking + functions that rely on the client thread during setup, + unless serverExplicitReady() is called just before + the blocking call (such as in setting up a client/server + connection and performing the accept() in setUp(). + """ + + def __init__(self): + # Swap the true setup function + self.__setUp = self.setUp + self.__tearDown = self.tearDown + self.setUp = self._setUp + self.tearDown = self._tearDown + + def serverExplicitReady(self): + """This method allows the server to explicitly indicate that + it wants the client thread to proceed. This is useful if the + server is about to execute a blocking routine that is + dependent upon the client thread during its setup routine.""" + self.server_ready.set() + + def _setUp(self): + self.server_ready = threading.Event() + self.client_ready = threading.Event() + self.done = threading.Event() + self.queue = Queue.Queue(1) + + # Do some munging to start the client test. + methodname = self.id() + i = methodname.rfind('.') + methodname = methodname[i+1:] + test_method = getattr(self, '_' + methodname) + self.client_thread = thread.start_new_thread( + self.clientRun, (test_method,)) + + self.__setUp() + if not self.server_ready.is_set(): + self.server_ready.set() + self.client_ready.wait() + + def _tearDown(self): + self.__tearDown() + self.done.wait() + + if not self.queue.empty(): + msg = self.queue.get() + self.fail(msg) + + def clientRun(self, test_func): + self.server_ready.wait() + self.clientSetUp() + self.client_ready.set() + if not callable(test_func): + raise TypeError("test_func must be a callable function.") + try: + test_func() + except Exception, strerror: + self.queue.put(strerror) + self.clientTearDown() + + def clientSetUp(self): + raise NotImplementedError("clientSetUp must be implemented.") + + def clientTearDown(self): + self.done.set() + thread.exit() + +class ThreadedTCPSocketTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketUDPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class SocketConnectedTest(ThreadedTCPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + ThreadedTCPSocketTest.setUp(self) + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + ThreadedTCPSocketTest.tearDown(self) + + def clientSetUp(self): + ThreadedTCPSocketTest.clientSetUp(self) + self.cli.connect((HOST, self.port)) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + ThreadedTCPSocketTest.clientTearDown(self) + +class SocketPairTest(unittest.TestCase, ThreadableTest): + + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.serv, self.cli = socket.socketpair() + + def tearDown(self): + self.serv.close() + self.serv = None + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +####################################################################### +## Begin Tests + +class GeneralModuleTests(unittest.TestCase): + + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_csocket_repr(self): + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + try: + expected = ('' + % (s.fileno(), s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + finally: + s.close() + expected = ('' + % (s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + + def test_weakref(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + p = weakref.proxy(s) + self.assertEqual(p.fileno(), s.fileno()) + s.close() + s = None + try: + p.fileno() + except ReferenceError: + pass + else: + self.fail('Socket proxy still exists') + + def test_weakref__sock(self): + s = socket.socket()._sock + w = weakref.ref(s) + self.assertIs(w(), s) + del s + test_support.gc_collect() + self.assertIsNone(w()) + + def testSocketError(self): + # Testing socket module exceptions + def raise_error(*args, **kwargs): + raise socket.error + def raise_herror(*args, **kwargs): + raise socket.herror + def raise_gaierror(*args, **kwargs): + raise socket.gaierror + self.assertRaises(socket.error, raise_error, + "Error raising socket exception.") + self.assertRaises(socket.error, raise_herror, + "Error raising socket exception.") + self.assertRaises(socket.error, raise_gaierror, + "Error raising socket exception.") + + def testSendtoErrors(self): + # Testing that sendto doens't masks failures. See #10169. + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind(('', 0)) + sockname = s.getsockname() + # 2 args + with self.assertRaises(UnicodeEncodeError): + s.sendto(u'\u2620', sockname) + with self.assertRaises(TypeError) as cm: + s.sendto(5j, sockname) + self.assertIn('not complex', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', None) + self.assertIn('not NoneType', str(cm.exception)) + # 3 args + with self.assertRaises(UnicodeEncodeError): + s.sendto(u'\u2620', 0, sockname) + with self.assertRaises(TypeError) as cm: + s.sendto(5j, 0, sockname) + self.assertIn('not complex', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', 0, None) + self.assertIn('not NoneType', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', 'bar', sockname) + self.assertIn('an integer is required', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', None, None) + self.assertIn('an integer is required', str(cm.exception)) + # wrong number of args + with self.assertRaises(TypeError) as cm: + s.sendto('foo') + self.assertIn('(1 given)', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', 0, sockname, 4) + self.assertIn('(4 given)', str(cm.exception)) + + + def testCrucialConstants(self): + # Testing for mission critical constants + socket.AF_INET + socket.SOCK_STREAM + socket.SOCK_DGRAM + socket.SOCK_RAW + socket.SOCK_RDM + socket.SOCK_SEQPACKET + socket.SOL_SOCKET + socket.SO_REUSEADDR + + def testHostnameRes(self): + # Testing hostname resolution mechanisms + hostname = socket.gethostname() + try: + ip = socket.gethostbyname(hostname) + except socket.error: + # Probably name lookup wasn't set up right; skip this test + self.skipTest('name lookup failure') + self.assertTrue(ip.find('.') >= 0, "Error resolving host to ip.") + try: + hname, aliases, ipaddrs = socket.gethostbyaddr(ip) + except socket.error: + # Probably a similar problem as above; skip this test + self.skipTest('address lookup failure') + all_host_names = [hostname, hname] + aliases + fqhn = socket.getfqdn(ip) + if not fqhn in all_host_names: + self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') + def testRefCountGetNameInfo(self): + # Testing reference count for getnameinfo + try: + # On some versions, this loses a reference + orig = sys.getrefcount(__name__) + socket.getnameinfo(__name__,0) + except TypeError: + self.assertEqual(sys.getrefcount(__name__), orig, + "socket.getnameinfo loses a reference") + + def testInterpreterCrash(self): + # Making sure getnameinfo doesn't crash the interpreter + try: + # On some versions, this crashes the interpreter. + socket.getnameinfo(('x', 0, 0, 0), 0) + except socket.error: + pass + + def testNtoH(self): + # This just checks that htons etc. are their own inverse, + # when looking at the lower 16 or 32 bits. + sizes = {socket.htonl: 32, socket.ntohl: 32, + socket.htons: 16, socket.ntohs: 16} + for func, size in sizes.items(): + mask = (1L<= _testcapi.ULONG_MAX: + self.skipTest('needs UINT_MAX < ULONG_MAX') + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.serv.setblocking(_testcapi.UINT_MAX + 1) + self.assertIsNone(self.serv.gettimeout()) + + _testSetBlocking_overflow = test_support.cpython_only(_testSetBlocking) + + def testAccept(self): + # Testing non-blocking accept + self.serv.setblocking(0) + try: + conn, addr = self.serv.accept() + except socket.error: + pass + else: + self.fail("Error trying to do non-blocking accept.") + read, write, err = select.select([self.serv], [], []) + if self.serv in read: + conn, addr = self.serv.accept() + conn.close() + else: + self.fail("Error trying to do accept after select.") + + def _testAccept(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + + def testConnect(self): + # Testing non-blocking connect + conn, addr = self.serv.accept() + conn.close() + + def _testConnect(self): + self.cli.settimeout(10) + self.cli.connect((HOST, self.port)) + + def testRecv(self): + # Testing non-blocking recv + conn, addr = self.serv.accept() + conn.setblocking(0) + try: + msg = conn.recv(len(MSG)) + except socket.error: + pass + else: + self.fail("Error trying to do non-blocking recv.") + read, write, err = select.select([conn], [], []) + if conn in read: + msg = conn.recv(len(MSG)) + conn.close() + self.assertEqual(msg, MSG) + else: + self.fail("Error during select call to non-blocking socket.") + + def _testRecv(self): + self.cli.connect((HOST, self.port)) + time.sleep(0.1) + self.cli.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class FileObjectClassTestCase(SocketConnectedTest): + + bufsize = -1 # Use default buffer size + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def setUp(self): + SocketConnectedTest.setUp(self) + self.serv_file = self.cli_conn.makefile('rb', self.bufsize) + + def tearDown(self): + self.serv_file.close() + self.assertTrue(self.serv_file.closed) + SocketConnectedTest.tearDown(self) + self.serv_file = None + + def clientSetUp(self): + SocketConnectedTest.clientSetUp(self) + self.cli_file = self.serv_conn.makefile('wb') + + def clientTearDown(self): + self.cli_file.close() + self.assertTrue(self.cli_file.closed) + self.cli_file = None + SocketConnectedTest.clientTearDown(self) + + def testSmallRead(self): + # Performing small file read test + first_seg = self.serv_file.read(len(MSG)-3) + second_seg = self.serv_file.read(3) + msg = first_seg + second_seg + self.assertEqual(msg, MSG) + + def _testSmallRead(self): + self.cli_file.write(MSG) + self.cli_file.flush() + + def testFullRead(self): + # read until EOF + msg = self.serv_file.read() + self.assertEqual(msg, MSG) + + def _testFullRead(self): + self.cli_file.write(MSG) + self.cli_file.close() + + def testUnbufferedRead(self): + # Performing unbuffered file read test + buf = '' + while 1: + char = self.serv_file.read(1) + if not char: + break + buf += char + self.assertEqual(buf, MSG) + + def _testUnbufferedRead(self): + self.cli_file.write(MSG) + self.cli_file.flush() + + def testReadline(self): + # Performing file readline test + line = self.serv_file.readline() + self.assertEqual(line, MSG) + + def _testReadline(self): + self.cli_file.write(MSG) + self.cli_file.flush() + + def testReadlineAfterRead(self): + a_baloo_is = self.serv_file.read(len("A baloo is")) + self.assertEqual("A baloo is", a_baloo_is) + _a_bear = self.serv_file.read(len(" a bear")) + self.assertEqual(" a bear", _a_bear) + line = self.serv_file.readline() + self.assertEqual("\n", line) + line = self.serv_file.readline() + self.assertEqual("A BALOO IS A BEAR.\n", line) + line = self.serv_file.readline() + self.assertEqual(MSG, line) + + def _testReadlineAfterRead(self): + self.cli_file.write("A baloo is a bear\n") + self.cli_file.write("A BALOO IS A BEAR.\n") + self.cli_file.write(MSG) + self.cli_file.flush() + + def testReadlineAfterReadNoNewline(self): + end_of_ = self.serv_file.read(len("End Of ")) + self.assertEqual("End Of ", end_of_) + line = self.serv_file.readline() + self.assertEqual("Line", line) + + def _testReadlineAfterReadNoNewline(self): + self.cli_file.write("End Of Line") + + def testClosedAttr(self): + self.assertTrue(not self.serv_file.closed) + + def _testClosedAttr(self): + self.assertTrue(not self.cli_file.closed) + + +class FileObjectInterruptedTestCase(unittest.TestCase): + """Test that the file object correctly handles EINTR internally.""" + + class MockSocket(object): + def __init__(self, recv_funcs=()): + # A generator that returns callables that we'll call for each + # call to recv(). + self._recv_step = iter(recv_funcs) + + def recv(self, size): + return self._recv_step.next()() + + @staticmethod + def _raise_eintr(): + raise socket.error(errno.EINTR) + + def _test_readline(self, size=-1, **kwargs): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : "This is the first line\nAnd the sec", + self._raise_eintr, + lambda : "ond line is here\n", + lambda : "", + ]) + fo = socket._fileobject(mock_sock, **kwargs) + self.assertEqual(fo.readline(size), "This is the first line\n") + self.assertEqual(fo.readline(size), "And the second line is here\n") + + def _test_read(self, size=-1, **kwargs): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : "This is the first line\nAnd the sec", + self._raise_eintr, + lambda : "ond line is here\n", + lambda : "", + ]) + fo = socket._fileobject(mock_sock, **kwargs) + self.assertEqual(fo.read(size), "This is the first line\n" + "And the second line is here\n") + + def test_default(self): + self._test_readline() + self._test_readline(size=100) + self._test_read() + self._test_read(size=100) + + def test_with_1k_buffer(self): + self._test_readline(bufsize=1024) + self._test_readline(size=100, bufsize=1024) + self._test_read(bufsize=1024) + self._test_read(size=100, bufsize=1024) + + def _test_readline_no_buffer(self, size=-1): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : "aa", + lambda : "\n", + lambda : "BB", + self._raise_eintr, + lambda : "bb", + lambda : "", + ]) + fo = socket._fileobject(mock_sock, bufsize=0) + self.assertEqual(fo.readline(size), "aa\n") + self.assertEqual(fo.readline(size), "BBbb") + + def test_no_buffer(self): + self._test_readline_no_buffer() + self._test_readline_no_buffer(size=4) + self._test_read(bufsize=0) + self._test_read(size=100, bufsize=0) + + +class UnbufferedFileObjectClassTestCase(FileObjectClassTestCase): + + """Repeat the tests from FileObjectClassTestCase with bufsize==0. + + In this case (and in this case only), it should be possible to + create a file object, read a line from it, create another file + object, read another line from it, without loss of data in the + first file object's buffer. Note that httplib relies on this + when reading multiple requests from the same socket.""" + + bufsize = 0 # Use unbuffered mode + + def testUnbufferedReadline(self): + # Read a line, create a new file object, read another line with it + line = self.serv_file.readline() # first line + self.assertEqual(line, "A. " + MSG) # first line + self.serv_file = self.cli_conn.makefile('rb', 0) + line = self.serv_file.readline() # second line + self.assertEqual(line, "B. " + MSG) # second line + + def _testUnbufferedReadline(self): + self.cli_file.write("A. " + MSG) + self.cli_file.write("B. " + MSG) + self.cli_file.flush() + +class LineBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 1 # Default-buffered for reading; line-buffered for writing + + class SocketMemo(object): + """A wrapper to keep track of sent data, needed to examine write behaviour""" + def __init__(self, sock): + self._sock = sock + self.sent = [] + + def send(self, data, flags=0): + n = self._sock.send(data, flags) + self.sent.append(data[:n]) + return n + + def sendall(self, data, flags=0): + self._sock.sendall(data, flags) + self.sent.append(data) + + def __getattr__(self, attr): + return getattr(self._sock, attr) + + def getsent(self): + return [e.tobytes() if isinstance(e, memoryview) else e for e in self.sent] + + def setUp(self): + FileObjectClassTestCase.setUp(self) + self.serv_file._sock = self.SocketMemo(self.serv_file._sock) + + def testLinebufferedWrite(self): + # Write two lines, in small chunks + msg = MSG.strip() + print >> self.serv_file, msg, + print >> self.serv_file, msg + + # second line: + print >> self.serv_file, msg, + print >> self.serv_file, msg, + print >> self.serv_file, msg + + # third line + print >> self.serv_file, '' + + self.serv_file.flush() + + msg1 = "%s %s\n"%(msg, msg) + msg2 = "%s %s %s\n"%(msg, msg, msg) + msg3 = "\n" + self.assertEqual(self.serv_file._sock.getsent(), [msg1, msg2, msg3]) + + def _testLinebufferedWrite(self): + msg = MSG.strip() + msg1 = "%s %s\n"%(msg, msg) + msg2 = "%s %s %s\n"%(msg, msg, msg) + msg3 = "\n" + l1 = self.cli_file.readline() + self.assertEqual(l1, msg1) + l2 = self.cli_file.readline() + self.assertEqual(l2, msg2) + l3 = self.cli_file.readline() + self.assertEqual(l3, msg3) + + +class SmallBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 2 # Exercise the buffering code + + +class NetworkConnectionTest(object): + """Prove network connection.""" + def clientSetUp(self): + # We're inherited below by BasicTCPTest2, which also inherits + # BasicTCPTest, which defines self.port referenced below. + self.cli = socket.create_connection((HOST, self.port)) + self.serv_conn = self.cli + +class BasicTCPTest2(NetworkConnectionTest, BasicTCPTest): + """Tests that NetworkConnection does not break existing TCP functionality. + """ + +class NetworkConnectionNoServer(unittest.TestCase): + class MockSocket(socket.socket): + def connect(self, *args): + raise socket.timeout('timed out') + + @contextlib.contextmanager + def mocked_socket_module(self): + """Return a socket which times out on connect""" + old_socket = socket.socket + import gevent.socket + old_g_socket = gevent.socket.socket + socket.socket = self.MockSocket + gevent.socket.socket = self.MockSocket + try: + yield + finally: + socket.socket = old_socket + gevent.socket.socket = old_g_socket + + def test_connect(self): + port = test_support.find_unused_port() + cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(cli.close) + with self.assertRaises(socket.error) as cm: + cli.connect((HOST, port)) + self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + + def test_create_connection(self): + # Issue #9792: errors raised by create_connection() should have + # a proper errno attribute. + port = test_support.find_unused_port() + with self.assertRaises(socket.error) as cm: + socket.create_connection((HOST, port)) + + # Issue #16257: create_connection() calls getaddrinfo() against + # 'localhost'. This may result in an IPV6 addr being returned + # as well as an IPV4 one: + # >>> socket.getaddrinfo('localhost', port, 0, SOCK_STREAM) + # >>> [(2, 2, 0, '', ('127.0.0.1', 41230)), + # (26, 2, 0, '', ('::1', 41230, 0, 0))] + # + # create_connection() enumerates through all the addresses returned + # and if it doesn't successfully bind to any of them, it propagates + # the last exception it encountered. + # + # On Solaris, ENETUNREACH is returned in this circumstance instead + # of ECONNREFUSED. So, if that errno exists, add it to our list of + # expected errnos. + expected_errnos = [ errno.ECONNREFUSED, ] + if hasattr(errno, 'ENETUNREACH'): + expected_errnos.append(errno.ENETUNREACH) + + self.assertIn(cm.exception.errno, expected_errnos) + + def test_create_connection_timeout(self): + # Issue #9792: create_connection() should not recast timeout errors + # as generic socket errors. + with self.mocked_socket_module(): + with self.assertRaises(socket.timeout): + socket.create_connection((HOST, 1234)) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.source_port = test_support.find_unused_port() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def _justAccept(self): + conn, addr = self.serv.accept() + conn.close() + + testFamily = _justAccept + def _testFamily(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.family, 2) + + testSourceAddress = _justAccept + def _testSourceAddress(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30, + source_address=('', self.source_port)) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.getsockname()[1], self.source_port) + # The port number being used is sufficient to show that the bind() + # call happened. + + testTimeoutDefault = _justAccept + def _testTimeoutDefault(self): + # passing no explicit timeout uses socket's global default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(42) + try: + self.cli = socket.create_connection((HOST, self.port)) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), 42) + + testTimeoutNone = _justAccept + def _testTimeoutNone(self): + # None timeout means the same as sock.settimeout(None) + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + self.cli = socket.create_connection((HOST, self.port), timeout=None) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), None) + + testTimeoutValueNamed = _justAccept + def _testTimeoutValueNamed(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.assertEqual(self.cli.gettimeout(), 30) + + testTimeoutValueNonamed = _justAccept + def _testTimeoutValueNonamed(self): + self.cli = socket.create_connection((HOST, self.port), 30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.gettimeout(), 30) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionBehaviourTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def testInsideTimeout(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + time.sleep(3) + conn.send("done!") + testOutsideTimeout = testInsideTimeout + + def _testInsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port)) + data = sock.recv(5) + self.assertEqual(data, "done!") + + def _testOutsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port), timeout=1) + self.assertRaises(socket.timeout, lambda: sock.recv(5)) + + +class Urllib2FileobjectTest(unittest.TestCase): + + # urllib2.HTTPHandler has "borrowed" socket._fileobject, and requires that + # it close the socket if the close c'tor argument is true + + def testClose(self): + class MockSocket: + closed = False + def flush(self): pass + def close(self): self.closed = True + + # must not close unless we request it: the original use of _fileobject + # by module socket requires that the underlying socket not be closed until + # the _socketobject that created the _fileobject is closed + s = MockSocket() + f = socket._fileobject(s) + f.close() + self.assertTrue(not s.closed) + + s = MockSocket() + f = socket._fileobject(s, close=True) + f.close() + self.assertTrue(s.closed) + +class TCPTimeoutTest(SocketTCPTest): + + def testTCPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.accept() + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (TCP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of error (TCP)") + except socket.error: + ok = True + except: + self.fail("caught unexpected exception (TCP)") + if not ok: + self.fail("accept() returned success when we did not expect it") + + @unittest.skipUnless(hasattr(signal, 'alarm'), + 'test needs signal.alarm()') + def testInterruptedTimeout(self): + # XXX I don't know how to do this test on MSWindows or any other + # plaform that doesn't support signal.alarm() or os.kill(), though + # the bug should have existed on all platforms. + self.serv.settimeout(5.0) # must be longer than alarm + class Alarm(Exception): + pass + def alarm_handler(signal, frame): + raise Alarm + old_alarm = signal.signal(signal.SIGALRM, alarm_handler) + try: + signal.alarm(2) # POSIX allows alarm to be up to 1 second early + try: + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of Alarm") + except Alarm: + pass + except: + self.fail("caught other exception instead of Alarm:" + " %s(%s):\n%s" % + (sys.exc_info()[:2] + (traceback.format_exc(),))) + else: + self.fail("nothing caught") + finally: + signal.alarm(0) # shut off alarm + except Alarm: + self.fail("got Alarm in wrong place") + finally: + # no alarm can be pending. Safe to restore old handler. + signal.signal(signal.SIGALRM, old_alarm) + +class UDPTimeoutTest(SocketUDPTest): + + def testUDPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.recv(1024) + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (UDP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.recv(1024) + except socket.timeout: + self.fail("caught timeout instead of error (UDP)") + except socket.error: + ok = True + except: + self.fail("caught unexpected exception (UDP)") + if not ok: + self.fail("recv() returned success when we did not expect it") + +class TestExceptions(unittest.TestCase): + + def testExceptionTree(self): + self.assertTrue(issubclass(socket.error, Exception)) + self.assertTrue(issubclass(socket.herror, socket.error)) + self.assertTrue(issubclass(socket.gaierror, socket.error)) + self.assertTrue(issubclass(socket.timeout, socket.error)) + +@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +class TestLinuxAbstractNamespace(unittest.TestCase): + + UNIX_PATH_MAX = 108 + + def testLinuxAbstractNamespace(self): + address = "\x00python-test-hello\x00\xff" + s1 = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s1.bind(address) + s1.listen(1) + s2 = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s2.connect(s1.getsockname()) + s1.accept() + self.assertEqual(s1.getsockname(), address) + self.assertEqual(s2.getpeername(), address) + + def testMaxName(self): + address = "\x00" + "h" * (self.UNIX_PATH_MAX - 1) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.bind(address) + self.assertEqual(s.getsockname(), address) + + def testNameOverflow(self): + address = "\x00" + "h" * self.UNIX_PATH_MAX + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.assertRaises(socket.error, s.bind, address) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BufferIOTest(SocketConnectedTest): + """ + Test the buffer versions of socket.recv() and socket.send(). + """ + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecvIntoArray(self): + buf = array.array('c', ' '*1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf.tostring()[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvIntoArray(self): + with test_support.check_py3k_warnings(): + buf = buffer(MSG) + self.serv_conn.send(buf) + + def testRecvIntoBytearray(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoBytearray = _testRecvIntoArray + + def testRecvIntoMemoryview(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoMemoryview = _testRecvIntoArray + + def testRecvFromIntoArray(self): + buf = array.array('c', ' '*1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf.tostring()[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvFromIntoArray(self): + with test_support.check_py3k_warnings(): + buf = buffer(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoBytearray(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoBytearray = _testRecvFromIntoArray + + def testRecvFromIntoMemoryview(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoMemoryview = _testRecvFromIntoArray + + def testRecvFromIntoSmallBuffer(self): + # See issue #20246. + buf = bytearray(8) + self.assertRaises(ValueError, self.cli_conn.recvfrom_into, buf, 1024) + + def _testRecvFromIntoSmallBuffer(self): + with test_support.check_py3k_warnings(): + buf = buffer(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoEmptyBuffer(self): + buf = bytearray() + self.cli_conn.recvfrom_into(buf) + self.cli_conn.recvfrom_into(buf, 0) + + _testRecvFromIntoEmptyBuffer = _testRecvFromIntoArray + + +TIPC_STYPE = 2000 +TIPC_LOWER = 200 +TIPC_UPPER = 210 + +def isTipcAvailable(): + """Check if the TIPC module is loaded + + The TIPC module is not loaded automatically on Ubuntu and probably + other Linux distros. + """ + if not hasattr(socket, "AF_TIPC"): + return False + if not os.path.isfile("/proc/modules"): + return False + with open("/proc/modules") as f: + for line in f: + if line.startswith("tipc "): + return True + return False + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCTest(unittest.TestCase): + def testRDM(self): + srv = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + cli = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + srv.bind(srvaddr) + + sendaddr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + (TIPC_UPPER - TIPC_LOWER) / 2, 0) + cli.sendto(MSG, sendaddr) + + msg, recvaddr = srv.recvfrom(1024) + + self.assertEqual(cli.getsockname(), recvaddr) + self.assertEqual(msg, MSG) + + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCThreadableTest(unittest.TestCase, ThreadableTest): + def __init__(self, methodName = 'runTest'): + unittest.TestCase.__init__(self, methodName = methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.srv = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + self.srv.bind(srvaddr) + self.srv.listen(5) + self.serverExplicitReady() + self.conn, self.connaddr = self.srv.accept() + + def clientSetUp(self): + # The is a hittable race between serverExplicitReady() and the + # accept() call; sleep a little while to avoid it, otherwise + # we could get an exception + time.sleep(0.1) + self.cli = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + addr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + (TIPC_UPPER - TIPC_LOWER) / 2, 0) + self.cli.connect(addr) + self.cliaddr = self.cli.getsockname() + + def testStream(self): + msg = self.conn.recv(1024) + self.assertEqual(msg, MSG) + self.assertEqual(self.cliaddr, self.connaddr) + + def _testStream(self): + self.cli.send(MSG) + self.cli.close() + + +def test_main(): + tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest, + TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, + UDPTimeoutTest ] + + tests.extend([ + NonBlockingTCPTests, + FileObjectClassTestCase, + FileObjectInterruptedTestCase, + UnbufferedFileObjectClassTestCase, + LineBufferedFileObjectClassTestCase, + SmallBufferedFileObjectClassTestCase, + Urllib2FileobjectTest, + NetworkConnectionNoServer, + NetworkConnectionAttributesTest, + NetworkConnectionBehaviourTest, + ]) + tests.append(BasicSocketPairTest) + tests.append(TestLinuxAbstractNamespace) + tests.extend([TIPCTest, TIPCThreadableTest]) + + thread_info = test_support.threading_setup() + test_support.run_unittest(*tests) + test_support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_socketserver.py b/src/greentest/2.7/test_socketserver.py new file mode 100644 index 0000000..714ca4a --- /dev/null +++ b/src/greentest/2.7/test_socketserver.py @@ -0,0 +1,338 @@ +""" +Test suite for SocketServer.py. +""" + +import contextlib +import imp +import os +import select +import signal +import socket +import select +import errno +import tempfile +import unittest +import SocketServer + +import test.test_support +from test.test_support import reap_children, reap_threads, verbose +try: + import threading +except ImportError: + threading = None + +test.test_support.requires("network") + +TEST_STR = "hello world\n" +HOST = test.test_support.HOST + +HAVE_UNIX_SOCKETS = hasattr(socket, "AF_UNIX") +requires_unix_sockets = unittest.skipUnless(HAVE_UNIX_SOCKETS, + 'requires Unix sockets') +HAVE_FORKING = hasattr(os, "fork") and os.name != "os2" +requires_forking = unittest.skipUnless(HAVE_FORKING, 'requires forking') + +def signal_alarm(n): + """Call signal.alarm when it exists (i.e. not on Windows).""" + if hasattr(signal, 'alarm'): + signal.alarm(n) + +# Remember real select() to avoid interferences with mocking +_real_select = select.select + +def receive(sock, n, timeout=20): + r, w, x = _real_select([sock], [], [], timeout) + if sock in r: + return sock.recv(n) + else: + raise RuntimeError, "timed out on %r" % (sock,) + +if HAVE_UNIX_SOCKETS: + class ForkingUnixStreamServer(SocketServer.ForkingMixIn, + SocketServer.UnixStreamServer): + pass + + class ForkingUnixDatagramServer(SocketServer.ForkingMixIn, + SocketServer.UnixDatagramServer): + pass + + +@contextlib.contextmanager +def simple_subprocess(testcase): + pid = os.fork() + if pid == 0: + # Don't raise an exception; it would be caught by the test harness. + os._exit(72) + yield None + pid2, status = os.waitpid(pid, 0) + testcase.assertEqual(pid2, pid) + testcase.assertEqual(72 << 8, status) + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class SocketServerTest(unittest.TestCase): + """Test all socket servers.""" + + def setUp(self): + signal_alarm(60) # Kill deadlocks after 60 seconds. + self.port_seed = 0 + self.test_files = [] + + def tearDown(self): + signal_alarm(0) # Didn't deadlock. + reap_children() + + for fn in self.test_files: + try: + os.remove(fn) + except os.error: + pass + self.test_files[:] = [] + + def pickaddr(self, proto): + if proto == socket.AF_INET: + return (HOST, 0) + else: + # XXX: We need a way to tell AF_UNIX to pick its own name + # like AF_INET provides port==0. + dir = None + if os.name == 'os2': + dir = '\socket' + fn = tempfile.mktemp(prefix='unix_socket.', dir=dir) + if os.name == 'os2': + # AF_UNIX socket names on OS/2 require a specific prefix + # which can't include a drive letter and must also use + # backslashes as directory separators + if fn[1] == ':': + fn = fn[2:] + if fn[0] in (os.sep, os.altsep): + fn = fn[1:] + if os.sep == '/': + fn = fn.replace(os.sep, os.altsep) + else: + fn = fn.replace(os.altsep, os.sep) + self.test_files.append(fn) + return fn + + def make_server(self, addr, svrcls, hdlrbase): + class MyServer(svrcls): + def handle_error(self, request, client_address): + self.close_request(request) + self.server_close() + raise + + class MyHandler(hdlrbase): + def handle(self): + line = self.rfile.readline() + self.wfile.write(line) + + if verbose: print "creating server" + server = MyServer(addr, MyHandler) + self.assertEqual(server.server_address, server.socket.getsockname()) + return server + + @reap_threads + def run_server(self, svrcls, hdlrbase, testfunc): + server = self.make_server(self.pickaddr(svrcls.address_family), + svrcls, hdlrbase) + # We had the OS pick a port, so pull the real address out of + # the server. + addr = server.server_address + if verbose: + print "server created" + print "ADDR =", addr + print "CLASS =", svrcls + t = threading.Thread( + name='%s serving' % svrcls, + target=server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval':0.01}) + t.daemon = True # In case this function raises. + t.start() + if verbose: print "server running" + for i in range(3): + if verbose: print "test client", i + testfunc(svrcls.address_family, addr) + if verbose: print "waiting for server" + server.shutdown() + t.join() + server.server_close() + self.assertRaises(socket.error, server.socket.fileno) + if verbose: print "done" + + def stream_examine(self, proto, addr): + s = socket.socket(proto, socket.SOCK_STREAM) + s.connect(addr) + s.sendall(TEST_STR) + buf = data = receive(s, 100) + while data and '\n' not in buf: + data = receive(s, 100) + buf += data + self.assertEqual(buf, TEST_STR) + s.close() + + def dgram_examine(self, proto, addr): + s = socket.socket(proto, socket.SOCK_DGRAM) + s.sendto(TEST_STR, addr) + buf = data = receive(s, 100) + while data and '\n' not in buf: + data = receive(s, 100) + buf += data + self.assertEqual(buf, TEST_STR) + s.close() + + def test_TCPServer(self): + self.run_server(SocketServer.TCPServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + def test_ThreadingTCPServer(self): + self.run_server(SocketServer.ThreadingTCPServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + @requires_forking + def test_ForkingTCPServer(self): + with simple_subprocess(self): + self.run_server(SocketServer.ForkingTCPServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + def test_UnixStreamServer(self): + self.run_server(SocketServer.UnixStreamServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + def test_ThreadingUnixStreamServer(self): + self.run_server(SocketServer.ThreadingUnixStreamServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + @requires_forking + def test_ForkingUnixStreamServer(self): + with simple_subprocess(self): + self.run_server(ForkingUnixStreamServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + def test_UDPServer(self): + self.run_server(SocketServer.UDPServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + def test_ThreadingUDPServer(self): + self.run_server(SocketServer.ThreadingUDPServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + @requires_forking + def test_ForkingUDPServer(self): + with simple_subprocess(self): + self.run_server(SocketServer.ForkingUDPServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + @contextlib.contextmanager + def mocked_select_module(self): + """Mocks the select.select() call to raise EINTR for first call""" + old_select = select.select + + class MockSelect: + def __init__(self): + self.called = 0 + + def __call__(self, *args): + self.called += 1 + if self.called == 1: + # raise the exception on first call + raise select.error(errno.EINTR, os.strerror(errno.EINTR)) + else: + # Return real select value for consecutive calls + return old_select(*args) + + select.select = MockSelect() + try: + yield select.select + finally: + select.select = old_select + + def test_InterruptServerSelectCall(self): + with self.mocked_select_module() as mock_select: + pid = self.run_server(SocketServer.TCPServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + # Make sure select was called again: + self.assertGreater(mock_select.called, 1) + + # Alas, on Linux (at least) recvfrom() doesn't return a meaningful + # client address so this cannot work: + + # @requires_unix_sockets + # def test_UnixDatagramServer(self): + # self.run_server(SocketServer.UnixDatagramServer, + # SocketServer.DatagramRequestHandler, + # self.dgram_examine) + # + # @requires_unix_sockets + # def test_ThreadingUnixDatagramServer(self): + # self.run_server(SocketServer.ThreadingUnixDatagramServer, + # SocketServer.DatagramRequestHandler, + # self.dgram_examine) + # + # @requires_unix_sockets + # @requires_forking + # def test_ForkingUnixDatagramServer(self): + # self.run_server(SocketServer.ForkingUnixDatagramServer, + # SocketServer.DatagramRequestHandler, + # self.dgram_examine) + + @reap_threads + def test_shutdown(self): + # Issue #2302: shutdown() should always succeed in making an + # other thread leave serve_forever(). + class MyServer(SocketServer.TCPServer): + pass + + class MyHandler(SocketServer.StreamRequestHandler): + pass + + threads = [] + for i in range(20): + s = MyServer((HOST, 0), MyHandler) + t = threading.Thread( + name='MyServer serving', + target=s.serve_forever, + kwargs={'poll_interval':0.01}) + t.daemon = True # In case this function raises. + threads.append((t, s)) + for t, s in threads: + t.start() + s.shutdown() + for t, s in threads: + t.join() + + def test_tcpserver_bind_leak(self): + # Issue #22435: the server socket wouldn't be closed if bind()/listen() + # failed. + # Create many servers for which bind() will fail, to see if this result + # in FD exhaustion. + for i in range(1024): + with self.assertRaises(OverflowError): + SocketServer.TCPServer((HOST, -1), + SocketServer.StreamRequestHandler) + + +def test_main(): + if imp.lock_held(): + # If the import lock is held, the threads will hang + raise unittest.SkipTest("can't run when import lock is held") + + test.test_support.run_unittest(SocketServerTest) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_ssl.py b/src/greentest/2.7/test_ssl.py new file mode 100644 index 0000000..29922e7 --- /dev/null +++ b/src/greentest/2.7/test_ssl.py @@ -0,0 +1,3152 @@ +# -*- coding: utf-8 -*- +# Test the support for SSL and sockets + +import sys +import unittest +from test import test_support as support +from test.script_helper import assert_python_ok +import asyncore +import socket +import select +import time +import datetime +import gc +import os +import errno +import pprint +import tempfile +import urllib2 +import traceback +import weakref +import platform +import functools +from contextlib import closing + +ssl = support.import_module("ssl") + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = support.HOST + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/make_ssl_certs.py. +# Other certificates are simply fetched from the Internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = CERTFILE.encode(sys.getfilesystemencoding()) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = ONLYCERT.encode(sys.getfilesystemencoding()) +BYTES_ONLYKEY = ONLYKEY.encode(sys.getfilesystemencoding()) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = CAPATH.encode(sys.getfilesystemencoding()) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") + + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNING_CA = data_file("pycacert.pem") + +REMOTE_HOST = "self-signed.pythontest.net" +REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +NONEXISTINGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") + +DHFILE = data_file("dh1024.pem") +BYTES_DHFILE = DHFILE.encode(sys.getfilesystemencoding()) + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + + +class BasicTests(unittest.TestCase): + + def test_sslwrap_simple(self): + # A crude test for the legacy API + try: + ssl.sslwrap_simple(socket.socket(socket.AF_INET)) + except IOError, e: + if e.errno == 32: # broken pipe when ssl_sock.do_handshake(), this test doesn't care about that + pass + else: + raise + try: + ssl.sslwrap_simple(socket.socket(socket.AF_INET)._sock) + except IOError, e: + if e.errno == 32: # broken pipe when ssl_sock.do_handshake(), this test doesn't care about that + pass + else: + raise + + +def can_clear_options(): + # 0.9.8m or higher + return ssl._OPENSSL_API_VERSION >= (0, 9, 8, 13, 15) + +def no_sslv2_implies_sslv3_hello(): + # 0.9.7h or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) + +def have_verify_flags(): + # 0.9.8 or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 8, 0, 15) + +def utc_offset(): #NOTE: ignore issues like #1647654 + # local time = utc time + utc offset + if time.daylight and time.localtime().tm_isdst > 0: + return -time.altzone # seconds + return -time.timezone + +def asn1time(cert_time): + # Some versions of OpenSSL ignore seconds, see #18207 + # 0.9.8.i + if ssl._OPENSSL_API_VERSION == (0, 9, 8, 9, 15): + fmt = "%b %d %H:%M:%S %Y GMT" + dt = datetime.datetime.strptime(cert_time, fmt) + dt = dt.replace(second=0) + cert_time = dt.strftime(fmt) + # %d adds leading zero but ASN1_TIME_print() uses leading space + if cert_time[4] == "0": + cert_time = cert_time[:4] + " " + cert_time[5:] + + return cert_time + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): + if hasattr(ssl, 'PROTOCOL_SSLv2'): + @functools.wraps(func) + def f(*args, **kwargs): + try: + ssl.SSLContext(ssl.PROTOCOL_SSLv2) + except ssl.SSLError: + if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and + platform.linux_distribution() == ('debian', 'squeeze/sid', '')): + raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") + return func(*args, **kwargs) + return f + else: + return func + +needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + if ssl.HAS_ECDH: + ssl.OP_SINGLE_ECDH_USE + if ssl.OPENSSL_VERSION_INFO >= (1, 0): + ssl.OP_NO_COMPRESSION + self.assertIn(ssl.HAS_SNI, {True, False}) + self.assertIn(ssl.HAS_ECDH, {True, False}) + + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + if hasattr(ssl, 'RAND_egd'): + self.assertRaises(TypeError, ssl.RAND_egd, 1) + self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1) + ssl.RAND_add("this is a random string", 75.0) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + p = ssl._ssl._test_decode_cert(CERTFILE) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['issuer'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + # Note the next three asserts will fail if the keys are regenerated + self.assertEqual(p['notAfter'], asn1time('Oct 5 23:01:56 2020 GMT')) + self.assertEqual(p['notBefore'], asn1time('Oct 8 23:01:56 2010 GMT')) + self.assertEqual(p['serialNumber'], 'D7C7381919AFC24E') + self.assertEqual(p['subject'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, (int, long)) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 0.9 + self.assertGreaterEqual(n, 0x900000) + # < 3.0 + self.assertLess(n, 0x30000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 0) + self.assertLess(major, 3) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + # Version string as returned by {Open,Libre}SSL, the format might change + if "LibreSSL" in s: + self.assertTrue(s.startswith("LibreSSL {:d}.{:d}".format(major, minor)), + (s, t)) + else: + self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)), + (s, t)) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + wr = weakref.ref(ss) + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # socket.error raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with closing(ssl.wrap_socket(s)) as ss: + self.assertRaises(socket.error, ss.recv, 1) + self.assertRaises(socket.error, ss.recv_into, bytearray(b'x')) + self.assertRaises(socket.error, ss.recvfrom, 1) + self.assertRaises(socket.error, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(socket.error, ss.send, b'x') + self.assertRaises(socket.error, ss.sendto, b'x', ('0.0.0.0', 0)) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with closing(ssl.wrap_socket(s)) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + def test_errors(self): + sock = socket.socket() + self.assertRaisesRegexp(ValueError, + "certfile must be specified", + ssl.wrap_socket, sock, keyfile=CERTFILE) + self.assertRaisesRegexp(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True) + self.assertRaisesRegexp(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True, certfile="") + with closing(ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE)) as s: + self.assertRaisesRegexp(ValueError, "can't connect in server-side mode", + s.connect, (HOST, 8080)) + with self.assertRaises(IOError) as cm: + with closing(socket.socket()) as sock: + ssl.wrap_socket(sock, certfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(IOError) as cm: + with closing(socket.socket()) as sock: + ssl.wrap_socket(sock, + certfile=CERTFILE, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(IOError) as cm: + with closing(socket.socket()) as sock: + ssl.wrap_socket(sock, + certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def bad_cert_test(self, certfile): + """Check that trying to use the given client certificate fails""" + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + certfile) + sock = socket.socket() + self.addCleanup(sock.close) + with self.assertRaises(ssl.SSLError): + ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) + + def test_empty_cert(self): + """Wrapping with an empty cert file""" + self.bad_cert_test("nullcert.pem") + + def test_malformed_cert(self): + """Wrapping with a badly formatted certificate (syntax error)""" + self.bad_cert_test("badcert.pem") + + def test_malformed_key(self): + """Wrapping with a badly formatted key (syntax error)""" + self.bad_cert_test("badkey.pem") + + def test_match_hostname(self): + def ok(cert, hostname): + ssl.match_hostname(cert, hostname) + def fail(cert, hostname): + self.assertRaises(ssl.CertificateError, + ssl.match_hostname, cert, hostname) + + cert = {'subject': ((('commonName', 'example.com'),),)} + ok(cert, 'example.com') + ok(cert, 'ExAmple.cOm') + fail(cert, 'www.example.com') + fail(cert, '.example.com') + fail(cert, 'example.org') + fail(cert, 'exampleXcom') + + cert = {'subject': ((('commonName', '*.a.com'),),)} + ok(cert, 'foo.a.com') + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + # only match one left-most wildcard + cert = {'subject': ((('commonName', 'f*.com'),),)} + ok(cert, 'foo.com') + ok(cert, 'f.com') + fail(cert, 'bar.com') + fail(cert, 'foo.a.com') + fail(cert, 'bar.foo.com') + + # NULL bytes are bad, CVE-2013-4073 + cert = {'subject': ((('commonName', + 'null.python.org\x00example.org'),),)} + ok(cert, 'null.python.org\x00example.org') # or raise an error? + fail(cert, 'example.org') + fail(cert, 'null.python.org') + + # error cases with wildcards + cert = {'subject': ((('commonName', '*.*.a.com'),),)} + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + cert = {'subject': ((('commonName', 'a.*.com'),),)} + fail(cert, 'a.foo.com') + fail(cert, 'a..com') + fail(cert, 'a.com') + + # wildcard doesn't match IDNA prefix 'xn--' + idna = u'püthon.python.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, idna) + cert = {'subject': ((('commonName', 'x*.python.org'),),)} + fail(cert, idna) + cert = {'subject': ((('commonName', 'xn--p*.python.org'),),)} + fail(cert, idna) + + # wildcard in first fragment and IDNA A-labels in sequent fragments + # are supported. + idna = u'www*.pythön.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, u'www.pythön.org'.encode("idna").decode("ascii")) + ok(cert, u'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, u'ftp.pythön.org'.encode("idna").decode("ascii")) + fail(cert, u'pythön.org'.encode("idna").decode("ascii")) + + # Slightly fake real-world example + cert = {'notAfter': 'Jun 26 21:41:46 2011 GMT', + 'subject': ((('commonName', 'linuxfrz.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('DNS', 'linuxfr.com'), + ('othername', ''))} + ok(cert, 'linuxfr.org') + ok(cert, 'linuxfr.com') + # Not a "DNS" entry + fail(cert, '') + # When there is a subjectAltName, commonName isn't used + fail(cert, 'linuxfrz.org') + + # A pristine real-world example + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),), + (('commonName', 'mail.google.com'),))} + ok(cert, 'mail.google.com') + fail(cert, 'gmail.com') + # Only commonName is considered + fail(cert, 'California') + + # Neither commonName nor subjectAltName + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),))} + fail(cert, 'mail.google.com') + + # No DNS entry in subjectAltName but a commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('commonName', 'mail.google.com'),)), + 'subjectAltName': (('othername', 'blabla'), )} + ok(cert, 'mail.google.com') + + # No DNS entry subjectAltName and no commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),)), + 'subjectAltName': (('othername', 'blabla'),)} + fail(cert, 'google.com') + + # Empty cert / no cert + self.assertRaises(ValueError, ssl.match_hostname, None, 'example.com') + self.assertRaises(ValueError, ssl.match_hostname, {}, 'example.com') + + # Issue #17980: avoid denials of service by refusing more than one + # wildcard per fragment. + cert = {'subject': ((('commonName', 'a*b.com'),),)} + ok(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b.co*'),),)} + fail(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b*.com'),),)} + with self.assertRaises(ssl.CertificateError) as cm: + ssl.match_hostname(cert, 'axxbxxc.com') + self.assertIn("too many wildcards", str(cm.exception)) + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with closing(socket.socket()) as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + with closing(ssl.wrap_socket(s)) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with closing(ssl.wrap_socket(s)) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with closing(ssl.wrap_socket(s, server_side=True, certfile=CERTFILE)) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (set, bool)) + if isinstance(trust, set): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegexp(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegexp(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + def cert_time_ok(self, timestring, timestamp): + self.assertEqual(ssl.cert_time_to_seconds(timestring), timestamp) + + def cert_time_fail(self, timestring): + with self.assertRaises(ValueError): + ssl.cert_time_to_seconds(timestring) + + @unittest.skipUnless(utc_offset(), + 'local time needs to be different from UTC') + def test_cert_time_to_seconds_timezone(self): + # Issue #19940: ssl.cert_time_to_seconds() returns wrong + # results if local timezone is not UTC + self.cert_time_ok("May 9 00:00:00 2007 GMT", 1178668800.0) + self.cert_time_ok("Jan 5 09:34:43 2018 GMT", 1515144883.0) + + def test_cert_time_to_seconds(self): + timestring = "Jan 5 09:34:43 2018 GMT" + ts = 1515144883.0 + self.cert_time_ok(timestring, ts) + # accept keyword parameter, assert its name + self.assertEqual(ssl.cert_time_to_seconds(cert_time=timestring), ts) + # accept both %e and %d (space or zero generated by strftime) + self.cert_time_ok("Jan 05 09:34:43 2018 GMT", ts) + # case-insensitive + self.cert_time_ok("JaN 5 09:34:43 2018 GmT", ts) + self.cert_time_fail("Jan 5 09:34 2018 GMT") # no seconds + self.cert_time_fail("Jan 5 09:34:43 2018") # no GMT + self.cert_time_fail("Jan 5 09:34:43 2018 UTC") # not GMT timezone + self.cert_time_fail("Jan 35 09:34:43 2018 GMT") # invalid day + self.cert_time_fail("Jon 5 09:34:43 2018 GMT") # invalid month + self.cert_time_fail("Jan 5 24:00:00 2018 GMT") # invalid hour + self.cert_time_fail("Jan 5 09:60:43 2018 GMT") # invalid minute + + newyear_ts = 1230768000.0 + # leap seconds + self.cert_time_ok("Dec 31 23:59:60 2008 GMT", newyear_ts) + # same timestamp + self.cert_time_ok("Jan 1 00:00:00 2009 GMT", newyear_ts) + + self.cert_time_ok("Jan 5 09:34:59 2018 GMT", 1515144899) + # allow 60th second (even if it is not a leap second) + self.cert_time_ok("Jan 5 09:34:60 2018 GMT", 1515144900) + # allow 2nd leap second for compatibility with time.strptime() + self.cert_time_ok("Jan 5 09:34:61 2018 GMT", 1515144901) + self.cert_time_fail("Jan 5 09:34:62 2018 GMT") # invalid seconds + + # no special treatement for the special value: + # 99991231235959Z (rfc 5280) + self.cert_time_ok("Dec 31 23:59:59 9999 GMT", 253402300799.0) + + @support.run_with_locale('LC_ALL', '') + def test_cert_time_to_seconds_locale(self): + # `cert_time_to_seconds()` should be locale independent + + def local_february_name(): + return time.strftime('%b', (1, 2, 3, 4, 5, 6, 0, 0, 0)) + + if local_february_name().lower() == 'feb': + self.skipTest("locale-specific month name needs to be " + "different from C locale") + + # locale-independent + self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) + self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + + +class ContextTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_constructor(self): + for protocol in PROTOCOLS: + ssl.SSLContext(protocol) + self.assertRaises(TypeError, ssl.SSLContext) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + @skip_if_broken_ubuntu_ssl + def test_protocol(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.protocol, proto) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegexp(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @skip_if_broken_ubuntu_ssl + def test_options(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3, + ctx.options) + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1, + ctx.options) + if can_clear_options(): + ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1 + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3, + ctx.options) + ctx.options = 0 + self.assertEqual(0, ctx.options) + else: + with self.assertRaises(ValueError): + ctx.options = 0 + + def test_verify_mode(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(IOError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaisesRegexp(ssl.SSLError, "key values mismatch"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegexp(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegexp(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegexp(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegexp(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegexp(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + ctx.load_verify_locations(cafile=BYTES_CERTFILE.decode('utf-8')) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(IOError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(IOError): + ctx.load_verify_locations(u'') + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read().decode("ascii") + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read().decode("ascii") + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegexp(ssl.SSLError, "no start line"): + ctx.load_verify_locations(cadata=u"broken") + with self.assertRaisesRegexp(ssl.SSLError, "not enough data"): + ctx.load_verify_locations(cadata=b"broken") + + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_dh_params(DHFILE) + if os.name != 'nt': + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(IOError) as cm: + ctx.load_dh_params(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + @skip_if_broken_ubuntu_ssl + def test_session_stats(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + @needs_sni + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @needs_sni + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': asn1time('Mar 29 12:29:49 2033 GMT'), + 'notBefore': asn1time('Mar 30 12:29:49 2003 GMT'), + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + with open(SIGNING_CA) as f: + cadata = f.read().decode("ascii") + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_DH_USE", 0), + getattr(ssl, "OP_SINGLE_DH_USE", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + ) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + def test__https_verify_certificates(self): + # Unit test to check the contect factory mapping + # The factories themselves are tested above + # This test will fail by design if run under PYTHONHTTPSVERIFY=0 + # (as will various test_httplib tests) + + # Uses a fresh SSL module to avoid affecting the real one + local_ssl = support.import_fresh_module("ssl") + # Certificate verification is enabled by default + self.assertIs(local_ssl._create_default_https_context, + local_ssl.create_default_context) + # Turn default verification off + local_ssl._https_verify_certificates(enable=False) + self.assertIs(local_ssl._create_default_https_context, + local_ssl._create_unverified_context) + # And back on + local_ssl._https_verify_certificates(enable=True) + self.assertIs(local_ssl._create_default_https_context, + local_ssl.create_default_context) + # The default behaviour is to enable + local_ssl._https_verify_certificates(enable=False) + local_ssl._https_verify_certificates() + self.assertIs(local_ssl._create_default_https_context, + local_ssl.create_default_context) + + def test__https_verify_envvar(self): + # Unit test to check the PYTHONHTTPSVERIFY handling + # Need to use a subprocess so it can still be run under -E + https_is_verified = """import ssl, sys; \ + status = "Error: _create_default_https_context does not verify certs" \ + if ssl._create_default_https_context is \ + ssl._create_unverified_context \ + else None; \ + sys.exit(status)""" + https_is_not_verified = """import ssl, sys; \ + status = "Error: _create_default_https_context verifies certs" \ + if ssl._create_default_https_context is \ + ssl.create_default_context \ + else None; \ + sys.exit(status)""" + extra_env = {} + # Omitting it leaves verification on + assert_python_ok("-c", https_is_verified, **extra_env) + # Setting it to zero turns verification off + extra_env[ssl._https_verify_envvar] = "0" + assert_python_ok("-c", https_is_not_verified, **extra_env) + # Any other value should also leave it on + for setting in ("", "1", "enabled", "foo"): + extra_env[ssl._https_verify_envvar] = setting + assert_python_ok("-c", https_is_verified, **extra_env) + + def test_check_hostname(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertFalse(ctx.check_hostname) + + # Requires CERT_REQUIRED or CERT_OPTIONAL + with self.assertRaises(ValueError): + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + self.assertEqual(cm.exception.library, 'PEM') + self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) + self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with closing(socket.socket()) as s: + s.bind(("127.0.0.1", 0)) + s.listen(5) + c = socket.socket() + c.connect(s.getsockname()) + c.setblocking(False) + with closing(ctx.wrap_socket(c, False, do_handshake_on_connect=False)) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertTrue(s.startswith("The operation did not complete (read)"), s) + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + +class NetworkedTests(unittest.TestCase): + + def test_connect(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) + try: + s.connect((REMOTE_HOST, 443)) + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + + # this should fail because we have no verification certs + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.assertRaisesRegexp(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + + # this should succeed because we specify the root cert + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + s.connect((REMOTE_HOST, 443)) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.setblocking(False) + rc = s.connect_ex((REMOTE_HOST, 443)) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + s.close() + + def test_connect_ex_error(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + rc = s.connect_ex((REMOTE_HOST, 444)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + finally: + s.close() + + def test_connect_with_context(self): + with support.transient_internet(REMOTE_HOST): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + # Same with a server hostname + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=REMOTE_HOST) + s.connect((REMOTE_HOST, 443)) + s.close() + # This should fail because we have no verification certs + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.assertRaisesRegexp(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + # This should succeed because we specify the root cert + ctx.load_verify_locations(REMOTE_ROOT_CERT) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_cadata(self): + with open(REMOTE_ROOT_CERT) as f: + pem = f.read().decode('ascii') + der = ssl.PEM_cert_to_DER_cert(pem) + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with closing(ctx.wrap_socket(socket.socket(socket.AF_INET))) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with closing(ctx.wrap_socket(socket.socket(socket.AF_INET))) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + with support.transient_internet(REMOTE_HOST): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss.connect((REMOTE_HOST, 443)) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + with support.transient_internet(REMOTE_HOST): + s = socket.socket(socket.AF_INET) + s.connect((REMOTE_HOST, 443)) + s.setblocking(False) + s = ssl.wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + s.close() + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + def _test_get_server_certificate(host, port, cert=None): + with support.transient_internet(host): + pem = ssl.get_server_certificate((host, port)) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + + try: + pem = ssl.get_server_certificate((host, port), + ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + pem = ssl.get_server_certificate((host, port), + ca_certs=cert) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + + _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) + if support.IPV6_ENABLED: + _test_get_server_certificate('ipv6.google.com', 443) + + def test_ciphers(self): + remote = (REMOTE_HOST, 443) + with support.transient_internet(remote[0]): + with closing(ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL")) as s: + s.connect(remote) + with closing(ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT")) as s: + s.connect(remote) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegexp(ssl.SSLError, "No cipher can be selected"): + with closing(socket.socket(socket.AF_INET)) as sock: + s = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(remote) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + # sha256.tbs-internet.com needs SNI to use the correct certificate + if not ssl.HAS_SNI: + self.skipTest("SNI needed for this test") + # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with support.transient_internet("sha256.tbs-internet.com"): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(sha256_cert) + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="sha256.tbs-internet.com") + try: + s.connect(remote) + if support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + self.assertEqual(len(ctx.get_ca_certs()), 1) + + @needs_sni + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + with support.transient_internet(REMOTE_HOST): + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with closing(ctx1.wrap_socket(s)) as ss: + ss.connect((REMOTE_HOST, 443)) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + + from test.ssl_servers import make_https_server + + class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(1) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) + self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) + except socket.error as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + if not isinstance(e, ssl.SSLError) and e.errno != errno.ECONNRESET: + raise + self.server.conn_errors.append(e) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.server.stop() + self.close() + return False + else: + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + sys.stdout.write(" cert binary is " + str(len(cert_binary)) + " bytes\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + sys.stdout.write(" server: selected protocol is now " + + str(self.sslconn.selected_npn_protocol()) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except ssl.SSLError: + if self.server.chatty: + handle_error("Test server failure:\n") + self.close() + self.running = False + # normally, we'd just stop here, but for the test + # harness, we want to stop the server + self.server.stop() + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + npn_protocols=None, alpn_protocols=None, + ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLSv1) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if npn_protocols: + self.context.set_npn_protocols(npn_protocols) + if alpn_protocols: + self.context.set_alpn_protocols(alpn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = support.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_npn_protocols = [] + self.selected_alpn_protocols = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.sock.settimeout(0.05) + self.sock.listen(5) + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except socket.timeout: + pass + except KeyboardInterrupt: + self.stop() + self.sock.close() + + def stop(self): + self.active = False + + class AsyncoreEchoServer(threading.Thread): + + class EchoServer(asyncore.dispatcher): + + class ConnectionHandler(asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = ssl.wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except socket.error, err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accept(self): + sock_obj, addr = self.accept() + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + + def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with closing(client_context.wrap_socket(socket.socket(), + server_hostname=sni_name)) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_alpn_protocol': s.selected_alpn_protocol(), + 'client_npn_protocol': s.selected_npn_protocol(), + 'version': s.version(), + }) + s.close() + stats['server_alpn_protocols'] = server.selected_alpn_protocols + stats['server_npn_protocols'] = server.selected_npn_protocols + return stats + + def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_SSLv23: + client_context.set_ciphers("ALL") + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(CERTFILE) + ctx.load_verify_locations(CERTFILE) + try: + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except socket.error as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) + + + class ThreadedTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + for protocol in PROTOCOLS: + context = ssl.SSLContext(protocol) + context.load_cert_chain(CERTFILE) + server_params_test(context, context, + chatty=True, connectionchatty=True) + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + s = context.wrap_socket(socket.socket(), + do_handshake_on_connect=False) + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + s.close() + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(SIGNING_CA) + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket())) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket())) as s: + with self.assertRaisesRegexp(ssl.SSLError, + "certificate verify failed"): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket())) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket(), + server_hostname="localhost")) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket(), + server_hostname="invalid")) as s: + with self.assertRaisesRegexp(ssl.CertificateError, + "hostname 'invalid' doesn't match u?'localhost'"): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(socket.socket()) as s: + with self.assertRaisesRegexp(ValueError, + "check_hostname requires server_hostname"): + context.wrap_socket(s) + + def test_wrong_cert(self): + """Connecting when the server rejects the client's certificate + + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with a wrong client certificate fails. + """ + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + "wrongcert.pem") + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_REQUIRED, + cacerts=CERTFILE, chatty=False, + connectionchatty=False) + with server, \ + closing(socket.socket()) as sock, \ + closing(ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1)) as s: + try: + # Expect either an SSL error about the server rejecting + # the connection, or a low-level connection reset (which + # sometimes happens on Windows) + s.connect((HOST, server.port)) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except socket.error as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = support.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen(5) + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with closing(socket.socket()) as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = ssl.wrap_socket(c) + except socket.error: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") + def test_protocol_sslv2(self): + """Connecting to an SSLv2 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) + # SSLv23 client with specific SSL options + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv2) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv23(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) + except socket.error as x: + # this fails on some older versions of OpenSSL (0.9.7l, for instance) + if support.verbose: + sys.stdout.write( + " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" + % str(x)) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + + # Server with specific SSL options + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), + "OpenSSL is compiled without SSLv3 support") + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, + False, client_options=ssl.OP_NO_SSLv2) + + @skip_if_broken_ubuntu_ssl + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), + "TLS version 1.1 not supported.") + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), + "TLS version 1.2 not supported.") + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(1) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using a SocketServer to create and manage SSL connections.""" + server = make_https_server(self, certfile=CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + with open(CERTFILE, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = 'https://localhost:%d/%s' % ( + server.port, os.path.split(CERTFILE)[1]) + context = ssl.create_default_context(cafile=CERTFILE) + f = urllib2.urlopen(url, context=context) + try: + dlen = f.info().getheader("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + indata = "TEST MESSAGE of mixed case\n" + + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = ssl.wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, whether to expect success, *args) + send_methods = [ + ('send', s.send, True, []), + ('sendto', s.sendto, False, ["some.address"]), + ('sendall', s.sendall, True, []), + ] + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = u"PREFIX_" + + for meth_name, send_meth, expect_success, args in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + send_meth(indata, *args) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # read(-1, buffer) is supported, even though read(-1) is not + data = b"data" + s.send(data) + buffer = bytearray(len(data)) + self.assertEqual(s.read(-1, buffer), len(data)) + self.assertEqual(buffer, data) + + s.write(b"over\n") + + self.assertRaises(ValueError, s.recv, -1) + self.assertRaises(ValueError, s.read, -1) + + s.close() + + def test_recv_zero(self): + server = ThreadedEchoServer(CERTFILE) + server.__enter__() + self.addCleanup(server.__exit__, None, None) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = ssl.wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + # recv/read(0) should return no data + s.send(b"data") + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.read(0), b"") + self.assertEqual(s.read(), b"data") + + # Should not block if the other end sends no data + s.setblocking(False) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.recv_into(bytearray()), 0) + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen(5) + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegexp(ssl.SSLError, "timed out", + ssl.wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = ssl.wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegexp(ssl.SSLError, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + server = context.wrap_socket(server, server_side=True) + + evt = threading.Event() + remote = [None] + peer = [None] + def serve(): + server.listen(5) + # Block on the accept and wait on the connection to close. + evt.set() + remote[0], peer[0] = server.accept() + remote[0].recv(1) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = context.wrap_socket(socket.socket()) + client.connect((host, port)) + client_addr = client.getsockname() + client.close() + t.join() + remote[0].close() + server.close() + # Sanity checks. + self.assertIsInstance(remote[0], ssl.SSLSocket) + self.assertEqual(peer[0], client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with closing(context.wrap_socket(socket.socket())) as sock: + with self.assertRaises(socket.error) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with closing(context.wrap_socket(socket.socket())) as sock: + with self.assertRaises(socket.error) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_default_ciphers(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + try: + # Force a set of weak ciphers on our client context + context.set_ciphers("DES") + except ssl.SSLError: + self.skipTest("no DES cipher available") + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_SSLv23, + chatty=False) as server: + with closing(context.wrap_socket(socket.socket())) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + self.assertIn("no shared cipher", str(server.conn_errors[0])) + + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + chatty=False) as server: + with closing(context.wrap_socket(socket.socket())) as s: + self.assertIs(s.version(), None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), "TLSv1") + self.assertIs(s.version(), None) + + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain(CERTFILE) + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + if ssl.OPENSSL_VERSION_INFO < (1, 0, 0): + context.set_ciphers("ECCdraft:ECDH") + with ThreadedEchoServer(context=context) as server: + with closing(context.wrap_socket(socket.socket())) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got channel binding data: {0!r}\n" + .format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + s.close() + + # now, again + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got another channel binding data: {0!r}\n" + .format(new_cb_data)) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + s.close() + + def test_compression(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['compression'], None) + + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.load_dh_params(DHFILE) + context.set_ciphers("kEDH") + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: + self.fail("Non-DH cipher: " + cipher[0]) + + def test_selected_alpn_protocol(self): + # selected_alpn_protocol() is None unless ALPN is used. + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support required") + def test_selected_alpn_protocol_if_server_uses_alpn(self): + # selected_alpn_protocol() is None unless ALPN is used by the client. + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_verify_locations(CERTFILE) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(['foo', 'bar']) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support needed for this test") + def test_alpn_protocols(self): + server_protocols = ['foo', 'bar', 'milkshake'] + protocol_tests = [ + (['foo', 'bar'], 'foo'), + (['bar', 'foo'], 'foo'), + (['milkshake'], 'milkshake'), + (['http/3.0', 'http/4.0'], None) + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_cert_chain(CERTFILE) + client_context.set_alpn_protocols(client_protocols) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_alpn_protocol'] + self.assertEqual(client_result, expected, msg % (client_result, "client")) + server_result = stats['server_alpn_protocols'][-1] \ + if len(stats['server_alpn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, msg % (server_result, "server")) + + def test_selected_npn_protocol(self): + # selected_npn_protocol() is None unless NPN is used + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_npn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test") + def test_npn_protocols(self): + server_protocols = ['http/1.1', 'spdy/2'] + protocol_tests = [ + (['http/1.1', 'spdy/2'], 'http/1.1'), + (['spdy/2', 'http/1.1'], 'http/1.1'), + (['spdy/2', 'test'], 'spdy/2'), + (['abc', 'def'], 'abc') + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_npn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_cert_chain(CERTFILE) + client_context.set_npn_protocols(client_protocols) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_npn_protocol'] + self.assertEqual(client_result, expected, msg % (client_result, "client")) + server_result = stats['server_npn_protocols'][-1] \ + if len(stats['server_npn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, msg % (server_result, "server")) + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + @needs_sni + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, 'localhost') + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, 'localhost') + self.assertEqual(calls, []) + + @needs_sni + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @needs_sni + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1.0/0.0 + server_context.set_servername_callback(cb_raising) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertIn("ZeroDivisionError", stderr.getvalue()) + + @needs_sni + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertIn("TypeError", stderr.getvalue()) + + def test_read_write_after_close_raises_valuerror(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + + with server: + s = context.wrap_socket(socket.socket()) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + +def test_main(verbose=False): + if support.verbose: + plats = { + 'Linux': platform.linux_distribution, + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + tests = [ContextTests, BasicTests, BasicSocketTests, SSLErrorTests] + + if support.is_resource_enabled('network'): + tests.append(NetworkedTests) + + if _have_threads: + thread_info = support.threading_setup() + if thread_info: + tests.append(ThreadedTests) + + try: + support.run_unittest(*tests) + finally: + if _have_threads: + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_subprocess.py b/src/greentest/2.7/test_subprocess.py new file mode 100644 index 0000000..06de108 --- /dev/null +++ b/src/greentest/2.7/test_subprocess.py @@ -0,0 +1,1422 @@ +import unittest +from test import test_support +import subprocess +import sys +import signal +import os +import errno +import tempfile +import time +import re +import sysconfig + +try: + import resource +except ImportError: + resource = None +try: + import threading +except ImportError: + threading = None + +mswindows = (sys.platform == "win32") + +# +# Depends on the following external programs: Python +# + +if mswindows: + SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), ' + 'os.O_BINARY);') +else: + SETBINARY = '' + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + # Try to minimize the number of children we have so this test + # doesn't crash on some buildbots (Alphas in particular). + test_support.reap_children() + + def tearDown(self): + for inst in subprocess._active: + inst.wait() + subprocess._cleanup() + self.assertFalse(subprocess._active, "subprocess._active not empty") + + def assertStderrEqual(self, stderr, expected, msg=None): + # In a debug build, stuff like "[6580 refs]" is printed to stderr at + # shutdown time. That frustrates tests trying to check stderr produced + # from a spawned Python process. + actual = re.sub(r"\[\d+ refs\]\r?\n?$", "", stderr) + self.assertEqual(actual, expected, msg) + + +class PopenTestException(Exception): + pass + + +class PopenExecuteChildRaises(subprocess.Popen): + """Popen subclass for testing cleanup of subprocess.PIPE filehandles when + _execute_child fails. + """ + def _execute_child(self, *args, **kwargs): + raise PopenTestException("Forced Exception for Test") + + +class ProcessTestCase(BaseTestCase): + + def test_call_seq(self): + # call() function with sequence argument + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(rc, 47) + + def test_check_call_zero(self): + # check_call() function with zero return code + rc = subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(0)"]) + self.assertEqual(rc, 0) + + def test_check_call_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(c.exception.returncode, 47) + + def test_check_output(self): + # check_output() function with zero return code + output = subprocess.check_output( + [sys.executable, "-c", "print 'BDFL'"]) + self.assertIn('BDFL', output) + + def test_check_output_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_output( + [sys.executable, "-c", "import sys; sys.exit(5)"]) + self.assertEqual(c.exception.returncode, 5) + + def test_check_output_stderr(self): + # check_output() function stderr redirected to stdout + output = subprocess.check_output( + [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], + stderr=subprocess.STDOUT) + self.assertIn('BDFL', output) + + def test_check_output_stdout_arg(self): + # check_output() function stderr redirected to stdout + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print 'will not be run'"], + stdout=sys.stdout) + self.fail("Expected ValueError when stdout arg supplied.") + self.assertIn('stdout', c.exception.args[0]) + + def test_call_kwargs(self): + # call() function with keyword args + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + rc = subprocess.call([sys.executable, "-c", + 'import sys, os;' + 'sys.exit(os.getenv("FRUIT")=="banana")'], + env=newenv) + self.assertEqual(rc, 1) + + def test_invalid_args(self): + # Popen() called with invalid arguments should raise TypeError + # but Popen.__del__ should not complain (issue #12085) + with test_support.captured_stderr() as s: + self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) + argcount = subprocess.Popen.__init__.__code__.co_argcount + too_many_args = [0] * (argcount + 1) + self.assertRaises(TypeError, subprocess.Popen, *too_many_args) + self.assertEqual(s.getvalue(), '') + + def test_stdin_none(self): + # .stdin is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print "banana"'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + p.wait() + self.assertEqual(p.stdin, None) + + def test_stdout_none(self): + # .stdout is None when not redirected, and the child's stdout will + # be inherited from the parent. In order to test this we run a + # subprocess in a subprocess: + # this_test + # \-- subprocess created by this test (parent) + # \-- subprocess created by the parent subprocess (child) + # The parent doesn't specify stdout, so the child will use the + # parent's stdout. This test checks that the message printed by the + # child goes to the parent stdout. The parent also checks that the + # child's stdout is None. See #11963. + code = ('import sys; from subprocess import Popen, PIPE;' + 'p = Popen([sys.executable, "-c", "print \'test_stdout_none\'"],' + ' stdin=PIPE, stderr=PIPE);' + 'p.wait(); assert p.stdout is None;') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), 'test_stdout_none') + + def test_stderr_none(self): + # .stderr is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print "banana"'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + p.wait() + self.assertEqual(p.stderr, None) + + def test_executable_with_cwd(self): + python_dir = os.path.dirname(os.path.realpath(sys.executable)) + p = subprocess.Popen(["somethingyoudonthave", "-c", + "import sys; sys.exit(47)"], + executable=sys.executable, cwd=python_dir) + p.wait() + self.assertEqual(p.returncode, 47) + + @unittest.skipIf(sysconfig.is_python_build(), + "need an installed Python. See #7774") + def test_executable_without_cwd(self): + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + p = subprocess.Popen(["somethingyoudonthave", "-c", + "import sys; sys.exit(47)"], + executable=sys.executable) + p.wait() + self.assertEqual(p.returncode, 47) + + def test_stdin_pipe(self): + # stdin redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.stdin.write("pear") + p.stdin.close() + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_filedes(self): + # stdin is set to open file descriptor + tf = tempfile.TemporaryFile() + d = tf.fileno() + os.write(d, "pear") + os.lseek(d, 0, 0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=d) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_fileobj(self): + # stdin is set to open file object + tf = tempfile.TemporaryFile() + tf.write("pear") + tf.seek(0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=tf) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdout_pipe(self): + # stdout redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), "orange") + + def test_stdout_filedes(self): + # stdout is set to open file descriptor + tf = tempfile.TemporaryFile() + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=d) + p.wait() + os.lseek(d, 0, 0) + self.assertEqual(os.read(d, 1024), "orange") + + def test_stdout_fileobj(self): + # stdout is set to open file object + tf = tempfile.TemporaryFile() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=tf) + p.wait() + tf.seek(0) + self.assertEqual(tf.read(), "orange") + + def test_stderr_pipe(self): + # stderr redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=subprocess.PIPE) + self.addCleanup(p.stderr.close) + self.assertStderrEqual(p.stderr.read(), "strawberry") + + def test_stderr_filedes(self): + # stderr is set to open file descriptor + tf = tempfile.TemporaryFile() + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=d) + p.wait() + os.lseek(d, 0, 0) + self.assertStderrEqual(os.read(d, 1024), "strawberry") + + def test_stderr_fileobj(self): + # stderr is set to open file object + tf = tempfile.TemporaryFile() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), "strawberry") + + def test_stdout_stderr_pipe(self): + # capture stdout and stderr to the same pipe + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + self.addCleanup(p.stdout.close) + self.assertStderrEqual(p.stdout.read(), "appleorange") + + def test_stdout_stderr_file(self): + # capture stdout and stderr to the same open file + tf = tempfile.TemporaryFile() + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=tf, + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), "appleorange") + + def test_stdout_filedes_of_stdout(self): + # stdout is set to 1 (#1531862). + # To avoid printing the text on stdout, we do something similar to + # test_stdout_none (see above). The parent subprocess calls the child + # subprocess passing stdout=1, and this test uses stdout=PIPE in + # order to capture and check the output of the parent. See #11963. + code = ('import sys, subprocess; ' + 'rc = subprocess.call([sys.executable, "-c", ' + ' "import os, sys; sys.exit(os.write(sys.stdout.fileno(), ' + '\'test with stdout=1\'))"], stdout=1); ' + 'assert rc == 18') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), 'test with stdout=1') + + def test_cwd(self): + tmpdir = tempfile.gettempdir() + # We cannot use os.path.realpath to canonicalize the path, + # since it doesn't expand Tru64 {memb} strings. See bug 1063571. + cwd = os.getcwd() + os.chdir(tmpdir) + tmpdir = os.getcwd() + os.chdir(cwd) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getcwd())'], + stdout=subprocess.PIPE, + cwd=tmpdir) + self.addCleanup(p.stdout.close) + normcase = os.path.normcase + self.assertEqual(normcase(p.stdout.read()), normcase(tmpdir)) + + def test_env(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "orange" + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), "orange") + + def test_communicate_stdin(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.communicate("pear") + self.assertEqual(p.returncode, 1) + + def test_communicate_stdout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("pineapple")'], + stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "pineapple") + self.assertEqual(stderr, None) + + def test_communicate_stderr(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("pineapple")'], + stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertStderrEqual(stderr, "pineapple") + + def test_communicate(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple");' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate("banana") + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr, "pineapple") + + # This test is Linux specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + # Test for the fd leak reported in http://bugs.python.org/issue2791. + def test_communicate_pipe_fd_leak(self): + fd_directory = '/proc/%d/fd' % os.getpid() + num_fds_before_popen = len(os.listdir(fd_directory)) + p = subprocess.Popen([sys.executable, "-c", "print()"], + stdout=subprocess.PIPE) + p.communicate() + num_fds_after_communicate = len(os.listdir(fd_directory)) + del p + num_fds_after_destruction = len(os.listdir(fd_directory)) + self.assertEqual(num_fds_before_popen, num_fds_after_destruction) + self.assertEqual(num_fds_before_popen, num_fds_after_communicate) + + def test_communicate_returns(self): + # communicate() should return None if no redirection is active + p = subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(47)"]) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertEqual(stderr, None) + + def test_communicate_pipe_buf(self): + # communicate() with writes larger than pipe_buf + # This test will probably deadlock rather than fail, if + # communicate() does not work properly. + x, y = os.pipe() + if mswindows: + pipe_buf = 512 + else: + pipe_buf = os.fpathconf(x, "PC_PIPE_BUF") + os.close(x) + os.close(y) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read(47));' + 'sys.stderr.write("xyz"*%d);' + 'sys.stdout.write(sys.stdin.read())' % pipe_buf], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + string_to_write = "abc"*pipe_buf + (stdout, stderr) = p.communicate(string_to_write) + self.assertEqual(stdout, string_to_write) + + def test_writes_before_communicate(self): + # stdin.write before communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.stdin.write("banana") + (stdout, stderr) = p.communicate("split") + self.assertEqual(stdout, "bananasplit") + self.assertStderrEqual(stderr, "") + + def test_universal_newlines(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'sys.stdout.write("line1\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line2\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("line3\\r\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line4\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline5");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline6");'], + stdout=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + stdout = p.stdout.read() + if hasattr(file, 'newlines'): + # Interpreter with universal newline support + self.assertEqual(stdout, + "line1\nline2\nline3\nline4\nline5\nline6") + else: + # Interpreter without universal newline support + self.assertEqual(stdout, + "line1\nline2\rline3\r\nline4\r\nline5\nline6") + + def test_universal_newlines_communicate(self): + # universal newlines through communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'sys.stdout.write("line1\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line2\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("line3\\r\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line4\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline5");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline6");'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate() + if hasattr(file, 'newlines'): + # Interpreter with universal newline support + self.assertEqual(stdout, + "line1\nline2\nline3\nline4\nline5\nline6") + else: + # Interpreter without universal newline support + self.assertEqual(stdout, + "line1\nline2\rline3\r\nline4\r\nline5\nline6") + + def test_no_leaking(self): + # Make sure we leak no resources + if not mswindows: + max_handles = 1026 # too much for most UNIX systems + else: + max_handles = 2050 # too much for (at least some) Windows setups + handles = [] + try: + for i in range(max_handles): + try: + handles.append(os.open(test_support.TESTFN, + os.O_WRONLY | os.O_CREAT)) + except OSError as e: + if e.errno != errno.EMFILE: + raise + break + else: + self.skipTest("failed to reach the file descriptor limit " + "(tried %d)" % max_handles) + # Close a couple of them (should be enough for a subprocess) + for i in range(10): + os.close(handles.pop()) + # Loop creating some subprocesses. If one of them leaks some fds, + # the next loop iteration will fail by reaching the max fd limit. + for i in range(15): + p = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write(sys.stdin.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + data = p.communicate(b"lime")[0] + self.assertEqual(data, b"lime") + finally: + for h in handles: + os.close(h) + test_support.unlink(test_support.TESTFN) + + def test_list2cmdline(self): + self.assertEqual(subprocess.list2cmdline(['a b c', 'd', 'e']), + '"a b c" d e') + self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']), + 'ab\\"c \\ d') + self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']), + 'ab\\"c " \\\\" d') + self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']), + 'a\\\\\\b "de fg" h') + self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']), + 'a\\\\\\"b c d') + self.assertEqual(subprocess.list2cmdline(['a\\\\b c', 'd', 'e']), + '"a\\\\b c" d e') + self.assertEqual(subprocess.list2cmdline(['a\\\\b\\ c', 'd', 'e']), + '"a\\\\b\\ c" d e') + self.assertEqual(subprocess.list2cmdline(['ab', '']), + 'ab ""') + + + def test_poll(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(1)"]) + count = 0 + while p.poll() is None: + time.sleep(0.1) + count += 1 + # We expect that the poll loop probably went around about 10 times, + # but, based on system scheduling we can't control, it's possible + # poll() never returned None. It "should be" very rare that it + # didn't go around at least twice. + self.assertGreaterEqual(count, 2) + # Subsequent invocations should just return the returncode + self.assertEqual(p.poll(), 0) + + + def test_wait(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(2)"]) + self.assertEqual(p.wait(), 0) + # Subsequent invocations should just return the returncode + self.assertEqual(p.wait(), 0) + + + def test_invalid_bufsize(self): + # an invalid type of the bufsize argument should raise + # TypeError. + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], "orange") + + def test_leaking_fds_on_error(self): + # see bug #5179: Popen leaks file descriptors to PIPEs if + # the child fails to execute; this will eventually exhaust + # the maximum number of open fds. 1024 seems a very common + # value for that limit, but Windows has 2048, so we loop + # 1024 times (each call leaked two fds). + for i in range(1024): + # Windows raises IOError. Others raise OSError. + with self.assertRaises(EnvironmentError) as c: + subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # ignore errors that indicate the command was not found + if c.exception.errno not in (errno.ENOENT, errno.EACCES): + raise c.exception + + @unittest.skipIf(threading is None, "threading required") + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(['nonexisting_i_hope'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + + def test_handles_closed_on_exception(self): + # If CreateProcess exits with an error, ensure the + # duplicate output handles are released + ifhandle, ifname = tempfile.mkstemp() + ofhandle, ofname = tempfile.mkstemp() + efhandle, efname = tempfile.mkstemp() + try: + subprocess.Popen (["*"], stdin=ifhandle, stdout=ofhandle, + stderr=efhandle) + except OSError: + os.close(ifhandle) + os.remove(ifname) + os.close(ofhandle) + os.remove(ofname) + os.close(efhandle) + os.remove(efname) + self.assertFalse(os.path.exists(ifname)) + self.assertFalse(os.path.exists(ofname)) + self.assertFalse(os.path.exists(efname)) + + def test_communicate_epipe(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.communicate("x" * 2**20) + + def test_communicate_epipe_only_stdin(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + time.sleep(2) + p.communicate("x" * 2**20) + + # This test is Linux-ish specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + def test_failed_child_execute_fd_leak(self): + """Test for the fork() failure fd leak reported in issue16327.""" + fd_directory = '/proc/%d/fd' % os.getpid() + fds_before_popen = os.listdir(fd_directory) + with self.assertRaises(PopenTestException): + PopenExecuteChildRaises( + [sys.executable, '-c', 'pass'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # NOTE: This test doesn't verify that the real _execute_child + # does not close the file descriptors itself on the way out + # during an exception. Code inspection has confirmed that. + + fds_after_exception = os.listdir(fd_directory) + self.assertEqual(fds_before_popen, fds_after_exception) + + +# context manager +class _SuppressCoreFiles(object): + """Try to prevent core files from being created.""" + old_limit = None + + def __enter__(self): + """Try to save previous ulimit, then set it to (0, 0).""" + if resource is not None: + try: + self.old_limit = resource.getrlimit(resource.RLIMIT_CORE) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + except (ValueError, resource.error): + pass + + if sys.platform == 'darwin': + # Check if the 'Crash Reporter' on OSX was configured + # in 'Developer' mode and warn that it will get triggered + # when it is. + # + # This assumes that this context manager is used in tests + # that might trigger the next manager. + value = subprocess.Popen(['/usr/bin/defaults', 'read', + 'com.apple.CrashReporter', 'DialogType'], + stdout=subprocess.PIPE).communicate()[0] + if value.strip() == b'developer': + print "this tests triggers the Crash Reporter, that is intentional" + sys.stdout.flush() + + def __exit__(self, *args): + """Return core file behavior to default.""" + if self.old_limit is None: + return + if resource is not None: + try: + resource.setrlimit(resource.RLIMIT_CORE, self.old_limit) + except (ValueError, resource.error): + pass + + @unittest.skipUnless(hasattr(signal, 'SIGALRM'), + "Requires signal.SIGALRM") + def test_communicate_eintr(self): + # Issue #12493: communicate() should handle EINTR + def handler(signum, frame): + pass + old_handler = signal.signal(signal.SIGALRM, handler) + self.addCleanup(signal.signal, signal.SIGALRM, old_handler) + + # the process is running for 2 seconds + args = [sys.executable, "-c", 'import time; time.sleep(2)'] + for stream in ('stdout', 'stderr'): + kw = {stream: subprocess.PIPE} + with subprocess.Popen(args, **kw) as process: + signal.alarm(1) + # communicate() will be interrupted by SIGALRM + process.communicate() + + +@unittest.skipIf(mswindows, "POSIX specific tests") +class POSIXProcessTestCase(BaseTestCase): + + def test_exceptions(self): + # caught & re-raised exceptions + with self.assertRaises(OSError) as c: + p = subprocess.Popen([sys.executable, "-c", ""], + cwd="/this/path/does/not/exist") + # The attribute child_traceback should contain "os.chdir" somewhere. + self.assertIn("os.chdir", c.exception.child_traceback) + + def test_run_abort(self): + # returncode handles signal termination + with _SuppressCoreFiles(): + p = subprocess.Popen([sys.executable, "-c", + "import os; os.abort()"]) + p.wait() + self.assertEqual(-p.returncode, signal.SIGABRT) + + def test_preexec(self): + # preexec function + p = subprocess.Popen([sys.executable, "-c", + "import sys, os;" + "sys.stdout.write(os.getenv('FRUIT'))"], + stdout=subprocess.PIPE, + preexec_fn=lambda: os.putenv("FRUIT", "apple")) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), "apple") + + class _TestExecuteChildPopen(subprocess.Popen): + """Used to test behavior at the end of _execute_child.""" + def __init__(self, testcase, *args, **kwargs): + self._testcase = testcase + subprocess.Popen.__init__(self, *args, **kwargs) + + def _execute_child( + self, args, executable, preexec_fn, close_fds, cwd, env, + universal_newlines, startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): + try: + subprocess.Popen._execute_child( + self, args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, + startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + finally: + # Open a bunch of file descriptors and verify that + # none of them are the same as the ones the Popen + # instance is using for stdin/stdout/stderr. + devzero_fds = [os.open("/dev/zero", os.O_RDONLY) + for _ in range(8)] + try: + for fd in devzero_fds: + self._testcase.assertNotIn( + fd, (p2cwrite, c2pread, errread)) + finally: + for fd in devzero_fds: + os.close(fd) + + @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") + def test_preexec_errpipe_does_not_double_close_pipes(self): + """Issue16140: Don't double close pipes on preexec error.""" + + def raise_it(): + raise RuntimeError("force the _execute_child() errpipe_data path.") + + with self.assertRaises(RuntimeError): + self._TestExecuteChildPopen( + self, [sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=raise_it) + + def test_args_string(self): + # args is a string + f, fname = tempfile.mkstemp() + os.write(f, "#!/bin/sh\n") + os.write(f, "exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.close(f) + os.chmod(fname, 0o700) + p = subprocess.Popen(fname) + p.wait() + os.remove(fname) + self.assertEqual(p.returncode, 47) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + startupinfo=47) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + creationflags=47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen(["echo $FRUIT"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), "apple") + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen("echo $FRUIT", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), "apple") + + def test_call_string(self): + # call() function with string argument on UNIX + f, fname = tempfile.mkstemp() + os.write(f, "#!/bin/sh\n") + os.write(f, "exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.close(f) + os.chmod(fname, 0700) + rc = subprocess.call(fname) + os.remove(fname) + self.assertEqual(rc, 47) + + def test_specific_shell(self): + # Issue #9265: Incorrect name passed as arg[0]. + shells = [] + for prefix in ['/bin', '/usr/bin/', '/usr/local/bin']: + for name in ['bash', 'ksh']: + sh = os.path.join(prefix, name) + if os.path.isfile(sh): + shells.append(sh) + if not shells: # Will probably work for any shell but csh. + self.skipTest("bash or ksh required for this test") + sh = '/bin/sh' + if os.path.isfile(sh) and not os.path.islink(sh): + # Test will fail if /bin/sh is a symlink to csh. + shells.append(sh) + for sh in shells: + p = subprocess.Popen("echo $0", executable=sh, shell=True, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), sh) + + def _kill_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + return p + + @unittest.skipIf(sys.platform.startswith(('netbsd', 'openbsd')), + "Due to known OS bug (issue #16762)") + def _kill_dead_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + p.communicate() + + def test_send_signal(self): + p = self._kill_process('send_signal', signal.SIGINT) + _, stderr = p.communicate() + self.assertIn('KeyboardInterrupt', stderr) + self.assertNotEqual(p.wait(), 0) + + def test_kill(self): + p = self._kill_process('kill') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, '') + self.assertEqual(p.wait(), -signal.SIGKILL) + + def test_terminate(self): + p = self._kill_process('terminate') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, '') + self.assertEqual(p.wait(), -signal.SIGTERM) + + def test_send_signal_dead(self): + # Sending a signal to a dead process + self._kill_dead_process('send_signal', signal.SIGINT) + + def test_kill_dead(self): + # Killing a dead process + self._kill_dead_process('kill') + + def test_terminate_dead(self): + # Terminating a dead process + self._kill_dead_process('terminate') + + def check_close_std_fds(self, fds): + # Issue #9905: test that subprocess pipes still work properly with + # some standard fds closed + stdin = 0 + newfds = [] + for a in fds: + b = os.dup(a) + newfds.append(b) + if a == 0: + stdin = b + try: + for fd in fds: + os.close(fd) + out, err = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + err = test_support.strip_python_stderr(err) + self.assertEqual((out, err), (b'apple', b'orange')) + finally: + for b, a in zip(newfds, fds): + os.dup2(b, a) + for b in newfds: + os.close(b) + + def test_close_fd_0(self): + self.check_close_std_fds([0]) + + def test_close_fd_1(self): + self.check_close_std_fds([1]) + + def test_close_fd_2(self): + self.check_close_std_fds([2]) + + def test_close_fds_0_1(self): + self.check_close_std_fds([0, 1]) + + def test_close_fds_0_2(self): + self.check_close_std_fds([0, 2]) + + def test_close_fds_1_2(self): + self.check_close_std_fds([1, 2]) + + def test_close_fds_0_1_2(self): + # Issue #10806: test that subprocess pipes still work properly with + # all standard fds closed. + self.check_close_std_fds([0, 1, 2]) + + def check_swap_fds(self, stdin_no, stdout_no, stderr_no): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + temp_fds = [fd for fd, fname in temps] + try: + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # save a copy of the standard file descriptors + saved_fds = [os.dup(fd) for fd in range(3)] + try: + # duplicate the temp files over the standard fd's 0, 1, 2 + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # write some data to what will become stdin, and rewind + os.write(stdin_no, b"STDIN") + os.lseek(stdin_no, 0, 0) + + # now use those files in the given order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=stdin_no, + stdout=stdout_no, + stderr=stderr_no) + p.wait() + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(stdout_no, 1024) + err = test_support.strip_python_stderr(os.read(stderr_no, 1024)) + finally: + for std, saved in enumerate(saved_fds): + os.dup2(saved, std) + os.close(saved) + + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + # When duping fds, if there arises a situation where one of the fds is + # either 0, 1 or 2, it is possible that it is overwritten (#12607). + # This tests all combinations of this. + def test_swap_fds(self): + self.check_swap_fds(0, 1, 2) + self.check_swap_fds(0, 2, 1) + self.check_swap_fds(1, 0, 2) + self.check_swap_fds(1, 2, 0) + self.check_swap_fds(2, 0, 1) + self.check_swap_fds(2, 1, 0) + + def test_wait_when_sigchild_ignored(self): + # NOTE: sigchild_ignore.py may not be an effective test on all OSes. + sigchild_ignore = test_support.findfile("sigchild_ignore.py", + subdir="subprocessdata") + p = subprocess.Popen([sys.executable, sigchild_ignore], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" + " non-zero with this error:\n%s" % stderr) + + def test_zombie_fast_process_del(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, it wouldn't be added to subprocess._active, and would + # remain a zombie. + # spawn a Popen, and delete its reference before it exits + p = subprocess.Popen([sys.executable, "-c", + 'import sys, time;' + 'time.sleep(0.2)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + def test_leak_fast_process_del_killed(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, and the process got killed by a signal, it would never + # be removed from subprocess._active, which triggered a FD and memory + # leak. + # spawn a Popen, delete its reference and kill it + p = subprocess.Popen([sys.executable, "-c", + 'import time;' + 'time.sleep(3)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + os.kill(pid, signal.SIGKILL) + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + # let some time for the process to exit, and create a new Popen: this + # should trigger the wait() of p + time.sleep(0.2) + with self.assertRaises(EnvironmentError) as c: + with subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + # p should have been wait()ed on, and removed from the _active list + self.assertRaises(OSError, os.waitpid, pid, 0) + self.assertNotIn(ident, [id(o) for o in subprocess._active]) + + def test_pipe_cloexec(self): + # Issue 12786: check that the communication pipes' FDs are set CLOEXEC, + # and are not inherited by another child process. + p1 = subprocess.Popen([sys.executable, "-c", + 'import os;' + 'os.read(0, 1)' + ], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + p2 = subprocess.Popen([sys.executable, "-c", """if True: + import os, errno, sys + for fd in %r: + try: + os.close(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + sys.exit(1) + sys.exit(0) + """ % [f.fileno() for f in (p1.stdin, p1.stdout, + p1.stderr)] + ], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=False) + p1.communicate('foo') + _, stderr = p2.communicate() + + self.assertEqual(p2.returncode, 0, "Unexpected error: " + repr(stderr)) + + +@unittest.skipUnless(mswindows, "Windows specific tests") +class Win32ProcessTestCase(BaseTestCase): + + def test_startupinfo(self): + # startupinfo argument + # We uses hardcoded constants, because we do not want to + # depend on win32all. + STARTF_USESHOWWINDOW = 1 + SW_MAXIMIZE = 3 + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = STARTF_USESHOWWINDOW + startupinfo.wShowWindow = SW_MAXIMIZE + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_creationflags(self): + # creationflags argument + CREATE_NEW_CONSOLE = 16 + sys.stderr.write(" a DOS box should flash briefly ...\n") + subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + preexec_fn=lambda: 1) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + stdout=subprocess.PIPE, + close_fds=True) + + def test_close_fds(self): + # close file descriptors + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"], + close_fds=True) + self.assertEqual(rc, 47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen(["set"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn("physalis", p.stdout.read()) + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn("physalis", p.stdout.read()) + + def test_call_string(self): + # call() function with string argument on Windows + rc = subprocess.call(sys.executable + + ' -c "import sys; sys.exit(47)"') + self.assertEqual(rc, 47) + + def _kill_process(self, method, *args): + # Some win32 buildbot raises EOFError if stdin is inherited + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, '') + returncode = p.wait() + self.assertNotEqual(returncode, 0) + + def _kill_dead_process(self, method, *args): + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + sys.exit(42) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() + self.assertEqual(rc, 42) + + def test_send_signal(self): + self._kill_process('send_signal', signal.SIGTERM) + + def test_kill(self): + self._kill_process('kill') + + def test_terminate(self): + self._kill_process('terminate') + + def test_send_signal_dead(self): + self._kill_dead_process('send_signal', signal.SIGTERM) + + def test_kill_dead(self): + self._kill_dead_process('kill') + + def test_terminate_dead(self): + self._kill_dead_process('terminate') + + +@unittest.skipUnless(getattr(subprocess, '_has_poll', False), + "poll system call not supported") +class ProcessTestCaseNoPoll(ProcessTestCase): + def setUp(self): + subprocess._has_poll = False + ProcessTestCase.setUp(self) + + def tearDown(self): + subprocess._has_poll = True + ProcessTestCase.tearDown(self) + + +class HelperFunctionTests(unittest.TestCase): + @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows") + def test_eintr_retry_call(self): + record_calls = [] + def fake_os_func(*args): + record_calls.append(args) + if len(record_calls) == 2: + raise OSError(errno.EINTR, "fake interrupted system call") + return tuple(reversed(args)) + + self.assertEqual((999, 256), + subprocess._eintr_retry_call(fake_os_func, 256, 999)) + self.assertEqual([(256, 999)], record_calls) + # This time there will be an EINTR so it will loop once. + self.assertEqual((666,), + subprocess._eintr_retry_call(fake_os_func, 666)) + self.assertEqual([(256, 999), (666,), (666,)], record_calls) + +@unittest.skipUnless(mswindows, "mswindows only") +class CommandsWithSpaces (BaseTestCase): + + def setUp(self): + super(CommandsWithSpaces, self).setUp() + f, fname = tempfile.mkstemp(".py", "te st") + self.fname = fname.lower () + os.write(f, b"import sys;" + b"sys.stdout.write('%d %s' % (len(sys.argv), [a.lower () for a in sys.argv]))" + ) + os.close(f) + + def tearDown(self): + os.remove(self.fname) + super(CommandsWithSpaces, self).tearDown() + + def with_spaces(self, *args, **kwargs): + kwargs['stdout'] = subprocess.PIPE + p = subprocess.Popen(*args, **kwargs) + self.addCleanup(p.stdout.close) + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) + + def test_shell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd"), shell=1) + + def test_shell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) + + def test_noshell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd")) + + def test_noshell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"]) + +def test_main(): + unit_tests = (ProcessTestCase, + POSIXProcessTestCase, + Win32ProcessTestCase, + ProcessTestCaseNoPoll, + HelperFunctionTests, + CommandsWithSpaces) + + test_support.run_unittest(*unit_tests) + test_support.reap_children() + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_telnetlib.py b/src/greentest/2.7/test_telnetlib.py new file mode 100644 index 0000000..6c122d7 --- /dev/null +++ b/src/greentest/2.7/test_telnetlib.py @@ -0,0 +1,461 @@ +import socket +import telnetlib +import time +import Queue + +import unittest +from unittest import TestCase +from test import test_support +threading = test_support.import_module('threading') + +HOST = test_support.HOST +EOF_sigil = object() + +def server(evt, serv, dataq=None): + """ Open a tcp server in three steps + 1) set evt to true to let the parent know we are ready + 2) [optional] if is not False, write the list of data from dataq.get() + to the socket. + """ + serv.listen(5) + evt.set() + try: + conn, addr = serv.accept() + if dataq: + data = '' + new_data = dataq.get(True, 0.5) + dataq.task_done() + for item in new_data: + if item == EOF_sigil: + break + if type(item) in [int, float]: + time.sleep(item) + else: + data += item + written = conn.send(data) + data = data[written:] + conn.close() + except socket.timeout: + pass + finally: + serv.close() + +class GeneralTests(TestCase): + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(60) # Safety net. Look issue 11812 + self.port = test_support.bind_port(self.sock) + self.thread = threading.Thread(target=server, args=(self.evt,self.sock)) + self.thread.setDaemon(True) + self.thread.start() + self.evt.wait() + + def tearDown(self): + self.thread.join() + + def testBasic(self): + # connects + telnet = telnetlib.Telnet(HOST, self.port) + telnet.sock.close() + + def testTimeoutDefault(self): + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + telnet = telnetlib.Telnet(HOST, self.port) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testTimeoutNone(self): + # None, having other default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + telnet = telnetlib.Telnet(HOST, self.port, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertTrue(telnet.sock.gettimeout() is None) + telnet.sock.close() + + def testTimeoutValue(self): + telnet = telnetlib.Telnet(HOST, self.port, timeout=30) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testTimeoutOpen(self): + telnet = telnetlib.Telnet() + telnet.open(HOST, self.port, timeout=30) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testGetters(self): + # Test telnet getter methods + telnet = telnetlib.Telnet(HOST, self.port, timeout=30) + t_sock = telnet.sock + self.assertEqual(telnet.get_socket(), t_sock) + self.assertEqual(telnet.fileno(), t_sock.fileno()) + telnet.sock.close() + +def _read_setUp(self): + self.evt = threading.Event() + self.dataq = Queue.Queue() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(10) + self.port = test_support.bind_port(self.sock) + self.thread = threading.Thread(target=server, args=(self.evt,self.sock, self.dataq)) + self.thread.start() + self.evt.wait() + +def _read_tearDown(self): + self.thread.join() + +class ReadTests(TestCase): + setUp = _read_setUp + tearDown = _read_tearDown + + # use a similar approach to testing timeouts as test_timeout.py + # these will never pass 100% but make the fuzz big enough that it is rare + block_long = 0.6 + block_short = 0.3 + def test_read_until_A(self): + """ + read_until(expected, [timeout]) + Read until the expected string has been seen, or a timeout is + hit (default is no timeout); may block. + """ + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + data = telnet.read_until('match') + self.assertEqual(data, ''.join(want[:-2])) + + def test_read_until_B(self): + # test the timeout - it does NOT raise socket.timeout + want = ['hello', self.block_long, 'not seen', EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + data = telnet.read_until('not seen', self.block_short) + self.assertEqual(data, want[0]) + self.assertEqual(telnet.read_all(), 'not seen') + + def test_read_until_with_poll(self): + """Use select.poll() to implement telnet.read_until().""" + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + if not telnet._has_poll: + raise unittest.SkipTest('select.poll() is required') + telnet._has_poll = True + self.dataq.join() + data = telnet.read_until('match') + self.assertEqual(data, ''.join(want[:-2])) + + def test_read_until_with_select(self): + """Use select.select() to implement telnet.read_until().""" + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + telnet._has_poll = False + self.dataq.join() + data = telnet.read_until('match') + self.assertEqual(data, ''.join(want[:-2])) + + def test_read_all_A(self): + """ + read_all() + Read all data until EOF; may block. + """ + want = ['x' * 500, 'y' * 500, 'z' * 500, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + data = telnet.read_all() + self.assertEqual(data, ''.join(want[:-1])) + + def _test_blocking(self, func): + self.dataq.put([self.block_long, EOF_sigil]) + self.dataq.join() + start = time.time() + data = func() + self.assertTrue(self.block_short <= time.time() - start) + + def test_read_all_B(self): + self._test_blocking(telnetlib.Telnet(HOST, self.port).read_all) + + def test_read_all_C(self): + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + telnet.read_all() + telnet.read_all() # shouldn't raise + + def test_read_some_A(self): + """ + read_some() + Read at least one byte or EOF; may block. + """ + # test 'at least one byte' + want = ['x' * 500, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + data = telnet.read_all() + self.assertTrue(len(data) >= 1) + + def test_read_some_B(self): + # test EOF + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + self.assertEqual('', telnet.read_some()) + + def test_read_some_C(self): + self._test_blocking(telnetlib.Telnet(HOST, self.port).read_some) + + def _test_read_any_eager_A(self, func_name): + """ + read_very_eager() + Read all data available already queued or on the socket, + without blocking. + """ + want = [self.block_long, 'x' * 100, 'y' * 100, EOF_sigil] + expects = want[1] + want[2] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + func = getattr(telnet, func_name) + data = '' + while True: + try: + data += func() + self.assertTrue(expects.startswith(data)) + except EOFError: + break + self.assertEqual(expects, data) + + def _test_read_any_eager_B(self, func_name): + # test EOF + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + time.sleep(self.block_short) + func = getattr(telnet, func_name) + self.assertRaises(EOFError, func) + + # read_eager and read_very_eager make the same gaurantees + # (they behave differently but we only test the gaurantees) + def test_read_very_eager_A(self): + self._test_read_any_eager_A('read_very_eager') + def test_read_very_eager_B(self): + self._test_read_any_eager_B('read_very_eager') + def test_read_eager_A(self): + self._test_read_any_eager_A('read_eager') + def test_read_eager_B(self): + self._test_read_any_eager_B('read_eager') + # NB -- we need to test the IAC block which is mentioned in the docstring + # but not in the module docs + + def _test_read_any_lazy_B(self, func_name): + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + func = getattr(telnet, func_name) + telnet.fill_rawq() + self.assertRaises(EOFError, func) + + def test_read_lazy_A(self): + want = ['x' * 100, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + time.sleep(self.block_short) + self.assertEqual('', telnet.read_lazy()) + data = '' + while True: + try: + read_data = telnet.read_lazy() + data += read_data + if not read_data: + telnet.fill_rawq() + except EOFError: + break + self.assertTrue(want[0].startswith(data)) + self.assertEqual(data, want[0]) + + def test_read_lazy_B(self): + self._test_read_any_lazy_B('read_lazy') + + def test_read_very_lazy_A(self): + want = ['x' * 100, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + time.sleep(self.block_short) + self.assertEqual('', telnet.read_very_lazy()) + data = '' + while True: + try: + read_data = telnet.read_very_lazy() + except EOFError: + break + data += read_data + if not read_data: + telnet.fill_rawq() + self.assertEqual('', telnet.cookedq) + telnet.process_rawq() + self.assertTrue(want[0].startswith(data)) + self.assertEqual(data, want[0]) + + def test_read_very_lazy_B(self): + self._test_read_any_lazy_B('read_very_lazy') + +class nego_collector(object): + def __init__(self, sb_getter=None): + self.seen = '' + self.sb_getter = sb_getter + self.sb_seen = '' + + def do_nego(self, sock, cmd, opt): + self.seen += cmd + opt + if cmd == tl.SE and self.sb_getter: + sb_data = self.sb_getter() + self.sb_seen += sb_data + +tl = telnetlib +class OptionTests(TestCase): + setUp = _read_setUp + tearDown = _read_tearDown + # RFC 854 commands + cmds = [tl.AO, tl.AYT, tl.BRK, tl.EC, tl.EL, tl.GA, tl.IP, tl.NOP] + + def _test_command(self, data): + """ helper for testing IAC + cmd """ + self.setUp() + self.dataq.put(data) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + nego = nego_collector() + telnet.set_option_negotiation_callback(nego.do_nego) + txt = telnet.read_all() + cmd = nego.seen + self.assertTrue(len(cmd) > 0) # we expect at least one command + self.assertIn(cmd[0], self.cmds) + self.assertEqual(cmd[1], tl.NOOPT) + self.assertEqual(len(''.join(data[:-1])), len(txt + cmd)) + nego.sb_getter = None # break the nego => telnet cycle + self.tearDown() + + def test_IAC_commands(self): + # reset our setup + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + self.tearDown() + + for cmd in self.cmds: + self._test_command(['x' * 100, tl.IAC + cmd, 'y'*100, EOF_sigil]) + self._test_command(['x' * 10, tl.IAC + cmd, 'y'*10, EOF_sigil]) + self._test_command([tl.IAC + cmd, EOF_sigil]) + # all at once + self._test_command([tl.IAC + cmd for (cmd) in self.cmds] + [EOF_sigil]) + self.assertEqual('', telnet.read_sb_data()) + + def test_SB_commands(self): + # RFC 855, subnegotiations portion + send = [tl.IAC + tl.SB + tl.IAC + tl.SE, + tl.IAC + tl.SB + tl.IAC + tl.IAC + tl.IAC + tl.SE, + tl.IAC + tl.SB + tl.IAC + tl.IAC + 'aa' + tl.IAC + tl.SE, + tl.IAC + tl.SB + 'bb' + tl.IAC + tl.IAC + tl.IAC + tl.SE, + tl.IAC + tl.SB + 'cc' + tl.IAC + tl.IAC + 'dd' + tl.IAC + tl.SE, + EOF_sigil, + ] + self.dataq.put(send) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + nego = nego_collector(telnet.read_sb_data) + telnet.set_option_negotiation_callback(nego.do_nego) + txt = telnet.read_all() + self.assertEqual(txt, '') + want_sb_data = tl.IAC + tl.IAC + 'aabb' + tl.IAC + 'cc' + tl.IAC + 'dd' + self.assertEqual(nego.sb_seen, want_sb_data) + self.assertEqual('', telnet.read_sb_data()) + nego.sb_getter = None # break the nego => telnet cycle + + +class ExpectTests(TestCase): + def setUp(self): + self.evt = threading.Event() + self.dataq = Queue.Queue() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(10) + self.port = test_support.bind_port(self.sock) + self.thread = threading.Thread(target=server, args=(self.evt,self.sock, + self.dataq)) + self.thread.start() + self.evt.wait() + + def tearDown(self): + self.thread.join() + + # use a similar approach to testing timeouts as test_timeout.py + # these will never pass 100% but make the fuzz big enough that it is rare + block_long = 0.6 + block_short = 0.3 + def test_expect_A(self): + """ + expect(expected, [timeout]) + Read until the expected string has been seen, or a timeout is + hit (default is no timeout); may block. + """ + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + (_,_,data) = telnet.expect(['match']) + self.assertEqual(data, ''.join(want[:-2])) + + def test_expect_B(self): + # test the timeout - it does NOT raise socket.timeout + want = ['hello', self.block_long, 'not seen', EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + (_,_,data) = telnet.expect(['not seen'], self.block_short) + self.assertEqual(data, want[0]) + self.assertEqual(telnet.read_all(), 'not seen') + + def test_expect_with_poll(self): + """Use select.poll() to implement telnet.expect().""" + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + if not telnet._has_poll: + raise unittest.SkipTest('select.poll() is required') + telnet._has_poll = True + self.dataq.join() + (_,_,data) = telnet.expect(['match']) + self.assertEqual(data, ''.join(want[:-2])) + + def test_expect_with_select(self): + """Use select.select() to implement telnet.expect().""" + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + telnet._has_poll = False + self.dataq.join() + (_,_,data) = telnet.expect(['match']) + self.assertEqual(data, ''.join(want[:-2])) + + +def test_main(verbose=None): + test_support.run_unittest(GeneralTests, ReadTests, OptionTests, + ExpectTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7/test_thread.py b/src/greentest/2.7/test_thread.py new file mode 100644 index 0000000..b056039 --- /dev/null +++ b/src/greentest/2.7/test_thread.py @@ -0,0 +1,261 @@ +import os +import unittest +import random +from test import test_support +thread = test_support.import_module('thread') +import time +import sys +import weakref + +from test import lock_tests + +NUMTASKS = 10 +NUMTRIPS = 3 + + +_print_mutex = thread.allocate_lock() + +def verbose_print(arg): + """Helper function for printing out debugging output.""" + if test_support.verbose: + with _print_mutex: + print arg + + +class BasicThreadTest(unittest.TestCase): + + def setUp(self): + self.done_mutex = thread.allocate_lock() + self.done_mutex.acquire() + self.running_mutex = thread.allocate_lock() + self.random_mutex = thread.allocate_lock() + self.created = 0 + self.running = 0 + self.next_ident = 0 + + +class ThreadRunningTests(BasicThreadTest): + + def newtask(self): + with self.running_mutex: + self.next_ident += 1 + verbose_print("creating task %s" % self.next_ident) + thread.start_new_thread(self.task, (self.next_ident,)) + self.created += 1 + self.running += 1 + + def task(self, ident): + with self.random_mutex: + delay = random.random() / 10000.0 + verbose_print("task %s will run for %sus" % (ident, round(delay*1e6))) + time.sleep(delay) + verbose_print("task %s done" % ident) + with self.running_mutex: + self.running -= 1 + if self.created == NUMTASKS and self.running == 0: + self.done_mutex.release() + + def test_starting_threads(self): + # Basic test for thread creation. + for i in range(NUMTASKS): + self.newtask() + verbose_print("waiting for tasks to complete...") + self.done_mutex.acquire() + verbose_print("all tasks done") + + def test_stack_size(self): + # Various stack size tests. + self.assertEqual(thread.stack_size(), 0, "initial stack size is not 0") + + thread.stack_size(0) + self.assertEqual(thread.stack_size(), 0, "stack_size not reset to default") + + @unittest.skipIf(os.name not in ("nt", "os2", "posix"), 'test meant for nt, os2, and posix') + def test_nt_and_posix_stack_size(self): + try: + thread.stack_size(4096) + except ValueError: + verbose_print("caught expected ValueError setting " + "stack_size(4096)") + except thread.error: + self.skipTest("platform does not support changing thread stack " + "size") + + fail_msg = "stack_size(%d) failed - should succeed" + for tss in (262144, 0x100000, 0): + thread.stack_size(tss) + self.assertEqual(thread.stack_size(), tss, fail_msg % tss) + verbose_print("successfully set stack_size(%d)" % tss) + + for tss in (262144, 0x100000): + verbose_print("trying stack_size = (%d)" % tss) + self.next_ident = 0 + self.created = 0 + for i in range(NUMTASKS): + self.newtask() + + verbose_print("waiting for all tasks to complete") + self.done_mutex.acquire() + verbose_print("all tasks done") + + thread.stack_size(0) + + def test__count(self): + # Test the _count() function. + orig = thread._count() + mut = thread.allocate_lock() + mut.acquire() + started = [] + def task(): + started.append(None) + mut.acquire() + mut.release() + thread.start_new_thread(task, ()) + while not started: + time.sleep(0.01) + self.assertEqual(thread._count(), orig + 1) + # Allow the task to finish. + mut.release() + # The only reliable way to be sure that the thread ended from the + # interpreter's point of view is to wait for the function object to be + # destroyed. + done = [] + wr = weakref.ref(task, lambda _: done.append(None)) + del task + while not done: + time.sleep(0.01) + self.assertEqual(thread._count(), orig) + + def test_save_exception_state_on_error(self): + # See issue #14474 + def task(): + started.release() + raise SyntaxError + def mywrite(self, *args): + try: + raise ValueError + except ValueError: + pass + real_write(self, *args) + c = thread._count() + started = thread.allocate_lock() + with test_support.captured_output("stderr") as stderr: + real_write = stderr.write + stderr.write = mywrite + started.acquire() + thread.start_new_thread(task, ()) + started.acquire() + while thread._count() > c: + time.sleep(0.01) + self.assertIn("Traceback", stderr.getvalue()) + + +class Barrier: + def __init__(self, num_threads): + self.num_threads = num_threads + self.waiting = 0 + self.checkin_mutex = thread.allocate_lock() + self.checkout_mutex = thread.allocate_lock() + self.checkout_mutex.acquire() + + def enter(self): + self.checkin_mutex.acquire() + self.waiting = self.waiting + 1 + if self.waiting == self.num_threads: + self.waiting = self.num_threads - 1 + self.checkout_mutex.release() + return + self.checkin_mutex.release() + + self.checkout_mutex.acquire() + self.waiting = self.waiting - 1 + if self.waiting == 0: + self.checkin_mutex.release() + return + self.checkout_mutex.release() + + +class BarrierTest(BasicThreadTest): + + def test_barrier(self): + self.bar = Barrier(NUMTASKS) + self.running = NUMTASKS + for i in range(NUMTASKS): + thread.start_new_thread(self.task2, (i,)) + verbose_print("waiting for tasks to end") + self.done_mutex.acquire() + verbose_print("tasks done") + + def task2(self, ident): + for i in range(NUMTRIPS): + if ident == 0: + # give it a good chance to enter the next + # barrier before the others are all out + # of the current one + delay = 0 + else: + with self.random_mutex: + delay = random.random() / 10000.0 + verbose_print("task %s will run for %sus" % + (ident, round(delay * 1e6))) + time.sleep(delay) + verbose_print("task %s entering %s" % (ident, i)) + self.bar.enter() + verbose_print("task %s leaving barrier" % ident) + with self.running_mutex: + self.running -= 1 + # Must release mutex before releasing done, else the main thread can + # exit and set mutex to None as part of global teardown; then + # mutex.release() raises AttributeError. + finished = self.running == 0 + if finished: + self.done_mutex.release() + + +class LockTests(lock_tests.LockTests): + locktype = thread.allocate_lock + + +class TestForkInThread(unittest.TestCase): + def setUp(self): + self.read_fd, self.write_fd = os.pipe() + + @unittest.skipIf(sys.platform.startswith('win'), + "This test is only appropriate for POSIX-like systems.") + @test_support.reap_threads + def test_forkinthread(self): + def thread1(): + try: + pid = os.fork() # fork in a thread + except RuntimeError: + sys.exit(0) # exit the child + + if pid == 0: # child + os.close(self.read_fd) + os.write(self.write_fd, "OK") + sys.exit(0) + else: # parent + os.close(self.write_fd) + + thread.start_new_thread(thread1, ()) + self.assertEqual(os.read(self.read_fd, 2), "OK", + "Unable to fork() in thread") + + def tearDown(self): + try: + os.close(self.read_fd) + except OSError: + pass + + try: + os.close(self.write_fd) + except OSError: + pass + + +def test_main(): + test_support.run_unittest(ThreadRunningTests, BarrierTest, LockTests, + TestForkInThread) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_threading.py b/src/greentest/2.7/test_threading.py new file mode 100644 index 0000000..9e0b03e --- /dev/null +++ b/src/greentest/2.7/test_threading.py @@ -0,0 +1,933 @@ +# Very rudimentary test of threading module + +import test.test_support +from test.test_support import verbose, cpython_only +from test.script_helper import assert_python_ok + +import random +import re +import sys +thread = test.test_support.import_module('thread') +threading = test.test_support.import_module('threading') +import time +import unittest +import weakref +import os +import subprocess +try: + import _testcapi +except ImportError: + _testcapi = None + +from gevent.tests import lock_tests # gevent: use local copy + +# A trivial mutable counter. +class Counter(object): + def __init__(self): + self.value = 0 + def inc(self): + self.value += 1 + def dec(self): + self.value -= 1 + def get(self): + return self.value + +class TestThread(threading.Thread): + def __init__(self, name, testcase, sema, mutex, nrunning): + threading.Thread.__init__(self, name=name) + self.testcase = testcase + self.sema = sema + self.mutex = mutex + self.nrunning = nrunning + + def run(self): + delay = random.random() / 10000.0 + if verbose: + print 'task %s will run for %.1f usec' % ( + self.name, delay * 1e6) + + with self.sema: + with self.mutex: + self.nrunning.inc() + if verbose: + print self.nrunning.get(), 'tasks are running' + self.testcase.assertTrue(self.nrunning.get() <= 3) + + time.sleep(delay) + if verbose: + print 'task', self.name, 'done' + + with self.mutex: + self.nrunning.dec() + self.testcase.assertTrue(self.nrunning.get() >= 0) + if verbose: + print '%s is finished. %d tasks are running' % ( + self.name, self.nrunning.get()) + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test.test_support.threading_setup() + + def tearDown(self): + test.test_support.threading_cleanup(*self._threads) + test.test_support.reap_children() + + +class ThreadTests(BaseTestCase): + + # Create a bunch of threads, let each do some work, wait until all are + # done. + def test_various_ops(self): + # This takes about n/3 seconds to run (about n/3 clumps of tasks, + # times about 1 second per clump). + NUMTASKS = 10 + + # no more than 3 of the 10 can run at once + sema = threading.BoundedSemaphore(value=3) + mutex = threading.RLock() + numrunning = Counter() + + threads = [] + + for i in range(NUMTASKS): + t = TestThread(""%i, self, sema, mutex, numrunning) + threads.append(t) + self.assertEqual(t.ident, None) + self.assertTrue(re.match('', repr(t))) + t.start() + + if verbose: + print 'waiting for all tasks to complete' + for t in threads: + t.join(NUMTASKS) + self.assertTrue(not t.is_alive()) + self.assertNotEqual(t.ident, 0) + self.assertFalse(t.ident is None) + self.assertTrue(re.match('', repr(t))) + if verbose: + print 'all tasks done' + self.assertEqual(numrunning.get(), 0) + + def test_ident_of_no_threading_threads(self): + # The ident still must work for the main thread and dummy threads. + self.assertFalse(threading.currentThread().ident is None) + def f(): + ident.append(threading.currentThread().ident) + done.set() + done = threading.Event() + ident = [] + thread.start_new_thread(f, ()) + done.wait() + self.assertFalse(ident[0] is None) + # Kill the "immortal" _DummyThread + del threading._active[ident[0]] + + # run with a small(ish) thread stack size (256kB) + def test_various_ops_small_stack(self): + if verbose: + print 'with 256kB thread stack size...' + try: + threading.stack_size(262144) + except thread.error: + self.skipTest('platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1MB) + def test_various_ops_large_stack(self): + if verbose: + print 'with 1MB thread stack size...' + try: + threading.stack_size(0x100000) + except thread.error: + self.skipTest('platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + def test_foreign_thread(self): + # Check that a "foreign" thread can use the threading module. + def f(mutex): + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + threading.current_thread() + mutex.release() + + mutex = threading.Lock() + mutex.acquire() + tid = thread.start_new_thread(f, (mutex,)) + # Wait for the thread to finish. + mutex.acquire() + self.assertIn(tid, threading._active) + self.assertIsInstance(threading._active[tid], threading._DummyThread) + del threading._active[tid] + + # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) + # exposed at the Python level. This test relies on ctypes to get at it. + def test_PyThreadState_SetAsyncExc(self): + try: + import ctypes + except ImportError: + self.skipTest('requires ctypes') + + set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc + + class AsyncExc(Exception): + pass + + exception = ctypes.py_object(AsyncExc) + + # First check it works when setting the exception from the same thread. + tid = thread.get_ident() + + try: + result = set_async_exc(ctypes.c_long(tid), exception) + # The exception is async, so we might have to keep the VM busy until + # it notices. + while True: + pass + except AsyncExc: + pass + else: + # This code is unreachable but it reflects the intent. If we wanted + # to be smarter the above loop wouldn't be infinite. + self.fail("AsyncExc not raised") + try: + self.assertEqual(result, 1) # one thread state modified + except UnboundLocalError: + # The exception was raised too quickly for us to get the result. + pass + + # `worker_started` is set by the thread when it's inside a try/except + # block waiting to catch the asynchronously set AsyncExc exception. + # `worker_saw_exception` is set by the thread upon catching that + # exception. + worker_started = threading.Event() + worker_saw_exception = threading.Event() + + class Worker(threading.Thread): + def run(self): + self.id = thread.get_ident() + self.finished = False + + try: + while True: + worker_started.set() + time.sleep(0.1) + except AsyncExc: + self.finished = True + worker_saw_exception.set() + + t = Worker() + t.daemon = True # so if this fails, we don't hang Python at shutdown + t.start() + if verbose: + print " started worker thread" + + # Try a thread id that doesn't make sense. + if verbose: + print " trying nonsensical thread id" + result = set_async_exc(ctypes.c_long(-1), exception) + self.assertEqual(result, 0) # no thread states modified + + # Now raise an exception in the worker thread. + if verbose: + print " waiting for worker thread to get started" + ret = worker_started.wait() + self.assertTrue(ret) + if verbose: + print " verifying worker hasn't exited" + self.assertTrue(not t.finished) + if verbose: + print " attempting to raise asynch exception in worker" + result = set_async_exc(ctypes.c_long(t.id), exception) + self.assertEqual(result, 1) # one thread state modified + if verbose: + print " waiting for worker to say it caught the exception" + worker_saw_exception.wait(timeout=10) + self.assertTrue(t.finished) + if verbose: + print " all OK -- joining worker" + if t.finished: + t.join() + # else the thread is still running, and we have no way to kill it + + def test_limbo_cleanup(self): + # Issue 7481: Failure to start thread should cleanup the limbo map. + def fail_new_thread(*args): + raise thread.error() + _start_new_thread = threading._start_new_thread + threading._start_new_thread = fail_new_thread + try: + t = threading.Thread(target=lambda: None) + self.assertRaises(thread.error, t.start) + self.assertFalse( + t in threading._limbo, + "Failed to cleanup _limbo map on failure of Thread.start().") + finally: + threading._start_new_thread = _start_new_thread + + def test_finalize_runnning_thread(self): + # Issue 1402: the PyGILState_Ensure / _Release functions may be called + # very late on python exit: on deallocation of a running thread for + # example. + try: + import ctypes + except ImportError: + self.skipTest('requires ctypes') + + rc = subprocess.call([sys.executable, "-c", """if 1: + import ctypes, sys, time, thread + + # This lock is used as a simple event variable. + ready = thread.allocate_lock() + ready.acquire() + + # Module globals are cleared before __del__ is run + # So we save the functions in class dict + class C: + ensure = ctypes.pythonapi.PyGILState_Ensure + release = ctypes.pythonapi.PyGILState_Release + def __del__(self): + state = self.ensure() + self.release(state) + + def waitingThread(): + x = C() + ready.release() + time.sleep(100) + + thread.start_new_thread(waitingThread, ()) + ready.acquire() # Be sure the other thread is waiting. + sys.exit(42) + """]) + self.assertEqual(rc, 42) + + def test_finalize_with_trace(self): + # Issue1733757 + # Avoid a deadlock when sys.settrace steps into threading._shutdown + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, threading + + # A deadlock-killer, to prevent the + # testsuite to hang forever + def killer(): + import os, time + time.sleep(2) + print 'program blocked; aborting' + os._exit(2) + t = threading.Thread(target=killer) + t.daemon = True + t.start() + + # This is the trace function + def func(frame, event, arg): + threading.current_thread() + return func + + sys.settrace(func) + """], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + stdout, stderr = p.communicate() + rc = p.returncode + self.assertFalse(rc == 2, "interpreted was blocked") + self.assertTrue(rc == 0, + "Unexpected error: " + repr(stderr)) + + def test_join_nondaemon_on_shutdown(self): + # Issue 1722344 + # Raising SystemExit skipped threading._shutdown + p = subprocess.Popen([sys.executable, "-c", """if 1: + import threading + from time import sleep + + def child(): + sleep(1) + # As a non-daemon thread we SHOULD wake up and nothing + # should be torn down yet + print "Woke up, sleep function is:", sleep + + threading.Thread(target=child).start() + raise SystemExit + """], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + stdout, stderr = p.communicate() + self.assertEqual(stdout.strip(), + "Woke up, sleep function is: ") + stderr = re.sub(r"^\[\d+ refs\]", "", stderr, re.MULTILINE).strip() + self.assertEqual(stderr, "") + + def test_enumerate_after_join(self): + # Try hard to trigger #1703448: a thread is still returned in + # threading.enumerate() after it has been join()ed. + enum = threading.enumerate + old_interval = sys.getcheckinterval() + try: + for i in xrange(1, 100): + # Try a couple times at each thread-switching interval + # to get more interleavings. + sys.setcheckinterval(i // 5) + t = threading.Thread(target=lambda: None) + t.start() + t.join() + l = enum() + self.assertNotIn(t, l, + "#1703448 triggered after %d trials: %s" % (i, l)) + finally: + sys.setcheckinterval(old_interval) + + def test_no_refcycle_through_target(self): + class RunSelfFunction(object): + def __init__(self, should_raise): + # The links in this refcycle from Thread back to self + # should be cleaned up when the thread completes. + self.should_raise = should_raise + self.thread = threading.Thread(target=self._run, + args=(self,), + kwargs={'yet_another':self}) + self.thread.start() + + def _run(self, other_ref, yet_another): + if self.should_raise: + raise SystemExit + + cyclic_object = RunSelfFunction(should_raise=False) + weak_cyclic_object = weakref.ref(cyclic_object) + cyclic_object.thread.join() + del cyclic_object + self.assertEqual(None, weak_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_cyclic_object()))) + + raising_cyclic_object = RunSelfFunction(should_raise=True) + weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) + raising_cyclic_object.thread.join() + del raising_cyclic_object + self.assertEqual(None, weak_raising_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_raising_cyclic_object()))) + + @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + def test_dummy_thread_after_fork(self): + # Issue #14308: a dummy thread in the active list doesn't mess up + # the after-fork mechanism. + code = """if 1: + import thread, threading, os, time + + def background_thread(evt): + # Creates and registers the _DummyThread instance + threading.current_thread() + evt.set() + time.sleep(10) + + evt = threading.Event() + thread.start_new_thread(background_thread, (evt,)) + evt.wait() + assert threading.active_count() == 2, threading.active_count() + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + os.wait() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out, '') + self.assertEqual(err, '') + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + old_interval = sys.getcheckinterval() + + # Make the bug more likely to manifest. + sys.setcheckinterval(10) + + try: + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + pid = os.fork() + if pid == 0: + os._exit(1 if t.is_alive() else 0) + else: + t.join() + pid, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + finally: + sys.setcheckinterval(old_interval) + + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) + +class ThreadJoinOnShutdown(BaseTestCase): + + # Between fork() and exec(), only async-safe functions are allowed (issues + # #12316 and #11870), and fork() from a worker thread is known to trigger + # problems with some operating systems (issue #3863): skip problematic tests + # on platforms known to behave badly. + platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', + 'os2emx') + + def _run_and_join(self, script): + script = """if 1: + import sys, os, time, threading + + # a thread, which waits for the main program to terminate + def joiningfunc(mainthread): + mainthread.join() + print 'end of thread' + \n""" + script + + p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE) + rc = p.wait() + data = p.stdout.read().replace('\r', '') + p.stdout.close() + self.assertEqual(data, "end of main\nend of thread\n") + self.assertFalse(rc == 2, "interpreter was blocked") + self.assertTrue(rc == 0, "Unexpected error") + + def test_1_join_on_shutdown(self): + # The usual case: on exit, wait for a non-daemon thread + script = """if 1: + import os + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + time.sleep(0.1) + print 'end of main' + """ + self._run_and_join(script) + + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_2_join_in_forked_process(self): + # Like the test above, but from a forked interpreter + script = """if 1: + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + print 'end of main' + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_3_join_in_forked_from_thread(self): + # Like the test above, but fork() was called from a worker thread + # In the forked process, the main Thread object must be marked as stopped. + script = """if 1: + main_thread = threading.current_thread() + def worker(): + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(main_thread,)) + print 'end of main' + t.start() + t.join() # Should not block: main_thread is already stopped + + w = threading.Thread(target=worker) + w.start() + """ + self._run_and_join(script) + + def assertScriptHasOutput(self, script, expected_output): + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE) + rc = p.wait() + data = p.stdout.read().decode().replace('\r', '') + self.assertEqual(rc, 0, "Unexpected error") + self.assertEqual(data, expected_output) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_4_joining_across_fork_in_worker_thread(self): + # There used to be a possible deadlock when forking from a child + # thread. See http://bugs.python.org/issue6643. + + # The script takes the following steps: + # - The main thread in the parent process starts a new thread and then + # tries to join it. + # - The join operation acquires the Lock inside the thread's _block + # Condition. (See threading.py:Thread.join().) + # - We stub out the acquire method on the condition to force it to wait + # until the child thread forks. (See LOCK ACQUIRED HERE) + # - The child thread forks. (See LOCK HELD and WORKER THREAD FORKS + # HERE) + # - The main thread of the parent process enters Condition.wait(), + # which releases the lock on the child thread. + # - The child process returns. Without the necessary fix, when the + # main thread of the child process (which used to be the child thread + # in the parent process) attempts to exit, it will try to acquire the + # lock in the Thread._block Condition object and hang, because the + # lock was held across the fork. + + script = """if 1: + import os, time, threading + + finish_join = False + start_fork = False + + def worker(): + # Wait until this thread's lock is acquired before forking to + # create the deadlock. + global finish_join + while not start_fork: + time.sleep(0.01) + # LOCK HELD: Main thread holds lock across this call. + childpid = os.fork() + finish_join = True + if childpid != 0: + # Parent process just waits for child. + os.waitpid(childpid, 0) + # Child process should just return. + + w = threading.Thread(target=worker) + + # Stub out the private condition variable's lock acquire method. + # This acquires the lock and then waits until the child has forked + # before returning, which will release the lock soon after. If + # someone else tries to fix this test case by acquiring this lock + # before forking instead of resetting it, the test case will + # deadlock when it shouldn't. + condition = w._block + orig_acquire = condition.acquire + call_count_lock = threading.Lock() + call_count = 0 + def my_acquire(): + global call_count + global start_fork + orig_acquire() # LOCK ACQUIRED HERE + start_fork = True + if call_count == 0: + while not finish_join: + time.sleep(0.01) # WORKER THREAD FORKS HERE + with call_count_lock: + call_count += 1 + condition.acquire = my_acquire + + w.start() + w.join() + print('end of main') + """ + self.assertScriptHasOutput(script, "end of main\n") + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_5_clear_waiter_locks_to_avoid_crash(self): + # Check that a spawned thread that forks doesn't segfault on certain + # platforms, namely OS X. This used to happen if there was a waiter + # lock in the thread's condition variable's waiters list. Even though + # we know the lock will be held across the fork, it is not safe to + # release locks held across forks on all platforms, so releasing the + # waiter lock caused a segfault on OS X. Furthermore, since locks on + # OS X are (as of this writing) implemented with a mutex + condition + # variable instead of a semaphore, while we know that the Python-level + # lock will be acquired, we can't know if the internal mutex will be + # acquired at the time of the fork. + + script = """if True: + import os, time, threading + + start_fork = False + + def worker(): + # Wait until the main thread has attempted to join this thread + # before continuing. + while not start_fork: + time.sleep(0.01) + childpid = os.fork() + if childpid != 0: + # Parent process just waits for child. + (cpid, rc) = os.waitpid(childpid, 0) + assert cpid == childpid + assert rc == 0 + print('end of worker thread') + else: + # Child process should just return. + pass + + w = threading.Thread(target=worker) + + # Stub out the private condition variable's _release_save method. + # This releases the condition's lock and flips the global that + # causes the worker to fork. At this point, the problematic waiter + # lock has been acquired once by the waiter and has been put onto + # the waiters list. + condition = w._block + orig_release_save = condition._release_save + def my_release_save(): + global start_fork + orig_release_save() + # Waiter lock held here, condition lock released. + start_fork = True + condition._release_save = my_release_save + + w.start() + w.join() + print('end of main thread') + """ + output = "end of worker thread\nend of main thread\n" + self.assertScriptHasOutput(script, output) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_reinit_tls_after_fork(self): + # Issue #13817: fork() would deadlock in a multithreaded program with + # the ad-hoc TLS implementation. + + def do_fork_and_wait(): + # just fork a child process and wait it + pid = os.fork() + if pid > 0: + os.waitpid(pid, 0) + else: + os._exit(0) + + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() + + for t in threads: + t.join() + + @cpython_only + @unittest.skipIf(_testcapi is None, "need _testcapi module") + def test_frame_tstate_tracing(self): + # Issue #14432: Crash when a generator is created in a C thread that is + # destroyed while the generator is still used. The issue was that a + # generator contains a frame, and the frame kept a reference to the + # Python state of the destroyed C thread. The crash occurs when a trace + # function is setup. + + def noop_trace(frame, event, arg): + # no operation + return noop_trace + + def generator(): + while 1: + yield "genereator" + + def callback(): + if callback.gen is None: + callback.gen = generator() + return next(callback.gen) + callback.gen = None + + old_trace = sys.gettrace() + sys.settrace(noop_trace) + try: + # Install a trace function + threading.settrace(noop_trace) + + # Create a generator in a C thread which exits after the call + _testcapi.call_in_temporary_c_thread(callback) + + # Call the generator in a different Python thread, check that the + # generator didn't keep a reference to the destroyed thread state + for test in range(3): + # The trace function is still called here + callback() + finally: + sys.settrace(old_trace) + + +class ThreadingExceptionTests(BaseTestCase): + # A RuntimeError should be raised if Thread.start() is called + # multiple times. + def test_start_thread_again(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, thread.start) + + def test_joining_current_thread(self): + current_thread = threading.current_thread() + self.assertRaises(RuntimeError, current_thread.join); + + def test_joining_inactive_thread(self): + thread = threading.Thread() + self.assertRaises(RuntimeError, thread.join) + + def test_daemonize_active_thread(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + + def test_print_exception(self): + script = r"""if 1: + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1.0/0.0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, '') + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_1(self): + script = r"""if 1: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1.0/0.0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + sys.stderr = None + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, '') + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_2(self): + script = r"""if 1: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1.0/0.0 + sys.stderr = None + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, '') + self.assertNotIn("Unhandled exception", err) + + +class LockTests(lock_tests.LockTests): + locktype = staticmethod(threading.Lock) + +class RLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading.RLock) + +class EventTests(lock_tests.EventTests): + eventtype = staticmethod(threading.Event) + +class ConditionAsRLockTests(lock_tests.RLockTests): + # An Condition uses an RLock by default and exports its API. + locktype = staticmethod(threading.Condition) + +class ConditionTests(lock_tests.ConditionTests): + condtype = staticmethod(threading.Condition) + +class SemaphoreTests(lock_tests.SemaphoreTests): + semtype = staticmethod(threading.Semaphore) + +class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): + semtype = staticmethod(threading.BoundedSemaphore) + + @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem') + def test_recursion_limit(self): + # Issue 9670 + # test that excessive recursion within a non-main thread causes + # an exception rather than crashing the interpreter on platforms + # like Mac OS X or FreeBSD which have small default stack sizes + # for threads + script = """if True: + import threading + + def recurse(): + return recurse() + + def outer(): + try: + recurse() + except RuntimeError: + pass + + w = threading.Thread(target=outer) + w.start() + w.join() + print('end of main thread') + """ + expected_output = "end of main thread\n" + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + data = stdout.decode().replace('\r', '') + self.assertEqual(p.returncode, 0, "Unexpected error") + self.assertEqual(data, expected_output) + +def test_main(): + test.test_support.run_unittest(LockTests, RLockTests, EventTests, + ConditionAsRLockTests, ConditionTests, + SemaphoreTests, BoundedSemaphoreTests, + ThreadTests, + ThreadJoinOnShutdown, + ThreadingExceptionTests, + ) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_threading_local.py b/src/greentest/2.7/test_threading_local.py new file mode 100644 index 0000000..b161315 --- /dev/null +++ b/src/greentest/2.7/test_threading_local.py @@ -0,0 +1,229 @@ +import unittest +from doctest import DocTestSuite +from test import test_support as support +import weakref +import gc + +# Modules under test +_thread = support.import_module('thread') +threading = support.import_module('threading') +import _threading_local + + +class Weak(object): + pass + +def target(local, weaklist): + weak = Weak() + local.weak = weak + weaklist.append(weakref.ref(weak)) + +class BaseLocalTest: + + def test_local_refs(self): + self._local_refs(20) + self._local_refs(50) + self._local_refs(100) + + def _local_refs(self, n): + local = self._local() + weaklist = [] + for i in range(n): + t = threading.Thread(target=target, args=(local, weaklist)) + t.start() + t.join() + del t + + gc.collect() + self.assertEqual(len(weaklist), n) + + # XXX _threading_local keeps the local of the last stopped thread alive. + deadlist = [weak for weak in weaklist if weak() is None] + self.assertIn(len(deadlist), (n-1, n)) + + # Assignment to the same thread local frees it sometimes (!) + local.someothervar = None + gc.collect() + deadlist = [weak for weak in weaklist if weak() is None] + self.assertIn(len(deadlist), (n-1, n), (n, len(deadlist))) + + def test_derived(self): + # Issue 3088: if there is a threads switch inside the __init__ + # of a threading.local derived class, the per-thread dictionary + # is created but not correctly set on the object. + # The first member set may be bogus. + import time + class Local(self._local): + def __init__(self): + time.sleep(0.01) + local = Local() + + def f(i): + local.x = i + # Simply check that the variable is correctly set + self.assertEqual(local.x, i) + + with support.start_threads(threading.Thread(target=f, args=(i,)) + for i in range(10)): + pass + + def test_derived_cycle_dealloc(self): + # http://bugs.python.org/issue6990 + class Local(self._local): + pass + locals = None + passed = [False] + e1 = threading.Event() + e2 = threading.Event() + + def f(): + # 1) Involve Local in a cycle + cycle = [Local()] + cycle.append(cycle) + cycle[0].foo = 'bar' + + # 2) GC the cycle (triggers threadmodule.c::local_clear + # before local_dealloc) + del cycle + gc.collect() + e1.set() + e2.wait() + + # 4) New Locals should be empty + passed[0] = all(not hasattr(local, 'foo') for local in locals) + + t = threading.Thread(target=f) + t.start() + e1.wait() + + # 3) New Locals should recycle the original's address. Creating + # them in the thread overwrites the thread state and avoids the + # bug + locals = [Local() for i in range(10)] + e2.set() + t.join() + + self.assertTrue(passed[0]) + + def test_arguments(self): + # Issue 1522237 + from thread import _local as local + from _threading_local import local as py_local + + for cls in (local, py_local): + class MyLocal(cls): + def __init__(self, *args, **kwargs): + pass + + MyLocal(a=1) + MyLocal(1) + self.assertRaises(TypeError, cls, a=1) + self.assertRaises(TypeError, cls, 1) + + def _test_one_class(self, c): + self._failed = "No error message set or cleared." + obj = c() + e1 = threading.Event() + e2 = threading.Event() + + def f1(): + obj.x = 'foo' + obj.y = 'bar' + del obj.y + e1.set() + e2.wait() + + def f2(): + try: + foo = obj.x + except AttributeError: + # This is expected -- we haven't set obj.x in this thread yet! + self._failed = "" # passed + else: + self._failed = ('Incorrectly got value %r from class %r\n' % + (foo, c)) + sys.stderr.write(self._failed) + + t1 = threading.Thread(target=f1) + t1.start() + e1.wait() + t2 = threading.Thread(target=f2) + t2.start() + t2.join() + # The test is done; just let t1 know it can exit, and wait for it. + e2.set() + t1.join() + + self.assertFalse(self._failed, self._failed) + + def test_threading_local(self): + self._test_one_class(self._local) + + def test_threading_local_subclass(self): + class LocalSubclass(self._local): + """To test that subclasses behave properly.""" + self._test_one_class(LocalSubclass) + + def _test_dict_attribute(self, cls): + obj = cls() + obj.x = 5 + self.assertEqual(obj.__dict__, {'x': 5}) + with self.assertRaises(AttributeError): + obj.__dict__ = {} + with self.assertRaises(AttributeError): + del obj.__dict__ + + def test_dict_attribute(self): + self._test_dict_attribute(self._local) + + def test_dict_attribute_subclass(self): + class LocalSubclass(self._local): + """To test that subclasses behave properly.""" + self._test_dict_attribute(LocalSubclass) + + +class ThreadLocalTest(unittest.TestCase, BaseLocalTest): + _local = _thread._local + + # Fails for the pure Python implementation + def test_cycle_collection(self): + class X: + pass + + x = X() + x.local = self._local() + x.local.x = x + wr = weakref.ref(x) + del x + gc.collect() + self.assertIs(wr(), None) + +class PyThreadingLocalTest(unittest.TestCase, BaseLocalTest): + _local = _threading_local.local + + +def test_main(): + suite = unittest.TestSuite() + suite.addTest(DocTestSuite('_threading_local')) + suite.addTest(unittest.makeSuite(ThreadLocalTest)) + suite.addTest(unittest.makeSuite(PyThreadingLocalTest)) + + try: + from thread import _local + except ImportError: + pass + else: + import _threading_local + local_orig = _threading_local.local + def setUp(test): + _threading_local.local = _local + def tearDown(test): + _threading_local.local = local_orig + suite.addTest(DocTestSuite('_threading_local', + setUp=setUp, tearDown=tearDown) + ) + + support.run_unittest(suite) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7/test_timeout.py b/src/greentest/2.7/test_timeout.py new file mode 100644 index 0000000..bb9252d --- /dev/null +++ b/src/greentest/2.7/test_timeout.py @@ -0,0 +1,205 @@ +"""Unit tests for socket timeout feature.""" + +import unittest +from test import test_support + +# This requires the 'network' resource as given on the regrtest command line. +skip_expected = not test_support.is_resource_enabled('network') + +import time +import socket + + +class CreationTestCase(unittest.TestCase): + """Test case for socket.gettimeout() and socket.settimeout()""" + + def setUp(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def tearDown(self): + self.sock.close() + + def testObjectCreation(self): + # Test Socket creation + self.assertEqual(self.sock.gettimeout(), None, + "timeout not disabled by default") + + def testFloatReturnValue(self): + # Test return value of gettimeout() + self.sock.settimeout(7.345) + self.assertEqual(self.sock.gettimeout(), 7.345) + + self.sock.settimeout(3) + self.assertEqual(self.sock.gettimeout(), 3) + + self.sock.settimeout(None) + self.assertEqual(self.sock.gettimeout(), None) + + def testReturnType(self): + # Test return type of gettimeout() + self.sock.settimeout(1) + self.assertEqual(type(self.sock.gettimeout()), type(1.0)) + + self.sock.settimeout(3.9) + self.assertEqual(type(self.sock.gettimeout()), type(1.0)) + + def testTypeCheck(self): + # Test type checking by settimeout() + self.sock.settimeout(0) + self.sock.settimeout(0L) + self.sock.settimeout(0.0) + self.sock.settimeout(None) + self.assertRaises(TypeError, self.sock.settimeout, "") + self.assertRaises(TypeError, self.sock.settimeout, u"") + self.assertRaises(TypeError, self.sock.settimeout, ()) + self.assertRaises(TypeError, self.sock.settimeout, []) + self.assertRaises(TypeError, self.sock.settimeout, {}) + self.assertRaises(TypeError, self.sock.settimeout, 0j) + + def testRangeCheck(self): + # Test range checking by settimeout() + self.assertRaises(ValueError, self.sock.settimeout, -1) + self.assertRaises(ValueError, self.sock.settimeout, -1L) + self.assertRaises(ValueError, self.sock.settimeout, -1.0) + + def testTimeoutThenBlocking(self): + # Test settimeout() followed by setblocking() + self.sock.settimeout(10) + self.sock.setblocking(1) + self.assertEqual(self.sock.gettimeout(), None) + self.sock.setblocking(0) + self.assertEqual(self.sock.gettimeout(), 0.0) + + self.sock.settimeout(10) + self.sock.setblocking(0) + self.assertEqual(self.sock.gettimeout(), 0.0) + self.sock.setblocking(1) + self.assertEqual(self.sock.gettimeout(), None) + + def testBlockingThenTimeout(self): + # Test setblocking() followed by settimeout() + self.sock.setblocking(0) + self.sock.settimeout(1) + self.assertEqual(self.sock.gettimeout(), 1) + + self.sock.setblocking(1) + self.sock.settimeout(1) + self.assertEqual(self.sock.gettimeout(), 1) + + +class TimeoutTestCase(unittest.TestCase): + """Test case for socket.socket() timeout functions""" + + # There are a number of tests here trying to make sure that an operation + # doesn't take too much longer than expected. But competing machine + # activity makes it inevitable that such tests will fail at times. + # When fuzz was at 1.0, I (tim) routinely saw bogus failures on Win2K + # and Win98SE. Boosting it to 2.0 helped a lot, but isn't a real + # solution. + fuzz = 2.0 + + def setUp(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addr_remote = ('www.python.org.', 80) + self.localhost = '127.0.0.1' + + def tearDown(self): + self.sock.close() + + def testConnectTimeout(self): + # Choose a private address that is unlikely to exist to prevent + # failures due to the connect succeeding before the timeout. + # Use a dotted IP address to avoid including the DNS lookup time + # with the connect time. This avoids failing the assertion that + # the timeout occurred fast enough. + addr = ('10.0.0.0', 12345) + + # Test connect() timeout + _timeout = 0.001 + self.sock.settimeout(_timeout) + + _t1 = time.time() + self.assertRaises(socket.error, self.sock.connect, addr) + _t2 = time.time() + + _delta = abs(_t1 - _t2) + self.assertTrue(_delta < _timeout + self.fuzz, + "timeout (%g) is more than %g seconds more than expected (%g)" + %(_delta, self.fuzz, _timeout)) + + def testRecvTimeout(self): + # Test recv() timeout + _timeout = 0.02 + + with test_support.transient_internet(self.addr_remote[0]): + self.sock.connect(self.addr_remote) + self.sock.settimeout(_timeout) + + _t1 = time.time() + self.assertRaises(socket.timeout, self.sock.recv, 1024) + _t2 = time.time() + + _delta = abs(_t1 - _t2) + self.assertTrue(_delta < _timeout + self.fuzz, + "timeout (%g) is %g seconds more than expected (%g)" + %(_delta, self.fuzz, _timeout)) + + def testAcceptTimeout(self): + # Test accept() timeout + _timeout = 2 + self.sock.settimeout(_timeout) + # Prevent "Address already in use" socket exceptions + test_support.bind_port(self.sock, self.localhost) + self.sock.listen(5) + + _t1 = time.time() + self.assertRaises(socket.error, self.sock.accept) + _t2 = time.time() + + _delta = abs(_t1 - _t2) + self.assertTrue(_delta < _timeout + self.fuzz, + "timeout (%g) is %g seconds more than expected (%g)" + %(_delta, self.fuzz, _timeout)) + + def testRecvfromTimeout(self): + # Test recvfrom() timeout + _timeout = 2 + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.settimeout(_timeout) + # Prevent "Address already in use" socket exceptions + test_support.bind_port(self.sock, self.localhost) + + _t1 = time.time() + self.assertRaises(socket.error, self.sock.recvfrom, 8192) + _t2 = time.time() + + _delta = abs(_t1 - _t2) + self.assertTrue(_delta < _timeout + self.fuzz, + "timeout (%g) is %g seconds more than expected (%g)" + %(_delta, self.fuzz, _timeout)) + + @unittest.skip('test not implemented') + def testSend(self): + # Test send() timeout + # couldn't figure out how to test it + pass + + @unittest.skip('test not implemented') + def testSendto(self): + # Test sendto() timeout + # couldn't figure out how to test it + pass + + @unittest.skip('test not implemented') + def testSendall(self): + # Test sendall() timeout + # couldn't figure out how to test it + pass + + +def test_main(): + test_support.requires('network') + test_support.run_unittest(CreationTestCase, TimeoutTestCase) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_urllib.py b/src/greentest/2.7/test_urllib.py new file mode 100644 index 0000000..adffb57 --- /dev/null +++ b/src/greentest/2.7/test_urllib.py @@ -0,0 +1,1023 @@ +"""Regresssion tests for urllib""" + +import urllib +import httplib +import unittest +import os +import sys +import mimetools +import tempfile +import StringIO + +from test import test_support +from base64 import b64encode + + +def hexescape(char): + """Escape char as RFC 2396 specifies""" + hex_repr = hex(ord(char))[2:].upper() + if len(hex_repr) == 1: + hex_repr = "0%s" % hex_repr + return "%" + hex_repr + + +class FakeHTTPMixin(object): + def fakehttp(self, fakedata): + class FakeSocket(StringIO.StringIO): + + def sendall(self, data): + FakeHTTPConnection.buf = data + + def makefile(self, *args, **kwds): + return self + + def read(self, amt=None): + if self.closed: + return "" + return StringIO.StringIO.read(self, amt) + + def readline(self, length=None): + if self.closed: + return "" + return StringIO.StringIO.readline(self, length) + + class FakeHTTPConnection(httplib.HTTPConnection): + + # buffer to store data for verification in urlopen tests. + buf = "" + + def connect(self): + self.sock = FakeSocket(fakedata) + + assert httplib.HTTP._connection_class == httplib.HTTPConnection + + httplib.HTTP._connection_class = FakeHTTPConnection + + def unfakehttp(self): + httplib.HTTP._connection_class = httplib.HTTPConnection + + +class urlopen_FileTests(unittest.TestCase): + """Test urlopen() opening a temporary file. + + Try to test as much functionality as possible so as to cut down on reliance + on connecting to the Net for testing. + + """ + + def setUp(self): + """Setup of a temp file to use for testing""" + self.text = "test_urllib: %s\n" % self.__class__.__name__ + FILE = file(test_support.TESTFN, 'wb') + try: + FILE.write(self.text) + finally: + FILE.close() + self.pathname = test_support.TESTFN + self.returned_obj = urllib.urlopen("file:%s" % self.pathname) + + def tearDown(self): + """Shut down the open object""" + self.returned_obj.close() + os.remove(test_support.TESTFN) + + def test_interface(self): + # Make sure object returned by urlopen() has the specified methods + for attr in ("read", "readline", "readlines", "fileno", + "close", "info", "geturl", "getcode", "__iter__"): + self.assertTrue(hasattr(self.returned_obj, attr), + "object returned by urlopen() lacks %s attribute" % + attr) + + def test_read(self): + self.assertEqual(self.text, self.returned_obj.read()) + + def test_readline(self): + self.assertEqual(self.text, self.returned_obj.readline()) + self.assertEqual('', self.returned_obj.readline(), + "calling readline() after exhausting the file did not" + " return an empty string") + + def test_readlines(self): + lines_list = self.returned_obj.readlines() + self.assertEqual(len(lines_list), 1, + "readlines() returned the wrong number of lines") + self.assertEqual(lines_list[0], self.text, + "readlines() returned improper text") + + def test_fileno(self): + file_num = self.returned_obj.fileno() + self.assertIsInstance(file_num, int, "fileno() did not return an int") + self.assertEqual(os.read(file_num, len(self.text)), self.text, + "Reading on the file descriptor returned by fileno() " + "did not return the expected text") + + def test_close(self): + # Test close() by calling it hear and then having it be called again + # by the tearDown() method for the test + self.returned_obj.close() + + def test_info(self): + self.assertIsInstance(self.returned_obj.info(), mimetools.Message) + + def test_geturl(self): + self.assertEqual(self.returned_obj.geturl(), self.pathname) + + def test_getcode(self): + self.assertEqual(self.returned_obj.getcode(), None) + + def test_iter(self): + # Test iterator + # Don't need to count number of iterations since test would fail the + # instant it returned anything beyond the first line from the + # comparison + for line in self.returned_obj.__iter__(): + self.assertEqual(line, self.text) + + def test_relativelocalfile(self): + self.assertRaises(ValueError,urllib.urlopen,'./' + self.pathname) + +class ProxyTests(unittest.TestCase): + + def setUp(self): + # Records changes to env vars + self.env = test_support.EnvironmentVarGuard() + # Delete all proxy related env vars + for k in os.environ.keys(): + if 'proxy' in k.lower(): + self.env.unset(k) + + def tearDown(self): + # Restore all proxy related env vars + self.env.__exit__() + del self.env + + def test_getproxies_environment_keep_no_proxies(self): + self.env.set('NO_PROXY', 'localhost') + proxies = urllib.getproxies_environment() + # getproxies_environment use lowered case truncated (no '_proxy') keys + self.assertEqual('localhost', proxies['no']) + # List of no_proxies with space. + self.env.set('NO_PROXY', 'localhost, anotherdomain.com, newdomain.com') + self.assertTrue(urllib.proxy_bypass_environment('anotherdomain.com')) + + +class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin): + """Test urlopen() opening a fake http connection.""" + + def test_read(self): + self.fakehttp('Hello!') + try: + fp = urllib.urlopen("http://python.org/") + self.assertEqual(fp.readline(), 'Hello!') + self.assertEqual(fp.readline(), '') + self.assertEqual(fp.geturl(), 'http://python.org/') + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + def test_url_fragment(self): + # Issue #11703: geturl() omits fragments in the original URL. + url = 'http://docs.python.org/library/urllib.html#OK' + self.fakehttp('Hello!') + try: + fp = urllib.urlopen(url) + self.assertEqual(fp.geturl(), url) + finally: + self.unfakehttp() + + def test_read_bogus(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp('''HTTP/1.1 401 Authentication Required +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Type: text/html; charset=iso-8859-1 +''') + try: + self.assertRaises(IOError, urllib.urlopen, "http://python.org/") + finally: + self.unfakehttp() + + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp("""HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file:README +Connection: close +Content-Type: text/html; charset=iso-8859-1 +""") + try: + self.assertRaises(IOError, urllib.urlopen, "http://python.org/") + finally: + self.unfakehttp() + + def test_empty_socket(self): + # urlopen() raises IOError if the underlying socket does not send any + # data. (#1680230) + self.fakehttp('') + try: + self.assertRaises(IOError, urllib.urlopen, 'http://something') + finally: + self.unfakehttp() + + def test_missing_localfile(self): + self.assertRaises(IOError, urllib.urlopen, + 'file://localhost/a/missing/file.py') + fd, tmp_file = tempfile.mkstemp() + tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + self.assertTrue(os.path.exists(tmp_file)) + try: + fp = urllib.urlopen(tmp_fileurl) + fp.close() + finally: + os.close(fd) + os.unlink(tmp_file) + + self.assertFalse(os.path.exists(tmp_file)) + self.assertRaises(IOError, urllib.urlopen, tmp_fileurl) + + def test_ftp_nonexisting(self): + self.assertRaises(IOError, urllib.urlopen, + 'ftp://localhost/not/existing/file.py') + + + def test_userpass_inurl(self): + self.fakehttp('Hello!') + try: + fakehttp_wrapper = httplib.HTTP._connection_class + fp = urllib.urlopen("http://user:pass@python.org/") + authorization = ("Authorization: Basic %s\r\n" % + b64encode('user:pass')) + # The authorization header must be in place + self.assertIn(authorization, fakehttp_wrapper.buf) + self.assertEqual(fp.readline(), "Hello!") + self.assertEqual(fp.readline(), "") + self.assertEqual(fp.geturl(), 'http://user:pass@python.org/') + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + def test_userpass_with_spaces_inurl(self): + self.fakehttp('Hello!') + try: + url = "http://a b:c d@python.org/" + fakehttp_wrapper = httplib.HTTP._connection_class + authorization = ("Authorization: Basic %s\r\n" % + b64encode('a b:c d')) + fp = urllib.urlopen(url) + # The authorization header must be in place + self.assertIn(authorization, fakehttp_wrapper.buf) + self.assertEqual(fp.readline(), "Hello!") + self.assertEqual(fp.readline(), "") + # the spaces are quoted in URL so no match + self.assertNotEqual(fp.geturl(), url) + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + +class urlretrieve_FileTests(unittest.TestCase): + """Test urllib.urlretrieve() on local files""" + + def setUp(self): + # Create a list of temporary files. Each item in the list is a file + # name (absolute path or relative to the current working directory). + # All files in this list will be deleted in the tearDown method. Note, + # this only helps to makes sure temporary files get deleted, but it + # does nothing about trying to close files that may still be open. It + # is the responsibility of the developer to properly close files even + # when exceptional conditions occur. + self.tempFiles = [] + + # Create a temporary file. + self.registerFileForCleanUp(test_support.TESTFN) + self.text = 'testing urllib.urlretrieve' + try: + FILE = file(test_support.TESTFN, 'wb') + FILE.write(self.text) + FILE.close() + finally: + try: FILE.close() + except: pass + + def tearDown(self): + # Delete the temporary files. + for each in self.tempFiles: + try: os.remove(each) + except: pass + + def constructLocalFileUrl(self, filePath): + return "file://%s" % urllib.pathname2url(os.path.abspath(filePath)) + + def createNewTempFile(self, data=""): + """Creates a new temporary file containing the specified data, + registers the file for deletion during the test fixture tear down, and + returns the absolute path of the file.""" + + newFd, newFilePath = tempfile.mkstemp() + try: + self.registerFileForCleanUp(newFilePath) + newFile = os.fdopen(newFd, "wb") + newFile.write(data) + newFile.close() + finally: + try: newFile.close() + except: pass + return newFilePath + + def registerFileForCleanUp(self, fileName): + self.tempFiles.append(fileName) + + def test_basic(self): + # Make sure that a local file just gets its own location returned and + # a headers value is returned. + result = urllib.urlretrieve("file:%s" % test_support.TESTFN) + self.assertEqual(result[0], test_support.TESTFN) + self.assertIsInstance(result[1], mimetools.Message, + "did not get a mimetools.Message instance as " + "second returned value") + + def test_copy(self): + # Test that setting the filename argument works. + second_temp = "%s.2" % test_support.TESTFN + self.registerFileForCleanUp(second_temp) + result = urllib.urlretrieve(self.constructLocalFileUrl( + test_support.TESTFN), second_temp) + self.assertEqual(second_temp, result[0]) + self.assertTrue(os.path.exists(second_temp), "copy of the file was not " + "made") + FILE = file(second_temp, 'rb') + try: + text = FILE.read() + FILE.close() + finally: + try: FILE.close() + except: pass + self.assertEqual(self.text, text) + + def test_reporthook(self): + # Make sure that the reporthook works. + def hooktester(count, block_size, total_size, count_holder=[0]): + self.assertIsInstance(count, int) + self.assertIsInstance(block_size, int) + self.assertIsInstance(total_size, int) + self.assertEqual(count, count_holder[0]) + count_holder[0] = count_holder[0] + 1 + second_temp = "%s.2" % test_support.TESTFN + self.registerFileForCleanUp(second_temp) + urllib.urlretrieve(self.constructLocalFileUrl(test_support.TESTFN), + second_temp, hooktester) + + def test_reporthook_0_bytes(self): + # Test on zero length file. Should call reporthook only 1 time. + report = [] + def hooktester(count, block_size, total_size, _report=report): + _report.append((count, block_size, total_size)) + srcFileName = self.createNewTempFile() + urllib.urlretrieve(self.constructLocalFileUrl(srcFileName), + test_support.TESTFN, hooktester) + self.assertEqual(len(report), 1) + self.assertEqual(report[0][2], 0) + + def test_reporthook_5_bytes(self): + # Test on 5 byte file. Should call reporthook only 2 times (once when + # the "network connection" is established and once when the block is + # read). Since the block size is 8192 bytes, only one block read is + # required to read the entire file. + report = [] + def hooktester(count, block_size, total_size, _report=report): + _report.append((count, block_size, total_size)) + srcFileName = self.createNewTempFile("x" * 5) + urllib.urlretrieve(self.constructLocalFileUrl(srcFileName), + test_support.TESTFN, hooktester) + self.assertEqual(len(report), 2) + self.assertEqual(report[0][1], 8192) + self.assertEqual(report[0][2], 5) + + def test_reporthook_8193_bytes(self): + # Test on 8193 byte file. Should call reporthook only 3 times (once + # when the "network connection" is established, once for the next 8192 + # bytes, and once for the last byte). + report = [] + def hooktester(count, block_size, total_size, _report=report): + _report.append((count, block_size, total_size)) + srcFileName = self.createNewTempFile("x" * 8193) + urllib.urlretrieve(self.constructLocalFileUrl(srcFileName), + test_support.TESTFN, hooktester) + self.assertEqual(len(report), 3) + self.assertEqual(report[0][1], 8192) + self.assertEqual(report[0][2], 8193) + + +class urlretrieve_HttpTests(unittest.TestCase, FakeHTTPMixin): + """Test urllib.urlretrieve() using fake http connections""" + + def test_short_content_raises_ContentTooShortError(self): + self.fakehttp('''HTTP/1.1 200 OK +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Length: 100 +Content-Type: text/html; charset=iso-8859-1 + +FF +''') + + def _reporthook(par1, par2, par3): + pass + + try: + self.assertRaises(urllib.ContentTooShortError, urllib.urlretrieve, + 'http://example.com', reporthook=_reporthook) + finally: + self.unfakehttp() + + def test_short_content_raises_ContentTooShortError_without_reporthook(self): + self.fakehttp('''HTTP/1.1 200 OK +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Length: 100 +Content-Type: text/html; charset=iso-8859-1 + +FF +''') + try: + self.assertRaises(urllib.ContentTooShortError, urllib.urlretrieve, 'http://example.com/') + finally: + self.unfakehttp() + +class QuotingTests(unittest.TestCase): + """Tests for urllib.quote() and urllib.quote_plus() + + According to RFC 2396 ("Uniform Resource Identifiers), to escape a + character you write it as '%' + <2 character US-ASCII hex value>. The Python + code of ``'%' + hex(ord())[2:]`` escapes a character properly. + Case does not matter on the hex letters. + + The various character sets specified are: + + Reserved characters : ";/?:@&=+$," + Have special meaning in URIs and must be escaped if not being used for + their special meaning + Data characters : letters, digits, and "-_.!~*'()" + Unreserved and do not need to be escaped; can be, though, if desired + Control characters : 0x00 - 0x1F, 0x7F + Have no use in URIs so must be escaped + space : 0x20 + Must be escaped + Delimiters : '<>#%"' + Must be escaped + Unwise : "{}|\^[]`" + Must be escaped + + """ + + def test_never_quote(self): + # Make sure quote() does not quote letters, digits, and "_,.-" + do_not_quote = '' .join(["ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "abcdefghijklmnopqrstuvwxyz", + "0123456789", + "_.-"]) + result = urllib.quote(do_not_quote) + self.assertEqual(do_not_quote, result, + "using quote(): %s != %s" % (do_not_quote, result)) + result = urllib.quote_plus(do_not_quote) + self.assertEqual(do_not_quote, result, + "using quote_plus(): %s != %s" % (do_not_quote, result)) + + def test_default_safe(self): + # Test '/' is default value for 'safe' parameter + self.assertEqual(urllib.quote.func_defaults[0], '/') + + def test_safe(self): + # Test setting 'safe' parameter does what it should do + quote_by_default = "<>" + result = urllib.quote(quote_by_default, safe=quote_by_default) + self.assertEqual(quote_by_default, result, + "using quote(): %s != %s" % (quote_by_default, result)) + result = urllib.quote_plus(quote_by_default, safe=quote_by_default) + self.assertEqual(quote_by_default, result, + "using quote_plus(): %s != %s" % + (quote_by_default, result)) + + def test_default_quoting(self): + # Make sure all characters that should be quoted are by default sans + # space (separate test for that). + should_quote = [chr(num) for num in range(32)] # For 0x00 - 0x1F + should_quote.append('<>#%"{}|\^[]`') + should_quote.append(chr(127)) # For 0x7F + should_quote = ''.join(should_quote) + for char in should_quote: + result = urllib.quote(char) + self.assertEqual(hexescape(char), result, + "using quote(): %s should be escaped to %s, not %s" % + (char, hexescape(char), result)) + result = urllib.quote_plus(char) + self.assertEqual(hexescape(char), result, + "using quote_plus(): " + "%s should be escapes to %s, not %s" % + (char, hexescape(char), result)) + del should_quote + partial_quote = "ab[]cd" + expected = "ab%5B%5Dcd" + result = urllib.quote(partial_quote) + self.assertEqual(expected, result, + "using quote(): %s != %s" % (expected, result)) + result = urllib.quote_plus(partial_quote) + self.assertEqual(expected, result, + "using quote_plus(): %s != %s" % (expected, result)) + self.assertRaises(TypeError, urllib.quote, None) + + def test_quoting_space(self): + # Make sure quote() and quote_plus() handle spaces as specified in + # their unique way + result = urllib.quote(' ') + self.assertEqual(result, hexescape(' '), + "using quote(): %s != %s" % (result, hexescape(' '))) + result = urllib.quote_plus(' ') + self.assertEqual(result, '+', + "using quote_plus(): %s != +" % result) + given = "a b cd e f" + expect = given.replace(' ', hexescape(' ')) + result = urllib.quote(given) + self.assertEqual(expect, result, + "using quote(): %s != %s" % (expect, result)) + expect = given.replace(' ', '+') + result = urllib.quote_plus(given) + self.assertEqual(expect, result, + "using quote_plus(): %s != %s" % (expect, result)) + + def test_quoting_plus(self): + self.assertEqual(urllib.quote_plus('alpha+beta gamma'), + 'alpha%2Bbeta+gamma') + self.assertEqual(urllib.quote_plus('alpha+beta gamma', '+'), + 'alpha+beta+gamma') + +class UnquotingTests(unittest.TestCase): + """Tests for unquote() and unquote_plus() + + See the doc string for quoting_Tests for details on quoting and such. + + """ + + def test_unquoting(self): + # Make sure unquoting of all ASCII values works + escape_list = [] + for num in range(128): + given = hexescape(chr(num)) + expect = chr(num) + result = urllib.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %s != %s" % (expect, result)) + result = urllib.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %s != %s" % + (expect, result)) + escape_list.append(given) + escape_string = ''.join(escape_list) + del escape_list + result = urllib.unquote(escape_string) + self.assertEqual(result.count('%'), 1, + "using quote(): not all characters escaped; %s" % + result) + result = urllib.unquote(escape_string) + self.assertEqual(result.count('%'), 1, + "using unquote(): not all characters escaped: " + "%s" % result) + + def test_unquoting_badpercent(self): + # Test unquoting on bad percent-escapes + given = '%xab' + expect = given + result = urllib.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + given = '%x' + expect = given + result = urllib.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + given = '%' + expect = given + result = urllib.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + + def test_unquoting_mixed_case(self): + # Test unquoting on mixed-case hex digits in the percent-escapes + given = '%Ab%eA' + expect = '\xab\xea' + result = urllib.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + + def test_unquoting_parts(self): + # Make sure unquoting works when have non-quoted characters + # interspersed + given = 'ab%sd' % hexescape('c') + expect = "abcd" + result = urllib.unquote(given) + self.assertEqual(expect, result, + "using quote(): %s != %s" % (expect, result)) + result = urllib.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %s != %s" % (expect, result)) + + def test_unquoting_plus(self): + # Test difference between unquote() and unquote_plus() + given = "are+there+spaces..." + expect = given + result = urllib.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %s != %s" % (expect, result)) + expect = given.replace('+', ' ') + result = urllib.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %s != %s" % (expect, result)) + + def test_unquote_with_unicode(self): + r = urllib.unquote(u'br%C3%BCckner_sapporo_20050930.doc') + self.assertEqual(r, u'br\xc3\xbcckner_sapporo_20050930.doc') + +class urlencode_Tests(unittest.TestCase): + """Tests for urlencode()""" + + def help_inputtype(self, given, test_type): + """Helper method for testing different input types. + + 'given' must lead to only the pairs: + * 1st, 1 + * 2nd, 2 + * 3rd, 3 + + Test cannot assume anything about order. Docs make no guarantee and + have possible dictionary input. + + """ + expect_somewhere = ["1st=1", "2nd=2", "3rd=3"] + result = urllib.urlencode(given) + for expected in expect_somewhere: + self.assertIn(expected, result, + "testing %s: %s not found in %s" % + (test_type, expected, result)) + self.assertEqual(result.count('&'), 2, + "testing %s: expected 2 '&'s; got %s" % + (test_type, result.count('&'))) + amp_location = result.index('&') + on_amp_left = result[amp_location - 1] + on_amp_right = result[amp_location + 1] + self.assertTrue(on_amp_left.isdigit() and on_amp_right.isdigit(), + "testing %s: '&' not located in proper place in %s" % + (test_type, result)) + self.assertEqual(len(result), (5 * 3) + 2, #5 chars per thing and amps + "testing %s: " + "unexpected number of characters: %s != %s" % + (test_type, len(result), (5 * 3) + 2)) + + def test_using_mapping(self): + # Test passing in a mapping object as an argument. + self.help_inputtype({"1st":'1', "2nd":'2', "3rd":'3'}, + "using dict as input type") + + def test_using_sequence(self): + # Test passing in a sequence of two-item sequences as an argument. + self.help_inputtype([('1st', '1'), ('2nd', '2'), ('3rd', '3')], + "using sequence of two-item tuples as input") + + def test_quoting(self): + # Make sure keys and values are quoted using quote_plus() + given = {"&":"="} + expect = "%s=%s" % (hexescape('&'), hexescape('=')) + result = urllib.urlencode(given) + self.assertEqual(expect, result) + given = {"key name":"A bunch of pluses"} + expect = "key+name=A+bunch+of+pluses" + result = urllib.urlencode(given) + self.assertEqual(expect, result) + + def test_doseq(self): + # Test that passing True for 'doseq' parameter works correctly + given = {'sequence':['1', '2', '3']} + expect = "sequence=%s" % urllib.quote_plus(str(['1', '2', '3'])) + result = urllib.urlencode(given) + self.assertEqual(expect, result) + result = urllib.urlencode(given, True) + for value in given["sequence"]: + expect = "sequence=%s" % value + self.assertIn(expect, result) + self.assertEqual(result.count('&'), 2, + "Expected 2 '&'s, got %s" % result.count('&')) + +class Pathname_Tests(unittest.TestCase): + """Test pathname2url() and url2pathname()""" + + def test_basic(self): + # Make sure simple tests pass + expected_path = os.path.join("parts", "of", "a", "path") + expected_url = "parts/of/a/path" + result = urllib.pathname2url(expected_path) + self.assertEqual(expected_url, result, + "pathname2url() failed; %s != %s" % + (result, expected_url)) + result = urllib.url2pathname(expected_url) + self.assertEqual(expected_path, result, + "url2pathame() failed; %s != %s" % + (result, expected_path)) + + def test_quoting(self): + # Test automatic quoting and unquoting works for pathnam2url() and + # url2pathname() respectively + given = os.path.join("needs", "quot=ing", "here") + expect = "needs/%s/here" % urllib.quote("quot=ing") + result = urllib.pathname2url(given) + self.assertEqual(expect, result, + "pathname2url() failed; %s != %s" % + (expect, result)) + expect = given + result = urllib.url2pathname(result) + self.assertEqual(expect, result, + "url2pathname() failed; %s != %s" % + (expect, result)) + given = os.path.join("make sure", "using_quote") + expect = "%s/using_quote" % urllib.quote("make sure") + result = urllib.pathname2url(given) + self.assertEqual(expect, result, + "pathname2url() failed; %s != %s" % + (expect, result)) + given = "make+sure/using_unquote" + expect = os.path.join("make+sure", "using_unquote") + result = urllib.url2pathname(given) + self.assertEqual(expect, result, + "url2pathname() failed; %s != %s" % + (expect, result)) + + @unittest.skipUnless(sys.platform == 'win32', + 'test specific to the nturl2path library') + def test_ntpath(self): + given = ('/C:/', '///C:/', '/C|//') + expect = 'C:\\' + for url in given: + result = urllib.url2pathname(url) + self.assertEqual(expect, result, + 'nturl2path.url2pathname() failed; %s != %s' % + (expect, result)) + given = '///C|/path' + expect = 'C:\\path' + result = urllib.url2pathname(given) + self.assertEqual(expect, result, + 'nturl2path.url2pathname() failed; %s != %s' % + (expect, result)) + +class Utility_Tests(unittest.TestCase): + """Testcase to test the various utility functions in the urllib.""" + # In Python 3 this test class is moved to test_urlparse. + + def test_splittype(self): + splittype = urllib.splittype + self.assertEqual(splittype('type:opaquestring'), ('type', 'opaquestring')) + self.assertEqual(splittype('opaquestring'), (None, 'opaquestring')) + self.assertEqual(splittype(':opaquestring'), (None, ':opaquestring')) + self.assertEqual(splittype('type:'), ('type', '')) + self.assertEqual(splittype('type:opaque:string'), ('type', 'opaque:string')) + + def test_splithost(self): + splithost = urllib.splithost + self.assertEqual(splithost('//www.example.org:80/foo/bar/baz.html'), + ('www.example.org:80', '/foo/bar/baz.html')) + self.assertEqual(splithost('//www.example.org:80'), + ('www.example.org:80', '')) + self.assertEqual(splithost('/foo/bar/baz.html'), + (None, '/foo/bar/baz.html')) + + def test_splituser(self): + splituser = urllib.splituser + self.assertEqual(splituser('User:Pass@www.python.org:080'), + ('User:Pass', 'www.python.org:080')) + self.assertEqual(splituser('@www.python.org:080'), + ('', 'www.python.org:080')) + self.assertEqual(splituser('www.python.org:080'), + (None, 'www.python.org:080')) + self.assertEqual(splituser('User:Pass@'), + ('User:Pass', '')) + self.assertEqual(splituser('User@example.com:Pass@www.python.org:080'), + ('User@example.com:Pass', 'www.python.org:080')) + + def test_splitpasswd(self): + # Some of the password examples are not sensible, but it is added to + # confirming to RFC2617 and addressing issue4675. + splitpasswd = urllib.splitpasswd + self.assertEqual(splitpasswd('user:ab'), ('user', 'ab')) + self.assertEqual(splitpasswd('user:a\nb'), ('user', 'a\nb')) + self.assertEqual(splitpasswd('user:a\tb'), ('user', 'a\tb')) + self.assertEqual(splitpasswd('user:a\rb'), ('user', 'a\rb')) + self.assertEqual(splitpasswd('user:a\fb'), ('user', 'a\fb')) + self.assertEqual(splitpasswd('user:a\vb'), ('user', 'a\vb')) + self.assertEqual(splitpasswd('user:a:b'), ('user', 'a:b')) + self.assertEqual(splitpasswd('user:a b'), ('user', 'a b')) + self.assertEqual(splitpasswd('user 2:ab'), ('user 2', 'ab')) + self.assertEqual(splitpasswd('user+1:a+b'), ('user+1', 'a+b')) + self.assertEqual(splitpasswd('user:'), ('user', '')) + self.assertEqual(splitpasswd('user'), ('user', None)) + self.assertEqual(splitpasswd(':ab'), ('', 'ab')) + + def test_splitport(self): + splitport = urllib.splitport + self.assertEqual(splitport('parrot:88'), ('parrot', '88')) + self.assertEqual(splitport('parrot'), ('parrot', None)) + self.assertEqual(splitport('parrot:'), ('parrot', None)) + self.assertEqual(splitport('127.0.0.1'), ('127.0.0.1', None)) + self.assertEqual(splitport('parrot:cheese'), ('parrot:cheese', None)) + self.assertEqual(splitport('[::1]:88'), ('[::1]', '88')) + self.assertEqual(splitport('[::1]'), ('[::1]', None)) + self.assertEqual(splitport(':88'), ('', '88')) + + def test_splitnport(self): + splitnport = urllib.splitnport + self.assertEqual(splitnport('parrot:88'), ('parrot', 88)) + self.assertEqual(splitnport('parrot'), ('parrot', -1)) + self.assertEqual(splitnport('parrot', 55), ('parrot', 55)) + self.assertEqual(splitnport('parrot:'), ('parrot', -1)) + self.assertEqual(splitnport('parrot:', 55), ('parrot', 55)) + self.assertEqual(splitnport('127.0.0.1'), ('127.0.0.1', -1)) + self.assertEqual(splitnport('127.0.0.1', 55), ('127.0.0.1', 55)) + self.assertEqual(splitnport('parrot:cheese'), ('parrot', None)) + self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None)) + + def test_splitquery(self): + # Normal cases are exercised by other tests; ensure that we also + # catch cases with no port specified (testcase ensuring coverage) + splitquery = urllib.splitquery + self.assertEqual(splitquery('http://python.org/fake?foo=bar'), + ('http://python.org/fake', 'foo=bar')) + self.assertEqual(splitquery('http://python.org/fake?foo=bar?'), + ('http://python.org/fake?foo=bar', '')) + self.assertEqual(splitquery('http://python.org/fake'), + ('http://python.org/fake', None)) + self.assertEqual(splitquery('?foo=bar'), ('', 'foo=bar')) + + def test_splittag(self): + splittag = urllib.splittag + self.assertEqual(splittag('http://example.com?foo=bar#baz'), + ('http://example.com?foo=bar', 'baz')) + self.assertEqual(splittag('http://example.com?foo=bar#'), + ('http://example.com?foo=bar', '')) + self.assertEqual(splittag('#baz'), ('', 'baz')) + self.assertEqual(splittag('http://example.com?foo=bar'), + ('http://example.com?foo=bar', None)) + self.assertEqual(splittag('http://example.com?foo=bar#baz#boo'), + ('http://example.com?foo=bar#baz', 'boo')) + + def test_splitattr(self): + splitattr = urllib.splitattr + self.assertEqual(splitattr('/path;attr1=value1;attr2=value2'), + ('/path', ['attr1=value1', 'attr2=value2'])) + self.assertEqual(splitattr('/path;'), ('/path', [''])) + self.assertEqual(splitattr(';attr1=value1;attr2=value2'), + ('', ['attr1=value1', 'attr2=value2'])) + self.assertEqual(splitattr('/path'), ('/path', [])) + + def test_splitvalue(self): + # Normal cases are exercised by other tests; test pathological cases + # with no key/value pairs. (testcase ensuring coverage) + splitvalue = urllib.splitvalue + self.assertEqual(splitvalue('foo=bar'), ('foo', 'bar')) + self.assertEqual(splitvalue('foo='), ('foo', '')) + self.assertEqual(splitvalue('=bar'), ('', 'bar')) + self.assertEqual(splitvalue('foobar'), ('foobar', None)) + self.assertEqual(splitvalue('foo=bar=baz'), ('foo', 'bar=baz')) + + def test_toBytes(self): + result = urllib.toBytes(u'http://www.python.org') + self.assertEqual(result, 'http://www.python.org') + self.assertRaises(UnicodeError, urllib.toBytes, + test_support.u(r'http://www.python.org/medi\u00e6val')) + + def test_unwrap(self): + url = urllib.unwrap('') + self.assertEqual(url, 'type://host/path') + + +class URLopener_Tests(unittest.TestCase): + """Testcase to test the open method of URLopener class.""" + + def test_quoted_open(self): + class DummyURLopener(urllib.URLopener): + def open_spam(self, url): + return url + + self.assertEqual(DummyURLopener().open( + 'spam://example/ /'),'//example/%20/') + + # test the safe characters are not quoted by urlopen + self.assertEqual(DummyURLopener().open( + "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"), + "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/") + + +# Just commented them out. +# Can't really tell why keep failing in windows and sparc. +# Everywhere else they work ok, but on those machines, sometimes +# fail in one of the tests, sometimes in other. I have a linux, and +# the tests go ok. +# If anybody has one of the problematic environments, please help! +# . Facundo +# +# def server(evt): +# import socket, time +# serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +# serv.settimeout(3) +# serv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +# serv.bind(("", 9093)) +# serv.listen(5) +# try: +# conn, addr = serv.accept() +# conn.send("1 Hola mundo\n") +# cantdata = 0 +# while cantdata < 13: +# data = conn.recv(13-cantdata) +# cantdata += len(data) +# time.sleep(.3) +# conn.send("2 No more lines\n") +# conn.close() +# except socket.timeout: +# pass +# finally: +# serv.close() +# evt.set() +# +# class FTPWrapperTests(unittest.TestCase): +# +# def setUp(self): +# import ftplib, time, threading +# ftplib.FTP.port = 9093 +# self.evt = threading.Event() +# threading.Thread(target=server, args=(self.evt,)).start() +# time.sleep(.1) +# +# def tearDown(self): +# self.evt.wait() +# +# def testBasic(self): +# # connects +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# ftp.close() +# +# def testTimeoutNone(self): +# # global default timeout is ignored +# import socket +# self.assertIsNone(socket.getdefaulttimeout()) +# socket.setdefaulttimeout(30) +# try: +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# finally: +# socket.setdefaulttimeout(None) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() +# +# def testTimeoutDefault(self): +# # global default timeout is used +# import socket +# self.assertIsNone(socket.getdefaulttimeout()) +# socket.setdefaulttimeout(30) +# try: +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# finally: +# socket.setdefaulttimeout(None) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() +# +# def testTimeoutValue(self): +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, [], +# timeout=30) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() + + + +def test_main(): + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', ".*urllib\.urlopen.*Python 3.0", + DeprecationWarning) + test_support.run_unittest( + urlopen_FileTests, + urlopen_HttpTests, + urlretrieve_FileTests, + urlretrieve_HttpTests, + ProxyTests, + QuotingTests, + UnquotingTests, + urlencode_Tests, + Pathname_Tests, + Utility_Tests, + URLopener_Tests, + #FTPWrapperTests, + ) + + + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7/test_urllib2.py b/src/greentest/2.7/test_urllib2.py new file mode 100644 index 0000000..32ffd0a --- /dev/null +++ b/src/greentest/2.7/test_urllib2.py @@ -0,0 +1,1392 @@ +import unittest +from test import test_support + +import os +import socket +import StringIO + +import urllib2 +from urllib2 import Request, OpenerDirector + +try: + import ssl +except ImportError: + ssl = None + +# XXX +# Request +# CacheFTPHandler (hard to write) +# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler + +class TrivialTests(unittest.TestCase): + def test_trivial(self): + # A couple trivial tests + + self.assertRaises(ValueError, urllib2.urlopen, 'bogus url') + + # XXX Name hacking to get this to work on Windows. + fname = os.path.abspath(urllib2.__file__).replace(os.sep, '/') + + # And more hacking to get it to work on MacOS. This assumes + # urllib.pathname2url works, unfortunately... + if os.name == 'riscos': + import string + fname = os.expand(fname) + fname = fname.translate(string.maketrans("/.", "./")) + + if os.name == 'nt': + file_url = "file:///%s" % fname + else: + file_url = "file://%s" % fname + + f = urllib2.urlopen(file_url) + + buf = f.read() + f.close() + + def test_parse_http_list(self): + tests = [('a,b,c', ['a', 'b', 'c']), + ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']), + ('a, b, "c", "d", "e,f", g, h', ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']), + ('a="b\\"c", d="e\\,f", g="h\\\\i"', ['a="b"c"', 'd="e,f"', 'g="h\\i"'])] + for string, list in tests: + self.assertEqual(urllib2.parse_http_list(string), list) + + @unittest.skipUnless(ssl, "ssl module required") + def test_cafile_and_context(self): + context = ssl.create_default_context() + with self.assertRaises(ValueError): + urllib2.urlopen( + "https://localhost", cafile="/nonexistent/path", context=context + ) + + +def test_request_headers_dict(): + """ + The Request.headers dictionary is not a documented interface. It should + stay that way, because the complete set of headers are only accessible + through the .get_header(), .has_header(), .header_items() interface. + However, .headers pre-dates those methods, and so real code will be using + the dictionary. + + The introduction in 2.4 of those methods was a mistake for the same reason: + code that previously saw all (urllib2 user)-provided headers in .headers + now sees only a subset (and the function interface is ugly and incomplete). + A better change would have been to replace .headers dict with a dict + subclass (or UserDict.DictMixin instance?) that preserved the .headers + interface and also provided access to the "unredirected" headers. It's + probably too late to fix that, though. + + + Check .capitalize() case normalization: + + >>> url = "http://example.com" + >>> Request(url, headers={"Spam-eggs": "blah"}).headers["Spam-eggs"] + 'blah' + >>> Request(url, headers={"spam-EggS": "blah"}).headers["Spam-eggs"] + 'blah' + + Currently, Request(url, "Spam-eggs").headers["Spam-Eggs"] raises KeyError, + but that could be changed in future. + + """ + +def test_request_headers_methods(): + """ + Note the case normalization of header names here, to .capitalize()-case. + This should be preserved for backwards-compatibility. (In the HTTP case, + normalization to .title()-case is done by urllib2 before sending headers to + httplib). + + >>> url = "http://example.com" + >>> r = Request(url, headers={"Spam-eggs": "blah"}) + >>> r.has_header("Spam-eggs") + True + >>> r.header_items() + [('Spam-eggs', 'blah')] + >>> r.add_header("Foo-Bar", "baz") + >>> items = r.header_items() + >>> items.sort() + >>> items + [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')] + + Note that e.g. r.has_header("spam-EggS") is currently False, and + r.get_header("spam-EggS") returns None, but that could be changed in + future. + + >>> r.has_header("Not-there") + False + >>> print r.get_header("Not-there") + None + >>> r.get_header("Not-there", "default") + 'default' + + """ + + +def test_password_manager(self): + """ + >>> mgr = urllib2.HTTPPasswordMgr() + >>> add = mgr.add_password + >>> add("Some Realm", "http://example.com/", "joe", "password") + >>> add("Some Realm", "http://example.com/ni", "ni", "ni") + >>> add("c", "http://example.com/foo", "foo", "ni") + >>> add("c", "http://example.com/bar", "bar", "nini") + >>> add("b", "http://example.com/", "first", "blah") + >>> add("b", "http://example.com/", "second", "spam") + >>> add("a", "http://example.com", "1", "a") + >>> add("Some Realm", "http://c.example.com:3128", "3", "c") + >>> add("Some Realm", "d.example.com", "4", "d") + >>> add("Some Realm", "e.example.com:3128", "5", "e") + + >>> mgr.find_user_password("Some Realm", "example.com") + ('joe', 'password') + >>> mgr.find_user_password("Some Realm", "http://example.com") + ('joe', 'password') + >>> mgr.find_user_password("Some Realm", "http://example.com/") + ('joe', 'password') + >>> mgr.find_user_password("Some Realm", "http://example.com/spam") + ('joe', 'password') + >>> mgr.find_user_password("Some Realm", "http://example.com/spam/spam") + ('joe', 'password') + >>> mgr.find_user_password("c", "http://example.com/foo") + ('foo', 'ni') + >>> mgr.find_user_password("c", "http://example.com/bar") + ('bar', 'nini') + + Actually, this is really undefined ATM +## Currently, we use the highest-level path where more than one match: + +## >>> mgr.find_user_password("Some Realm", "http://example.com/ni") +## ('joe', 'password') + + Use latest add_password() in case of conflict: + + >>> mgr.find_user_password("b", "http://example.com/") + ('second', 'spam') + + No special relationship between a.example.com and example.com: + + >>> mgr.find_user_password("a", "http://example.com/") + ('1', 'a') + >>> mgr.find_user_password("a", "http://a.example.com/") + (None, None) + + Ports: + + >>> mgr.find_user_password("Some Realm", "c.example.com") + (None, None) + >>> mgr.find_user_password("Some Realm", "c.example.com:3128") + ('3', 'c') + >>> mgr.find_user_password("Some Realm", "http://c.example.com:3128") + ('3', 'c') + >>> mgr.find_user_password("Some Realm", "d.example.com") + ('4', 'd') + >>> mgr.find_user_password("Some Realm", "e.example.com:3128") + ('5', 'e') + + """ + pass + + +def test_password_manager_default_port(self): + """ + >>> mgr = urllib2.HTTPPasswordMgr() + >>> add = mgr.add_password + + The point to note here is that we can't guess the default port if there's + no scheme. This applies to both add_password and find_user_password. + + >>> add("f", "http://g.example.com:80", "10", "j") + >>> add("g", "http://h.example.com", "11", "k") + >>> add("h", "i.example.com:80", "12", "l") + >>> add("i", "j.example.com", "13", "m") + >>> mgr.find_user_password("f", "g.example.com:100") + (None, None) + >>> mgr.find_user_password("f", "g.example.com:80") + ('10', 'j') + >>> mgr.find_user_password("f", "g.example.com") + (None, None) + >>> mgr.find_user_password("f", "http://g.example.com:100") + (None, None) + >>> mgr.find_user_password("f", "http://g.example.com:80") + ('10', 'j') + >>> mgr.find_user_password("f", "http://g.example.com") + ('10', 'j') + >>> mgr.find_user_password("g", "h.example.com") + ('11', 'k') + >>> mgr.find_user_password("g", "h.example.com:80") + ('11', 'k') + >>> mgr.find_user_password("g", "http://h.example.com:80") + ('11', 'k') + >>> mgr.find_user_password("h", "i.example.com") + (None, None) + >>> mgr.find_user_password("h", "i.example.com:80") + ('12', 'l') + >>> mgr.find_user_password("h", "http://i.example.com:80") + ('12', 'l') + >>> mgr.find_user_password("i", "j.example.com") + ('13', 'm') + >>> mgr.find_user_password("i", "j.example.com:80") + (None, None) + >>> mgr.find_user_password("i", "http://j.example.com") + ('13', 'm') + >>> mgr.find_user_password("i", "http://j.example.com:80") + (None, None) + + """ + +class MockOpener: + addheaders = [] + def open(self, req, data=None,timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.req, self.data, self.timeout = req, data, timeout + def error(self, proto, *args): + self.proto, self.args = proto, args + +class MockFile: + def read(self, count=None): pass + def readline(self, count=None): pass + def close(self): pass + +class MockHeaders(dict): + def getheaders(self, name): + return self.values() + +class MockResponse(StringIO.StringIO): + def __init__(self, code, msg, headers, data, url=None): + StringIO.StringIO.__init__(self, data) + self.code, self.msg, self.headers, self.url = code, msg, headers, url + def info(self): + return self.headers + def geturl(self): + return self.url + +class MockCookieJar: + def add_cookie_header(self, request): + self.ach_req = request + def extract_cookies(self, response, request): + self.ec_req, self.ec_r = request, response + +class FakeMethod: + def __init__(self, meth_name, action, handle): + self.meth_name = meth_name + self.handle = handle + self.action = action + def __call__(self, *args): + return self.handle(self.meth_name, self.action, *args) + +class MockHTTPResponse: + def __init__(self, fp, msg, status, reason): + self.fp = fp + self.msg = msg + self.status = status + self.reason = reason + def read(self): + return '' + +class MockHTTPClass: + def __init__(self): + self.req_headers = [] + self.data = None + self.raise_on_endheaders = False + self._tunnel_headers = {} + + def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.host = host + self.timeout = timeout + return self + + def set_debuglevel(self, level): + self.level = level + + def set_tunnel(self, host, port=None, headers=None): + self._tunnel_host = host + self._tunnel_port = port + if headers: + self._tunnel_headers = headers + else: + self._tunnel_headers.clear() + + def request(self, method, url, body=None, headers=None): + self.method = method + self.selector = url + if headers is not None: + self.req_headers += headers.items() + self.req_headers.sort() + if body: + self.data = body + if self.raise_on_endheaders: + import socket + raise socket.error() + + def getresponse(self): + return MockHTTPResponse(MockFile(), {}, 200, "OK") + + def close(self): + pass + +class MockHandler: + # useful for testing handler machinery + # see add_ordered_mock_handlers() docstring + handler_order = 500 + def __init__(self, methods): + self._define_methods(methods) + def _define_methods(self, methods): + for spec in methods: + if len(spec) == 2: name, action = spec + else: name, action = spec, None + meth = FakeMethod(name, action, self.handle) + setattr(self.__class__, name, meth) + def handle(self, fn_name, action, *args, **kwds): + self.parent.calls.append((self, fn_name, args, kwds)) + if action is None: + return None + elif action == "return self": + return self + elif action == "return response": + res = MockResponse(200, "OK", {}, "") + return res + elif action == "return request": + return Request("http://blah/") + elif action.startswith("error"): + code = action[action.rfind(" ")+1:] + try: + code = int(code) + except ValueError: + pass + res = MockResponse(200, "OK", {}, "") + return self.parent.error("http", args[0], res, code, "", {}) + elif action == "raise": + raise urllib2.URLError("blah") + assert False + def close(self): pass + def add_parent(self, parent): + self.parent = parent + self.parent.calls = [] + def __lt__(self, other): + if not hasattr(other, "handler_order"): + # No handler_order, leave in original order. Yuck. + return True + return self.handler_order < other.handler_order + +def add_ordered_mock_handlers(opener, meth_spec): + """Create MockHandlers and add them to an OpenerDirector. + + meth_spec: list of lists of tuples and strings defining methods to define + on handlers. eg: + + [["http_error", "ftp_open"], ["http_open"]] + + defines methods .http_error() and .ftp_open() on one handler, and + .http_open() on another. These methods just record their arguments and + return None. Using a tuple instead of a string causes the method to + perform some action (see MockHandler.handle()), eg: + + [["http_error"], [("http_open", "return request")]] + + defines .http_error() on one handler (which simply returns None), and + .http_open() on another handler, which returns a Request object. + + """ + handlers = [] + count = 0 + for meths in meth_spec: + class MockHandlerSubclass(MockHandler): pass + h = MockHandlerSubclass(meths) + h.handler_order += count + h.add_parent(opener) + count = count + 1 + handlers.append(h) + opener.add_handler(h) + return handlers + +def build_test_opener(*handler_instances): + opener = OpenerDirector() + for h in handler_instances: + opener.add_handler(h) + return opener + +class MockHTTPHandler(urllib2.BaseHandler): + # useful for testing redirections and auth + # sends supplied headers and code as first response + # sends 200 OK as second response + def __init__(self, code, headers): + self.code = code + self.headers = headers + self.reset() + def reset(self): + self._count = 0 + self.requests = [] + def http_open(self, req): + import mimetools, httplib, copy + from StringIO import StringIO + self.requests.append(copy.deepcopy(req)) + if self._count == 0: + self._count = self._count + 1 + name = httplib.responses[self.code] + msg = mimetools.Message(StringIO(self.headers)) + return self.parent.error( + "http", req, MockFile(), self.code, name, msg) + else: + self.req = req + msg = mimetools.Message(StringIO("\r\n\r\n")) + return MockResponse(200, "OK", msg, "", req.get_full_url()) + +class MockHTTPSHandler(urllib2.AbstractHTTPHandler): + # Useful for testing the Proxy-Authorization request by verifying the + # properties of httpcon + + def __init__(self): + urllib2.AbstractHTTPHandler.__init__(self) + self.httpconn = MockHTTPClass() + + def https_open(self, req): + return self.do_open(self.httpconn, req) + +class MockPasswordManager: + def add_password(self, realm, uri, user, password): + self.realm = realm + self.url = uri + self.user = user + self.password = password + def find_user_password(self, realm, authuri): + self.target_realm = realm + self.target_url = authuri + return self.user, self.password + + +class OpenerDirectorTests(unittest.TestCase): + + def test_add_non_handler(self): + class NonHandler(object): + pass + self.assertRaises(TypeError, + OpenerDirector().add_handler, NonHandler()) + + def test_badly_named_methods(self): + # test work-around for three methods that accidentally follow the + # naming conventions for handler methods + # (*_open() / *_request() / *_response()) + + # These used to call the accidentally-named methods, causing a + # TypeError in real code; here, returning self from these mock + # methods would either cause no exception, or AttributeError. + + from urllib2 import URLError + + o = OpenerDirector() + meth_spec = [ + [("do_open", "return self"), ("proxy_open", "return self")], + [("redirect_request", "return self")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + o.add_handler(urllib2.UnknownHandler()) + for scheme in "do", "proxy", "redirect": + self.assertRaises(URLError, o.open, scheme+"://example.com/") + + def test_handled(self): + # handler returning non-None means no more handlers will be called + o = OpenerDirector() + meth_spec = [ + ["http_open", "ftp_open", "http_error_302"], + ["ftp_open"], + [("http_open", "return self")], + [("http_open", "return self")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + r = o.open(req) + # Second .http_open() gets called, third doesn't, since second returned + # non-None. Handlers without .http_open() never get any methods called + # on them. + # In fact, second mock handler defining .http_open() returns self + # (instead of response), which becomes the OpenerDirector's return + # value. + self.assertEqual(r, handlers[2]) + calls = [(handlers[0], "http_open"), (handlers[2], "http_open")] + for expected, got in zip(calls, o.calls): + handler, name, args, kwds = got + self.assertEqual((handler, name), expected) + self.assertEqual(args, (req,)) + + def test_handler_order(self): + o = OpenerDirector() + handlers = [] + for meths, handler_order in [ + ([("http_open", "return self")], 500), + (["http_open"], 0), + ]: + class MockHandlerSubclass(MockHandler): pass + h = MockHandlerSubclass(meths) + h.handler_order = handler_order + handlers.append(h) + o.add_handler(h) + + r = o.open("http://example.com/") + # handlers called in reverse order, thanks to their sort order + self.assertEqual(o.calls[0][0], handlers[1]) + self.assertEqual(o.calls[1][0], handlers[0]) + + def test_raise(self): + # raising URLError stops processing of request + o = OpenerDirector() + meth_spec = [ + [("http_open", "raise")], + [("http_open", "return self")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + self.assertRaises(urllib2.URLError, o.open, req) + self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) + +## def test_error(self): +## # XXX this doesn't actually seem to be used in standard library, +## # but should really be tested anyway... + + def test_http_error(self): + # XXX http_error_default + # http errors are a special case + o = OpenerDirector() + meth_spec = [ + [("http_open", "error 302")], + [("http_error_400", "raise"), "http_open"], + [("http_error_302", "return response"), "http_error_303", + "http_error"], + [("http_error_302")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + class Unknown: + def __eq__(self, other): return True + + req = Request("http://example.com/") + r = o.open(req) + assert len(o.calls) == 2 + calls = [(handlers[0], "http_open", (req,)), + (handlers[2], "http_error_302", + (req, Unknown(), 302, "", {}))] + for expected, got in zip(calls, o.calls): + handler, method_name, args = expected + self.assertEqual((handler, method_name), got[:2]) + self.assertEqual(args, got[2]) + + def test_processors(self): + # *_request / *_response methods get called appropriately + o = OpenerDirector() + meth_spec = [ + [("http_request", "return request"), + ("http_response", "return response")], + [("http_request", "return request"), + ("http_response", "return response")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + r = o.open(req) + # processor methods are called on *all* handlers that define them, + # not just the first handler that handles the request + calls = [ + (handlers[0], "http_request"), (handlers[1], "http_request"), + (handlers[0], "http_response"), (handlers[1], "http_response")] + + for i, (handler, name, args, kwds) in enumerate(o.calls): + if i < 2: + # *_request + self.assertEqual((handler, name), calls[i]) + self.assertEqual(len(args), 1) + self.assertIsInstance(args[0], Request) + else: + # *_response + self.assertEqual((handler, name), calls[i]) + self.assertEqual(len(args), 2) + self.assertIsInstance(args[0], Request) + # response from opener.open is None, because there's no + # handler that defines http_open to handle it + if args[1] is not None: + self.assertIsInstance(args[1], MockResponse) + + +def sanepathname2url(path): + import urllib + urlpath = urllib.pathname2url(path) + if os.name == "nt" and urlpath.startswith("///"): + urlpath = urlpath[2:] + # XXX don't ask me about the mac... + return urlpath + +class HandlerTests(unittest.TestCase): + + def test_ftp(self): + class MockFTPWrapper: + def __init__(self, data): self.data = data + def retrfile(self, filename, filetype): + self.filename, self.filetype = filename, filetype + return StringIO.StringIO(self.data), len(self.data) + def close(self): pass + + class NullFTPHandler(urllib2.FTPHandler): + def __init__(self, data): self.data = data + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.user, self.passwd = user, passwd + self.host, self.port = host, port + self.dirs = dirs + self.ftpwrapper = MockFTPWrapper(self.data) + return self.ftpwrapper + + import ftplib + data = "rheum rhaponicum" + h = NullFTPHandler(data) + o = h.parent = MockOpener() + + for url, host, port, user, passwd, type_, dirs, filename, mimetype in [ + ("ftp://localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://%25parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "%parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://%2542parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "%42parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://localhost:80/foo/bar/", + "localhost", 80, "", "", "D", + ["foo", "bar"], "", None), + ("ftp://localhost/baz.gif;type=a", + "localhost", ftplib.FTP_PORT, "", "", "A", + [], "baz.gif", None), # XXX really this should guess image/gif + ]: + req = Request(url) + req.timeout = None + r = h.ftp_open(req) + # ftp authentication not yet implemented by FTPHandler + self.assertEqual(h.user, user) + self.assertEqual(h.passwd, passwd) + self.assertEqual(h.host, socket.gethostbyname(host)) + self.assertEqual(h.port, port) + self.assertEqual(h.dirs, dirs) + self.assertEqual(h.ftpwrapper.filename, filename) + self.assertEqual(h.ftpwrapper.filetype, type_) + headers = r.info() + self.assertEqual(headers.get("Content-type"), mimetype) + self.assertEqual(int(headers["Content-length"]), len(data)) + + def test_file(self): + import rfc822, socket + h = urllib2.FileHandler() + o = h.parent = MockOpener() + + TESTFN = test_support.TESTFN + urlpath = sanepathname2url(os.path.abspath(TESTFN)) + towrite = "hello, world\n" + urls = [ + "file://localhost%s" % urlpath, + "file://%s" % urlpath, + "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), + ] + try: + localaddr = socket.gethostbyname(socket.gethostname()) + except socket.gaierror: + localaddr = '' + if localaddr: + urls.append("file://%s%s" % (localaddr, urlpath)) + + for url in urls: + f = open(TESTFN, "wb") + try: + try: + f.write(towrite) + finally: + f.close() + + r = h.file_open(Request(url)) + try: + data = r.read() + headers = r.info() + respurl = r.geturl() + finally: + r.close() + stats = os.stat(TESTFN) + modified = rfc822.formatdate(stats.st_mtime) + finally: + os.remove(TESTFN) + self.assertEqual(data, towrite) + self.assertEqual(headers["Content-type"], "text/plain") + self.assertEqual(headers["Content-length"], "13") + self.assertEqual(headers["Last-modified"], modified) + self.assertEqual(respurl, url) + + for url in [ + "file://localhost:80%s" % urlpath, + "file:///file_does_not_exist.txt", + "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), + os.getcwd(), TESTFN), + "file://somerandomhost.ontheinternet.com%s/%s" % + (os.getcwd(), TESTFN), + ]: + try: + f = open(TESTFN, "wb") + try: + f.write(towrite) + finally: + f.close() + + self.assertRaises(urllib2.URLError, + h.file_open, Request(url)) + finally: + os.remove(TESTFN) + + h = urllib2.FileHandler() + o = h.parent = MockOpener() + # XXXX why does // mean ftp (and /// mean not ftp!), and where + # is file: scheme specified? I think this is really a bug, and + # what was intended was to distinguish between URLs like: + # file:/blah.txt (a file) + # file://localhost/blah.txt (a file) + # file:///blah.txt (a file) + # file://ftp.example.com/blah.txt (an ftp URL) + for url, ftp in [ + ("file://ftp.example.com//foo.txt", True), + ("file://ftp.example.com///foo.txt", False), +# XXXX bug: fails with OSError, should be URLError + ("file://ftp.example.com/foo.txt", False), + ("file://somehost//foo/something.txt", True), + ("file://localhost//foo/something.txt", False), + ]: + req = Request(url) + try: + h.file_open(req) + # XXXX remove OSError when bug fixed + except (urllib2.URLError, OSError): + self.assertTrue(not ftp) + else: + self.assertTrue(o.req is req) + self.assertEqual(req.type, "ftp") + self.assertEqual(req.type == "ftp", ftp) + + def test_http(self): + + h = urllib2.AbstractHTTPHandler() + o = h.parent = MockOpener() + + url = "http://example.com/" + for method, data in [("GET", None), ("POST", "blah")]: + req = Request(url, data, {"Foo": "bar"}) + req.timeout = None + req.add_unredirected_header("Spam", "eggs") + http = MockHTTPClass() + r = h.do_open(http, req) + + # result attributes + r.read; r.readline # wrapped MockFile methods + r.info; r.geturl # addinfourl methods + r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply() + hdrs = r.info() + hdrs.get; hdrs.has_key # r.info() gives dict from .getreply() + self.assertEqual(r.geturl(), url) + + self.assertEqual(http.host, "example.com") + self.assertEqual(http.level, 0) + self.assertEqual(http.method, method) + self.assertEqual(http.selector, "/") + self.assertEqual(http.req_headers, + [("Connection", "close"), + ("Foo", "bar"), ("Spam", "eggs")]) + self.assertEqual(http.data, data) + + # check socket.error converted to URLError + http.raise_on_endheaders = True + self.assertRaises(urllib2.URLError, h.do_open, http, req) + + # check adding of standard headers + o.addheaders = [("Spam", "eggs")] + for data in "", None: # POST, GET + req = Request("http://example.com/", data) + r = MockResponse(200, "OK", {}, "") + newreq = h.do_request_(req) + if data is None: # GET + self.assertNotIn("Content-length", req.unredirected_hdrs) + self.assertNotIn("Content-type", req.unredirected_hdrs) + else: # POST + self.assertEqual(req.unredirected_hdrs["Content-length"], "0") + self.assertEqual(req.unredirected_hdrs["Content-type"], + "application/x-www-form-urlencoded") + # XXX the details of Host could be better tested + self.assertEqual(req.unredirected_hdrs["Host"], "example.com") + self.assertEqual(req.unredirected_hdrs["Spam"], "eggs") + + # don't clobber existing headers + req.add_unredirected_header("Content-length", "foo") + req.add_unredirected_header("Content-type", "bar") + req.add_unredirected_header("Host", "baz") + req.add_unredirected_header("Spam", "foo") + newreq = h.do_request_(req) + self.assertEqual(req.unredirected_hdrs["Content-length"], "foo") + self.assertEqual(req.unredirected_hdrs["Content-type"], "bar") + self.assertEqual(req.unredirected_hdrs["Host"], "baz") + self.assertEqual(req.unredirected_hdrs["Spam"], "foo") + + def test_http_doubleslash(self): + # Checks that the presence of an unnecessary double slash in a url doesn't break anything + # Previously, a double slash directly after the host could cause incorrect parsing of the url + h = urllib2.AbstractHTTPHandler() + o = h.parent = MockOpener() + + data = "" + ds_urls = [ + "http://example.com/foo/bar/baz.html", + "http://example.com//foo/bar/baz.html", + "http://example.com/foo//bar/baz.html", + "http://example.com/foo/bar//baz.html", + ] + + for ds_url in ds_urls: + ds_req = Request(ds_url, data) + + # Check whether host is determined correctly if there is no proxy + np_ds_req = h.do_request_(ds_req) + self.assertEqual(np_ds_req.unredirected_hdrs["Host"],"example.com") + + # Check whether host is determined correctly if there is a proxy + ds_req.set_proxy("someproxy:3128",None) + p_ds_req = h.do_request_(ds_req) + self.assertEqual(p_ds_req.unredirected_hdrs["Host"],"example.com") + + def test_fixpath_in_weirdurls(self): + # Issue4493: urllib2 to supply '/' when to urls where path does not + # start with'/' + + h = urllib2.AbstractHTTPHandler() + o = h.parent = MockOpener() + + weird_url = 'http://www.python.org?getspam' + req = Request(weird_url) + newreq = h.do_request_(req) + self.assertEqual(newreq.get_host(),'www.python.org') + self.assertEqual(newreq.get_selector(),'/?getspam') + + url_without_path = 'http://www.python.org' + req = Request(url_without_path) + newreq = h.do_request_(req) + self.assertEqual(newreq.get_host(),'www.python.org') + self.assertEqual(newreq.get_selector(),'') + + def test_errors(self): + h = urllib2.HTTPErrorProcessor() + o = h.parent = MockOpener() + + url = "http://example.com/" + req = Request(url) + # all 2xx are passed through + r = MockResponse(200, "OK", {}, "", url) + newr = h.http_response(req, r) + self.assertTrue(r is newr) + self.assertTrue(not hasattr(o, "proto")) # o.error not called + r = MockResponse(202, "Accepted", {}, "", url) + newr = h.http_response(req, r) + self.assertTrue(r is newr) + self.assertTrue(not hasattr(o, "proto")) # o.error not called + r = MockResponse(206, "Partial content", {}, "", url) + newr = h.http_response(req, r) + self.assertTrue(r is newr) + self.assertTrue(not hasattr(o, "proto")) # o.error not called + # anything else calls o.error (and MockOpener returns None, here) + r = MockResponse(502, "Bad gateway", {}, "", url) + self.assertTrue(h.http_response(req, r) is None) + self.assertEqual(o.proto, "http") # o.error called + self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) + + def test_cookies(self): + cj = MockCookieJar() + h = urllib2.HTTPCookieProcessor(cj) + o = h.parent = MockOpener() + + req = Request("http://example.com/") + r = MockResponse(200, "OK", {}, "") + newreq = h.http_request(req) + self.assertTrue(cj.ach_req is req is newreq) + self.assertEqual(req.get_origin_req_host(), "example.com") + self.assertTrue(not req.is_unverifiable()) + newr = h.http_response(req, r) + self.assertTrue(cj.ec_req is req) + self.assertTrue(cj.ec_r is r is newr) + + def test_redirect(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib2.HTTPRedirectHandler() + o = h.parent = MockOpener() + + # ordinary redirect behaviour + for code in 301, 302, 303, 307: + for data in None, "blah\nblah\n": + method = getattr(h, "http_error_%s" % code) + req = Request(from_url, data) + req.add_header("Nonsense", "viking=withhold") + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + if data is not None: + req.add_header("Content-Length", str(len(data))) + req.add_unredirected_header("Spam", "spam") + try: + method(req, MockFile(), code, "Blah", + MockHeaders({"location": to_url})) + except urllib2.HTTPError: + # 307 in response to POST requires user OK + self.assertEqual(code, 307) + self.assertIsNotNone(data) + self.assertEqual(o.req.get_full_url(), to_url) + try: + self.assertEqual(o.req.get_method(), "GET") + except AttributeError: + self.assertTrue(not o.req.has_data()) + + # now it's a GET, there should not be headers regarding content + # (possibly dragged from before being a POST) + headers = [x.lower() for x in o.req.headers] + self.assertNotIn("content-length", headers) + self.assertNotIn("content-type", headers) + + self.assertEqual(o.req.headers["Nonsense"], + "viking=withhold") + self.assertNotIn("Spam", o.req.headers) + self.assertNotIn("Spam", o.req.unredirected_hdrs) + + # loop detection + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + def redirect(h, req, url=to_url): + h.http_error_302(req, MockFile(), 302, "Blah", + MockHeaders({"location": url})) + # Note that the *original* request shares the same record of + # redirections with the sub-requests caused by the redirections. + + # detect infinite loop redirect of a URL to itself + req = Request(from_url, origin_req_host="example.com") + count = 0 + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + try: + while 1: + redirect(h, req, "http://example.com/") + count = count + 1 + except urllib2.HTTPError: + # don't stop until max_repeats, because cookies may introduce state + self.assertEqual(count, urllib2.HTTPRedirectHandler.max_repeats) + + # detect endless non-repeating chain of redirects + req = Request(from_url, origin_req_host="example.com") + count = 0 + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + try: + while 1: + redirect(h, req, "http://example.com/%d" % count) + count = count + 1 + except urllib2.HTTPError: + self.assertEqual(count, + urllib2.HTTPRedirectHandler.max_redirections) + + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http', 'https', 'ftp'] + invalid_schemes = ['file', 'imap', 'ldap'] + schemeless_url = "example.com/b.html" + h = urllib2.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib2.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + + def test_cookie_redirect(self): + # cookies shouldn't leak into redirected requests + from cookielib import CookieJar + + from test.test_cookielib import interact_netscape + + cj = CookieJar() + interact_netscape(cj, "http://www.example.com/", "spam=eggs") + hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n") + hdeh = urllib2.HTTPDefaultErrorHandler() + hrh = urllib2.HTTPRedirectHandler() + cp = urllib2.HTTPCookieProcessor(cj) + o = build_test_opener(hh, hdeh, hrh, cp) + o.open("http://www.example.com/") + self.assertTrue(not hh.req.has_header("Cookie")) + + def test_redirect_fragment(self): + redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' + hh = MockHTTPHandler(302, 'Location: ' + redirected_url) + hdeh = urllib2.HTTPDefaultErrorHandler() + hrh = urllib2.HTTPRedirectHandler() + o = build_test_opener(hh, hdeh, hrh) + fp = o.open('http://www.example.com') + self.assertEqual(fp.geturl(), redirected_url.strip()) + + def test_proxy(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(http="proxy.example.com:3128")) + o.add_handler(ph) + meth_spec = [ + [("http_open", "return response")] + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://acme.example.com/") + self.assertEqual(req.get_host(), "acme.example.com") + r = o.open(req) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + + self.assertEqual([(handlers[0], "http_open")], + [tup[0:2] for tup in o.calls]) + + def test_proxy_no_proxy(self): + os.environ['no_proxy'] = 'python.org' + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(http="proxy.example.com")) + o.add_handler(ph) + req = Request("http://www.perl.org/") + self.assertEqual(req.get_host(), "www.perl.org") + r = o.open(req) + self.assertEqual(req.get_host(), "proxy.example.com") + req = Request("http://www.python.org") + self.assertEqual(req.get_host(), "www.python.org") + r = o.open(req) + self.assertEqual(req.get_host(), "www.python.org") + del os.environ['no_proxy'] + + + def test_proxy_https(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(https='proxy.example.com:3128')) + o.add_handler(ph) + meth_spec = [ + [("https_open","return response")] + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + req = Request("https://www.example.com/") + self.assertEqual(req.get_host(), "www.example.com") + r = o.open(req) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + self.assertEqual([(handlers[0], "https_open")], + [tup[0:2] for tup in o.calls]) + + def test_proxy_https_proxy_authorization(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(https='proxy.example.com:3128')) + o.add_handler(ph) + https_handler = MockHTTPSHandler() + o.add_handler(https_handler) + req = Request("https://www.example.com/") + req.add_header("Proxy-Authorization","FooBar") + req.add_header("User-Agent","Grail") + self.assertEqual(req.get_host(), "www.example.com") + self.assertIsNone(req._tunnel_host) + r = o.open(req) + # Verify Proxy-Authorization gets tunneled to request. + # httpsconn req_headers do not have the Proxy-Authorization header but + # the req will have. + self.assertNotIn(("Proxy-Authorization","FooBar"), + https_handler.httpconn.req_headers) + self.assertIn(("User-Agent","Grail"), + https_handler.httpconn.req_headers) + self.assertIsNotNone(req._tunnel_host) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + self.assertEqual(req.get_header("Proxy-authorization"),"FooBar") + + def test_basic_auth(self, quote_char='"'): + opener = OpenerDirector() + password_manager = MockPasswordManager() + auth_handler = urllib2.HTTPBasicAuthHandler(password_manager) + realm = "ACME Widget Store" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' % + (quote_char, realm, quote_char) ) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + self._test_basic_auth(opener, auth_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected" + ) + + def test_basic_auth_with_single_quoted_realm(self): + self.test_basic_auth(quote_char="'") + + def test_basic_auth_with_unquoted_realm(self): + opener = OpenerDirector() + password_manager = MockPasswordManager() + auth_handler = urllib2.HTTPBasicAuthHandler(password_manager) + realm = "ACME Widget Store" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + msg = "Basic Auth Realm was unquoted" + with test_support.check_warnings((msg, UserWarning)): + self._test_basic_auth(opener, auth_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected" + ) + + + def test_proxy_basic_auth(self): + opener = OpenerDirector() + ph = urllib2.ProxyHandler(dict(http="proxy.example.com:3128")) + opener.add_handler(ph) + password_manager = MockPasswordManager() + auth_handler = urllib2.ProxyBasicAuthHandler(password_manager) + realm = "ACME Networks" + http_handler = MockHTTPHandler( + 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + self._test_basic_auth(opener, auth_handler, "Proxy-authorization", + realm, http_handler, password_manager, + "http://acme.example.com:3128/protected", + "proxy.example.com:3128", + ) + + def test_basic_and_digest_auth_handlers(self): + # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* + # response (http://python.org/sf/1479302), where it should instead + # return None to allow another handler (especially + # HTTPBasicAuthHandler) to handle the response. + + # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must + # try digest first (since it's the strongest auth scheme), so we record + # order of calls here to check digest comes first: + class RecordingOpenerDirector(OpenerDirector): + def __init__(self): + OpenerDirector.__init__(self) + self.recorded = [] + def record(self, info): + self.recorded.append(info) + class TestDigestAuthHandler(urllib2.HTTPDigestAuthHandler): + def http_error_401(self, *args, **kwds): + self.parent.record("digest") + urllib2.HTTPDigestAuthHandler.http_error_401(self, + *args, **kwds) + class TestBasicAuthHandler(urllib2.HTTPBasicAuthHandler): + def http_error_401(self, *args, **kwds): + self.parent.record("basic") + urllib2.HTTPBasicAuthHandler.http_error_401(self, + *args, **kwds) + + opener = RecordingOpenerDirector() + password_manager = MockPasswordManager() + digest_handler = TestDigestAuthHandler(password_manager) + basic_handler = TestBasicAuthHandler(password_manager) + realm = "ACME Networks" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) + opener.add_handler(basic_handler) + opener.add_handler(digest_handler) + opener.add_handler(http_handler) + + # check basic auth isn't blocked by digest handler failing + self._test_basic_auth(opener, basic_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected", + ) + # check digest was tried before basic (twice, because + # _test_basic_auth called .open() twice) + self.assertEqual(opener.recorded, ["digest", "basic"]*2) + + def _test_basic_auth(self, opener, auth_handler, auth_header, + realm, http_handler, password_manager, + request_url, protected_url): + import base64 + user, password = "wile", "coyote" + + # .add_password() fed through to password manager + auth_handler.add_password(realm, request_url, user, password) + self.assertEqual(realm, password_manager.realm) + self.assertEqual(request_url, password_manager.url) + self.assertEqual(user, password_manager.user) + self.assertEqual(password, password_manager.password) + + r = opener.open(request_url) + + # should have asked the password manager for the username/password + self.assertEqual(password_manager.target_realm, realm) + self.assertEqual(password_manager.target_url, protected_url) + + # expect one request without authorization, then one with + self.assertEqual(len(http_handler.requests), 2) + self.assertFalse(http_handler.requests[0].has_header(auth_header)) + userpass = '%s:%s' % (user, password) + auth_hdr_value = 'Basic '+base64.encodestring(userpass).strip() + self.assertEqual(http_handler.requests[1].get_header(auth_header), + auth_hdr_value) + self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header], + auth_hdr_value) + # if the password manager can't find a password, the handler won't + # handle the HTTP auth error + password_manager.user = password_manager.password = None + http_handler.reset() + r = opener.open(request_url) + self.assertEqual(len(http_handler.requests), 1) + self.assertFalse(http_handler.requests[0].has_header(auth_header)) + +class MiscTests(unittest.TestCase): + + def test_build_opener(self): + class MyHTTPHandler(urllib2.HTTPHandler): pass + class FooHandler(urllib2.BaseHandler): + def foo_open(self): pass + class BarHandler(urllib2.BaseHandler): + def bar_open(self): pass + + build_opener = urllib2.build_opener + + o = build_opener(FooHandler, BarHandler) + self.opener_has_handler(o, FooHandler) + self.opener_has_handler(o, BarHandler) + + # can take a mix of classes and instances + o = build_opener(FooHandler, BarHandler()) + self.opener_has_handler(o, FooHandler) + self.opener_has_handler(o, BarHandler) + + # subclasses of default handlers override default handlers + o = build_opener(MyHTTPHandler) + self.opener_has_handler(o, MyHTTPHandler) + + # a particular case of overriding: default handlers can be passed + # in explicitly + o = build_opener() + self.opener_has_handler(o, urllib2.HTTPHandler) + o = build_opener(urllib2.HTTPHandler) + self.opener_has_handler(o, urllib2.HTTPHandler) + o = build_opener(urllib2.HTTPHandler()) + self.opener_has_handler(o, urllib2.HTTPHandler) + + # Issue2670: multiple handlers sharing the same base class + class MyOtherHTTPHandler(urllib2.HTTPHandler): pass + o = build_opener(MyHTTPHandler, MyOtherHTTPHandler) + self.opener_has_handler(o, MyHTTPHandler) + self.opener_has_handler(o, MyOtherHTTPHandler) + + def opener_has_handler(self, opener, handler_class): + for h in opener.handlers: + if h.__class__ == handler_class: + break + else: + self.assertTrue(False) + +class RequestTests(unittest.TestCase): + + def setUp(self): + self.get = urllib2.Request("http://www.python.org/~jeremy/") + self.post = urllib2.Request("http://www.python.org/~jeremy/", + "data", + headers={"X-Test": "test"}) + + def test_method(self): + self.assertEqual("POST", self.post.get_method()) + self.assertEqual("GET", self.get.get_method()) + + def test_add_data(self): + self.assertTrue(not self.get.has_data()) + self.assertEqual("GET", self.get.get_method()) + self.get.add_data("spam") + self.assertTrue(self.get.has_data()) + self.assertEqual("POST", self.get.get_method()) + + def test_get_full_url(self): + self.assertEqual("http://www.python.org/~jeremy/", + self.get.get_full_url()) + + def test_selector(self): + self.assertEqual("/~jeremy/", self.get.get_selector()) + req = urllib2.Request("http://www.python.org/") + self.assertEqual("/", req.get_selector()) + + def test_get_type(self): + self.assertEqual("http", self.get.get_type()) + + def test_get_host(self): + self.assertEqual("www.python.org", self.get.get_host()) + + def test_get_host_unquote(self): + req = urllib2.Request("http://www.%70ython.org/") + self.assertEqual("www.python.org", req.get_host()) + + def test_proxy(self): + self.assertTrue(not self.get.has_proxy()) + self.get.set_proxy("www.perl.org", "http") + self.assertTrue(self.get.has_proxy()) + self.assertEqual("www.python.org", self.get.get_origin_req_host()) + self.assertEqual("www.perl.org", self.get.get_host()) + + def test_wrapped_url(self): + req = Request("") + self.assertEqual("www.python.org", req.get_host()) + + def test_url_fragment(self): + req = Request("http://www.python.org/?qs=query#fragment=true") + self.assertEqual("/?qs=query", req.get_selector()) + req = Request("http://www.python.org/#fun=true") + self.assertEqual("/", req.get_selector()) + + # Issue 11703: geturl() omits fragment in the original URL. + url = 'http://docs.python.org/library/urllib2.html#OK' + req = Request(url) + self.assertEqual(req.get_full_url(), url) + + def test_HTTPError_interface(self): + """ + Issue 13211 reveals that HTTPError didn't implement the URLError + interface even though HTTPError is a subclass of URLError. + + >>> err = urllib2.HTTPError(msg='something bad happened', url=None, code=None, hdrs=None, fp=None) + >>> assert hasattr(err, 'reason') + >>> err.reason + 'something bad happened' + """ + + def test_HTTPError_interface_call(self): + """ + Issue 15701= - HTTPError interface has info method available from URLError. + """ + err = urllib2.HTTPError(msg='something bad happened', url=None, + code=None, hdrs='Content-Length:42', fp=None) + self.assertTrue(hasattr(err, 'reason')) + assert hasattr(err, 'reason') + assert hasattr(err, 'info') + assert callable(err.info) + try: + err.info() + except AttributeError: + self.fail("err.info() failed") + self.assertEqual(err.info(), "Content-Length:42") + +def test_main(verbose=None): + from test import test_urllib2 + test_support.run_doctest(test_urllib2, verbose) + test_support.run_doctest(urllib2, verbose) + tests = (TrivialTests, + OpenerDirectorTests, + HandlerTests, + MiscTests, + RequestTests) + test_support.run_unittest(*tests) + +if __name__ == "__main__": + test_main(verbose=True) diff --git a/src/greentest/2.7/test_urllib2_localnet.py b/src/greentest/2.7/test_urllib2_localnet.py new file mode 100644 index 0000000..bb82b26 --- /dev/null +++ b/src/greentest/2.7/test_urllib2_localnet.py @@ -0,0 +1,709 @@ +import os +import base64 +import urlparse +import urllib2 +import BaseHTTPServer +import unittest +import hashlib + +from test import test_support + +mimetools = test_support.import_module('mimetools', deprecated=True) +threading = test_support.import_module('threading') + +try: + import ssl +except ImportError: + ssl = None + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') + +# Loopback http server infrastructure + +class LoopbackHttpServer(BaseHTTPServer.HTTPServer): + """HTTP server w/ a few modifications that make it useful for + loopback testing purposes. + """ + + def __init__(self, server_address, RequestHandlerClass): + BaseHTTPServer.HTTPServer.__init__(self, + server_address, + RequestHandlerClass) + + # Set the timeout of our listening socket really low so + # that we can stop the server easily. + self.socket.settimeout(0.1) + + def get_request(self): + """BaseHTTPServer method, overridden.""" + + request, client_address = self.socket.accept() + + # It's a loopback connection, so setting the timeout + # really low shouldn't affect anything, but should make + # deadlocks less likely to occur. + request.settimeout(10.0) + + return (request, client_address) + +class LoopbackHttpServerThread(threading.Thread): + """Stoppable thread that runs a loopback http server.""" + + def __init__(self, request_handler): + threading.Thread.__init__(self) + self._stop = False + self.ready = threading.Event() + request_handler.protocol_version = "HTTP/1.0" + self.httpd = LoopbackHttpServer(('127.0.0.1', 0), + request_handler) + #print "Serving HTTP on %s port %s" % (self.httpd.server_name, + # self.httpd.server_port) + self.port = self.httpd.server_port + + def stop(self): + """Stops the webserver if it's currently running.""" + + # Set the stop flag. + self._stop = True + + self.join() + + def run(self): + self.ready.set() + while not self._stop: + self.httpd.handle_request() + +# Authentication infrastructure + + +class BasicAuthHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """Handler for performing Basic Authentication.""" + # Server side values + USER = "testUser" + PASSWD = "testPass" + REALM = "Test" + USER_PASSWD = "%s:%s" % (USER, PASSWD) + ENCODED_AUTH = base64.b64encode(USER_PASSWD) + + def __init__(self, *args, **kwargs): + BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def log_message(self, format, *args): + # Supress the HTTP Console log output + pass + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_AUTHHEAD(self): + self.send_response(401) + self.send_header("WWW-Authenticate", "Basic realm=\"%s\"" % self.REALM) + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_GET(self): + if self.headers.getheader("Authorization") == None: + self.do_AUTHHEAD() + self.wfile.write("No Auth Header Received") + elif self.headers.getheader( + "Authorization") == "Basic " + self.ENCODED_AUTH: + self.wfile.write("It works!") + else: + # Unauthorized Request + self.do_AUTHHEAD() + + +class DigestAuthHandler: + """Handler for performing digest authentication.""" + + def __init__(self): + self._request_num = 0 + self._nonces = [] + self._users = {} + self._realm_name = "Test Realm" + self._qop = "auth" + + def set_qop(self, qop): + self._qop = qop + + def set_users(self, users): + assert isinstance(users, dict) + self._users = users + + def set_realm(self, realm): + self._realm_name = realm + + def _generate_nonce(self): + self._request_num += 1 + nonce = hashlib.md5(str(self._request_num)).hexdigest() + self._nonces.append(nonce) + return nonce + + def _create_auth_dict(self, auth_str): + first_space_index = auth_str.find(" ") + auth_str = auth_str[first_space_index+1:] + + parts = auth_str.split(",") + + auth_dict = {} + for part in parts: + name, value = part.split("=") + name = name.strip() + if value[0] == '"' and value[-1] == '"': + value = value[1:-1] + else: + value = value.strip() + auth_dict[name] = value + return auth_dict + + def _validate_auth(self, auth_dict, password, method, uri): + final_dict = {} + final_dict.update(auth_dict) + final_dict["password"] = password + final_dict["method"] = method + final_dict["uri"] = uri + HA1_str = "%(username)s:%(realm)s:%(password)s" % final_dict + HA1 = hashlib.md5(HA1_str).hexdigest() + HA2_str = "%(method)s:%(uri)s" % final_dict + HA2 = hashlib.md5(HA2_str).hexdigest() + final_dict["HA1"] = HA1 + final_dict["HA2"] = HA2 + response_str = "%(HA1)s:%(nonce)s:%(nc)s:" \ + "%(cnonce)s:%(qop)s:%(HA2)s" % final_dict + response = hashlib.md5(response_str).hexdigest() + + return response == auth_dict["response"] + + def _return_auth_challenge(self, request_handler): + request_handler.send_response(407, "Proxy Authentication Required") + request_handler.send_header("Content-Type", "text/html") + request_handler.send_header( + 'Proxy-Authenticate', 'Digest realm="%s", ' + 'qop="%s",' + 'nonce="%s", ' % \ + (self._realm_name, self._qop, self._generate_nonce())) + # XXX: Not sure if we're supposed to add this next header or + # not. + #request_handler.send_header('Connection', 'close') + request_handler.end_headers() + request_handler.wfile.write("Proxy Authentication Required.") + return False + + def handle_request(self, request_handler): + """Performs digest authentication on the given HTTP request + handler. Returns True if authentication was successful, False + otherwise. + + If no users have been set, then digest auth is effectively + disabled and this method will always return True. + """ + + if len(self._users) == 0: + return True + + if 'Proxy-Authorization' not in request_handler.headers: + return self._return_auth_challenge(request_handler) + else: + auth_dict = self._create_auth_dict( + request_handler.headers['Proxy-Authorization'] + ) + if auth_dict["username"] in self._users: + password = self._users[ auth_dict["username"] ] + else: + return self._return_auth_challenge(request_handler) + if not auth_dict.get("nonce") in self._nonces: + return self._return_auth_challenge(request_handler) + else: + self._nonces.remove(auth_dict["nonce"]) + + auth_validated = False + + # MSIE uses short_path in its validation, but Python's + # urllib2 uses the full path, so we're going to see if + # either of them works here. + + for path in [request_handler.path, request_handler.short_path]: + if self._validate_auth(auth_dict, + password, + request_handler.command, + path): + auth_validated = True + + if not auth_validated: + return self._return_auth_challenge(request_handler) + return True + +# Proxy test infrastructure + +class FakeProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """This is a 'fake proxy' that makes it look like the entire + internet has gone down due to a sudden zombie invasion. It main + utility is in providing us with authentication support for + testing. + """ + + def __init__(self, digest_auth_handler, *args, **kwargs): + # This has to be set before calling our parent's __init__(), which will + # try to call do_GET(). + self.digest_auth_handler = digest_auth_handler + BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def log_message(self, format, *args): + # Uncomment the next line for debugging. + #sys.stderr.write(format % args) + pass + + def do_GET(self): + (scm, netloc, path, params, query, fragment) = urlparse.urlparse( + self.path, 'http') + self.short_path = path + if self.digest_auth_handler.handle_request(self): + self.send_response(200, "OK") + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write("You've reached %s!
" % self.path) + self.wfile.write("Our apologies, but our server is down due to " + "a sudden zombie invasion.") + +# Test cases + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test_support.threading_setup() + + def tearDown(self): + test_support.threading_cleanup(*self._threads) + + +class BasicAuthTests(BaseTestCase): + USER = "testUser" + PASSWD = "testPass" + INCORRECT_PASSWD = "Incorrect" + REALM = "Test" + + def setUp(self): + super(BasicAuthTests, self).setUp() + # With Basic Authentication + def http_server_with_basic_auth_handler(*args, **kwargs): + return BasicAuthHandler(*args, **kwargs) + self.server = LoopbackHttpServerThread(http_server_with_basic_auth_handler) + self.server_url = 'http://127.0.0.1:%s' % self.server.port + self.server.start() + self.server.ready.wait() + + def tearDown(self): + self.server.stop() + super(BasicAuthTests, self).tearDown() + + def test_basic_auth_success(self): + ah = urllib2.HTTPBasicAuthHandler() + ah.add_password(self.REALM, self.server_url, self.USER, self.PASSWD) + urllib2.install_opener(urllib2.build_opener(ah)) + try: + self.assertTrue(urllib2.urlopen(self.server_url)) + except urllib2.HTTPError: + self.fail("Basic Auth Failed for url: %s" % self.server_url) + except Exception as e: + raise e + + def test_basic_auth_httperror(self): + ah = urllib2.HTTPBasicAuthHandler() + ah.add_password(self.REALM, self.server_url, self.USER, + self.INCORRECT_PASSWD) + urllib2.install_opener(urllib2.build_opener(ah)) + self.assertRaises(urllib2.HTTPError, urllib2.urlopen, self.server_url) + + +class ProxyAuthTests(BaseTestCase): + URL = "http://localhost" + + USER = "tester" + PASSWD = "test123" + REALM = "TestRealm" + + def setUp(self): + super(ProxyAuthTests, self).setUp() + self.digest_auth_handler = DigestAuthHandler() + self.digest_auth_handler.set_users({self.USER: self.PASSWD}) + self.digest_auth_handler.set_realm(self.REALM) + # With Digest Authentication + def create_fake_proxy_handler(*args, **kwargs): + return FakeProxyHandler(self.digest_auth_handler, *args, **kwargs) + + self.server = LoopbackHttpServerThread(create_fake_proxy_handler) + self.server.start() + self.server.ready.wait() + proxy_url = "http://127.0.0.1:%d" % self.server.port + handler = urllib2.ProxyHandler({"http" : proxy_url}) + self.proxy_digest_handler = urllib2.ProxyDigestAuthHandler() + self.opener = urllib2.build_opener(handler, self.proxy_digest_handler) + + def tearDown(self): + self.server.stop() + super(ProxyAuthTests, self).tearDown() + + def test_proxy_with_bad_password_raises_httperror(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD+"bad") + self.digest_auth_handler.set_qop("auth") + self.assertRaises(urllib2.HTTPError, + self.opener.open, + self.URL) + + def test_proxy_with_no_password_raises_httperror(self): + self.digest_auth_handler.set_qop("auth") + self.assertRaises(urllib2.HTTPError, + self.opener.open, + self.URL) + + def test_proxy_qop_auth_works(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD) + self.digest_auth_handler.set_qop("auth") + result = self.opener.open(self.URL) + while result.read(): + pass + result.close() + + def test_proxy_qop_auth_int_works_or_throws_urlerror(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD) + self.digest_auth_handler.set_qop("auth-int") + try: + result = self.opener.open(self.URL) + except urllib2.URLError: + # It's okay if we don't support auth-int, but we certainly + # shouldn't receive any kind of exception here other than + # a URLError. + result = None + if result: + while result.read(): + pass + result.close() + + +def GetRequestHandler(responses): + + class FakeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + + server_version = "TestHTTP/" + requests = [] + headers_received = [] + port = 80 + + def do_GET(self): + body = self.send_head() + if body: + self.wfile.write(body) + + def do_POST(self): + content_length = self.headers['Content-Length'] + post_data = self.rfile.read(int(content_length)) + self.do_GET() + self.requests.append(post_data) + + def send_head(self): + FakeHTTPRequestHandler.headers_received = self.headers + self.requests.append(self.path) + response_code, headers, body = responses.pop(0) + + self.send_response(response_code) + + for (header, value) in headers: + self.send_header(header, value % self.port) + if body: + self.send_header('Content-type', 'text/plain') + self.end_headers() + return body + self.end_headers() + + def log_message(self, *args): + pass + + + return FakeHTTPRequestHandler + + +class TestUrlopen(BaseTestCase): + """Tests urllib2.urlopen using the network. + + These tests are not exhaustive. Assuming that testing using files does a + good job overall of some of the basic interface features. There are no + tests exercising the optional 'data' and 'proxies' arguments. No tests + for transparent redirection have been written. + """ + + def setUp(self): + proxy_handler = urllib2.ProxyHandler({}) + opener = urllib2.build_opener(proxy_handler) + urllib2.install_opener(opener) + super(TestUrlopen, self).setUp() + + def urlopen(self, url, data=None, **kwargs): + l = [] + f = urllib2.urlopen(url, data, **kwargs) + try: + # Exercise various methods + l.extend(f.readlines(200)) + l.append(f.readline()) + l.append(f.read(1024)) + l.append(f.read()) + finally: + f.close() + return b"".join(l) + + def start_server(self, responses): + handler = GetRequestHandler(responses) + + self.server = LoopbackHttpServerThread(handler) + self.server.start() + self.server.ready.wait() + port = self.server.port + handler.port = port + return handler + + def start_https_server(self, responses=None, **kwargs): + if not hasattr(urllib2, 'HTTPSHandler'): + self.skipTest('ssl support required') + from test.ssl_servers import make_https_server + if responses is None: + responses = [(200, [], b"we care a bit")] + handler = GetRequestHandler(responses) + server = make_https_server(self, handler_class=handler, **kwargs) + handler.port = server.port + return handler + + def test_redirection(self): + expected_response = 'We got here...' + responses = [ + (302, [('Location', 'http://localhost:%s/somewhere_else')], ''), + (200, [], expected_response) + ] + + handler = self.start_server(responses) + + try: + f = urllib2.urlopen('http://localhost:%s/' % handler.port) + data = f.read() + f.close() + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ['/', '/somewhere_else']) + finally: + self.server.stop() + + + def test_404(self): + expected_response = 'Bad bad bad...' + handler = self.start_server([(404, [], expected_response)]) + + try: + try: + urllib2.urlopen('http://localhost:%s/weeble' % handler.port) + except urllib2.URLError, f: + pass + else: + self.fail('404 should raise URLError') + + data = f.read() + f.close() + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ['/weeble']) + finally: + self.server.stop() + + + def test_200(self): + expected_response = 'pycon 2008...' + handler = self.start_server([(200, [], expected_response)]) + + try: + f = urllib2.urlopen('http://localhost:%s/bizarre' % handler.port) + data = f.read() + f.close() + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ['/bizarre']) + finally: + self.server.stop() + + def test_200_with_parameters(self): + expected_response = 'pycon 2008...' + handler = self.start_server([(200, [], expected_response)]) + + try: + f = urllib2.urlopen('http://localhost:%s/bizarre' % handler.port, 'get=with_feeling') + data = f.read() + f.close() + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ['/bizarre', 'get=with_feeling']) + finally: + self.server.stop() + + def test_https(self): + handler = self.start_https_server() + context = ssl.create_default_context(cafile=CERT_localhost) + data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context) + self.assertEqual(data, b"we care a bit") + + def test_https_with_cafile(self): + handler = self.start_https_server(certfile=CERT_localhost) + # Good cert + data = self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_localhost) + self.assertEqual(data, b"we care a bit") + # Bad cert + with self.assertRaises(urllib2.URLError): + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_fakehostname) + # Good cert, but mismatching hostname + handler = self.start_https_server(certfile=CERT_fakehostname) + with self.assertRaises(ssl.CertificateError): + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_fakehostname) + + def test_https_with_cadefault(self): + handler = self.start_https_server(certfile=CERT_localhost) + # Self-signed cert should fail verification with system certificate store + with self.assertRaises(urllib2.URLError): + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cadefault=True) + + def test_https_sni(self): + if ssl is None: + self.skipTest("ssl module required") + if not ssl.HAS_SNI: + self.skipTest("SNI support required in OpenSSL") + sni_name = [None] + def cb_sni(ssl_sock, server_name, initial_context): + sni_name[0] = server_name + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.set_servername_callback(cb_sni) + handler = self.start_https_server(context=context, certfile=CERT_localhost) + context = ssl.create_default_context(cafile=CERT_localhost) + self.urlopen("https://localhost:%s" % handler.port, context=context) + self.assertEqual(sni_name[0], "localhost") + + def test_sending_headers(self): + handler = self.start_server([(200, [], "we don't care")]) + + try: + req = urllib2.Request("http://localhost:%s/" % handler.port, + headers={'Range': 'bytes=20-39'}) + urllib2.urlopen(req) + self.assertEqual(handler.headers_received['Range'], 'bytes=20-39') + finally: + self.server.stop() + + def test_basic(self): + handler = self.start_server([(200, [], "we don't care")]) + + try: + open_url = urllib2.urlopen("http://localhost:%s" % handler.port) + for attr in ("read", "close", "info", "geturl"): + self.assertTrue(hasattr(open_url, attr), "object returned from " + "urlopen lacks the %s attribute" % attr) + try: + self.assertTrue(open_url.read(), "calling 'read' failed") + finally: + open_url.close() + finally: + self.server.stop() + + def test_info(self): + handler = self.start_server([(200, [], "we don't care")]) + + try: + open_url = urllib2.urlopen("http://localhost:%s" % handler.port) + info_obj = open_url.info() + self.assertIsInstance(info_obj, mimetools.Message, + "object returned by 'info' is not an " + "instance of mimetools.Message") + self.assertEqual(info_obj.getsubtype(), "plain") + finally: + self.server.stop() + + def test_geturl(self): + # Make sure same URL as opened is returned by geturl. + handler = self.start_server([(200, [], "we don't care")]) + + try: + open_url = urllib2.urlopen("http://localhost:%s" % handler.port) + url = open_url.geturl() + self.assertEqual(url, "http://localhost:%s" % handler.port) + finally: + self.server.stop() + + + def test_bad_address(self): + # Make sure proper exception is raised when connecting to a bogus + # address. + + # as indicated by the comment below, this might fail with some ISP, + # so we run the test only when -unetwork/-uall is specified to + # mitigate the problem a bit (see #17564) + test_support.requires('network') + self.assertRaises(IOError, + # Given that both VeriSign and various ISPs have in + # the past or are presently hijacking various invalid + # domain name requests in an attempt to boost traffic + # to their own sites, finding a domain name to use + # for this test is difficult. RFC2606 leads one to + # believe that '.invalid' should work, but experience + # seemed to indicate otherwise. Single character + # TLDs are likely to remain invalid, so this seems to + # be the best choice. The trailing '.' prevents a + # related problem: The normal DNS resolver appends + # the domain names from the search path if there is + # no '.' the end and, and if one of those domains + # implements a '*' rule a result is returned. + # However, none of this will prevent the test from + # failing if the ISP hijacks all invalid domain + # requests. The real solution would be to be able to + # parameterize the framework with a mock resolver. + urllib2.urlopen, "http://sadflkjsasf.i.nvali.d./") + + def test_iteration(self): + expected_response = "pycon 2008..." + handler = self.start_server([(200, [], expected_response)]) + try: + data = urllib2.urlopen("http://localhost:%s" % handler.port) + for line in data: + self.assertEqual(line, expected_response) + finally: + self.server.stop() + + def ztest_line_iteration(self): + lines = ["We\n", "got\n", "here\n", "verylong " * 8192 + "\n"] + expected_response = "".join(lines) + handler = self.start_server([(200, [], expected_response)]) + try: + data = urllib2.urlopen("http://localhost:%s" % handler.port) + for index, line in enumerate(data): + self.assertEqual(line, lines[index], + "Fetched line number %s doesn't match expected:\n" + " Expected length was %s, got %s" % + (index, len(lines[index]), len(line))) + finally: + self.server.stop() + self.assertEqual(index + 1, len(lines)) + +def test_main(): + # We will NOT depend on the network resource flag + # (Lib/test/regrtest.py -u network) since all tests here are only + # localhost. However, if this is a bad rationale, then uncomment + # the next line. + #test_support.requires("network") + + test_support.run_unittest(BasicAuthTests, ProxyAuthTests, TestUrlopen) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_urllib2net.py b/src/greentest/2.7/test_urllib2net.py new file mode 100644 index 0000000..13b09ba --- /dev/null +++ b/src/greentest/2.7/test_urllib2net.py @@ -0,0 +1,328 @@ +import unittest +from test import test_support +from test.test_urllib2 import sanepathname2url + +import socket +import urllib2 +import os +import sys + +TIMEOUT = 60 # seconds + + +def _retry_thrice(func, exc, *args, **kwargs): + for i in range(3): + try: + return func(*args, **kwargs) + except exc, last_exc: + continue + except: + raise + raise last_exc + +def _wrap_with_retry_thrice(func, exc): + def wrapped(*args, **kwargs): + return _retry_thrice(func, exc, *args, **kwargs) + return wrapped + +# Connecting to remote hosts is flaky. Make it more robust by retrying +# the connection several times. +_urlopen_with_retry = _wrap_with_retry_thrice(urllib2.urlopen, urllib2.URLError) + + +class AuthTests(unittest.TestCase): + """Tests urllib2 authentication features.""" + +## Disabled at the moment since there is no page under python.org which +## could be used to HTTP authentication. +# +# def test_basic_auth(self): +# import httplib +# +# test_url = "http://www.python.org/test/test_urllib2/basic_auth" +# test_hostport = "www.python.org" +# test_realm = 'Test Realm' +# test_user = 'test.test_urllib2net' +# test_password = 'blah' +# +# # failure +# try: +# _urlopen_with_retry(test_url) +# except urllib2.HTTPError, exc: +# self.assertEqual(exc.code, 401) +# else: +# self.fail("urlopen() should have failed with 401") +# +# # success +# auth_handler = urllib2.HTTPBasicAuthHandler() +# auth_handler.add_password(test_realm, test_hostport, +# test_user, test_password) +# opener = urllib2.build_opener(auth_handler) +# f = opener.open('http://localhost/') +# response = _urlopen_with_retry("http://www.python.org/") +# +# # The 'userinfo' URL component is deprecated by RFC 3986 for security +# # reasons, let's not implement it! (it's already implemented for proxy +# # specification strings (that is, URLs or authorities specifying a +# # proxy), so we must keep that) +# self.assertRaises(httplib.InvalidURL, +# urllib2.urlopen, "http://evil:thing@example.com") + + +class CloseSocketTest(unittest.TestCase): + + def test_close(self): + import httplib + + # calling .close() on urllib2's response objects should close the + # underlying socket + + # delve deep into response to fetch socket._socketobject + response = _urlopen_with_retry("http://www.example.com/") + abused_fileobject = response.fp + #self.assertIs(abused_fileobject.__class__, socket._fileobject) # JAM: gevent: disable + httpresponse = abused_fileobject._sock + self.assertIs(httpresponse.__class__, httplib.HTTPResponse) + fileobject = httpresponse.fp + #self.assertIs(fileobject.__class__, socket._fileobject) # JAM: gevent: disable + + self.assertTrue(not fileobject.closed) + response.close() + self.assertTrue(fileobject.closed) + +class OtherNetworkTests(unittest.TestCase): + def setUp(self): + if 0: # for debugging + import logging + logger = logging.getLogger("test_urllib2net") + logger.addHandler(logging.StreamHandler()) + + # XXX The rest of these tests aren't very good -- they don't check much. + # They do sometimes catch some major disasters, though. + + def test_ftp(self): + urls = [ + 'ftp://ftp.debian.org/debian/README', + ('ftp://ftp.debian.org/debian/non-existent-file', + None, urllib2.URLError), + ] + self._test_urls(urls, self._extra_handlers()) + + def test_file(self): + TESTFN = test_support.TESTFN + f = open(TESTFN, 'w') + try: + f.write('hi there\n') + f.close() + urls = [ + 'file:'+sanepathname2url(os.path.abspath(TESTFN)), + ('file:///nonsensename/etc/passwd', None, urllib2.URLError), + ] + self._test_urls(urls, self._extra_handlers(), retry=True) + finally: + os.remove(TESTFN) + + self.assertRaises(ValueError, urllib2.urlopen,'./relative_path/to/file') + + # XXX Following test depends on machine configurations that are internal + # to CNRI. Need to set up a public server with the right authentication + # configuration for test purposes. + +## def test_cnri(self): +## if socket.gethostname() == 'bitdiddle': +## localhost = 'bitdiddle.cnri.reston.va.us' +## elif socket.gethostname() == 'bitdiddle.concentric.net': +## localhost = 'localhost' +## else: +## localhost = None +## if localhost is not None: +## urls = [ +## 'file://%s/etc/passwd' % localhost, +## 'http://%s/simple/' % localhost, +## 'http://%s/digest/' % localhost, +## 'http://%s/not/found.h' % localhost, +## ] + +## bauth = HTTPBasicAuthHandler() +## bauth.add_password('basic_test_realm', localhost, 'jhylton', +## 'password') +## dauth = HTTPDigestAuthHandler() +## dauth.add_password('digest_test_realm', localhost, 'jhylton', +## 'password') + +## self._test_urls(urls, self._extra_handlers()+[bauth, dauth]) + + def test_urlwithfrag(self): + urlwith_frag = "http://www.pythontest.net/index.html#frag" + with test_support.transient_internet(urlwith_frag): + req = urllib2.Request(urlwith_frag) + res = urllib2.urlopen(req) + self.assertEqual(res.geturl(), + "http://www.pythontest.net/index.html#frag") + + def test_fileno(self): + req = urllib2.Request("http://www.example.com") + opener = urllib2.build_opener() + res = opener.open(req) + try: + res.fileno() + except AttributeError: + self.fail("HTTPResponse object should return a valid fileno") + finally: + res.close() + + def test_custom_headers(self): + url = "http://www.example.com" + with test_support.transient_internet(url): + opener = urllib2.build_opener() + request = urllib2.Request(url) + self.assertFalse(request.header_items()) + opener.open(request) + self.assertTrue(request.header_items()) + self.assertTrue(request.has_header('User-agent')) + request.add_header('User-Agent','Test-Agent') + opener.open(request) + self.assertEqual(request.get_header('User-agent'),'Test-Agent') + + def test_sites_no_connection_close(self): + # Some sites do not send Connection: close header. + # Verify that those work properly. (#issue12576) + + URL = 'http://www.imdb.com' # No Connection:close + with test_support.transient_internet(URL): + req = urllib2.urlopen(URL) + res = req.read() + self.assertTrue(res) + + def _test_urls(self, urls, handlers, retry=True): + import time + import logging + debug = logging.getLogger("test_urllib2").debug + + urlopen = urllib2.build_opener(*handlers).open + if retry: + urlopen = _wrap_with_retry_thrice(urlopen, urllib2.URLError) + + for url in urls: + if isinstance(url, tuple): + url, req, expected_err = url + else: + req = expected_err = None + with test_support.transient_internet(url): + debug(url) + try: + f = urlopen(url, req, TIMEOUT) + except EnvironmentError as err: + debug(err) + if expected_err: + msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % + (expected_err, url, req, type(err), err)) + self.assertIsInstance(err, expected_err, msg) + except urllib2.URLError as err: + if isinstance(err[0], socket.timeout): + print >>sys.stderr, "" % url + continue + else: + raise + else: + try: + with test_support.transient_internet(url): + buf = f.read() + debug("read %d bytes" % len(buf)) + except socket.timeout: + print >>sys.stderr, "" % url + f.close() + debug("******** next url coming up...") + time.sleep(0.1) + + def _extra_handlers(self): + handlers = [] + + cfh = urllib2.CacheFTPHandler() + self.addCleanup(cfh.clear_cache) + cfh.setTimeout(1) + handlers.append(cfh) + + return handlers + + +class TimeoutTest(unittest.TestCase): + def test_http_basic(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with test_support.transient_internet(url, timeout=None): + u = _urlopen_with_retry(url) + self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) + + def test_http_default_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with test_support.transient_internet(url): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(url) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60) + + def test_http_no_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with test_support.transient_internet(url): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(url, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) + + def test_http_timeout(self): + url = "http://www.example.com" + with test_support.transient_internet(url): + u = _urlopen_with_retry(url, timeout=120) + self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) + + FTP_HOST = 'ftp://ftp.debian.org/debian/' + + def test_ftp_basic(self): + self.assertIsNone(socket.getdefaulttimeout()) + with test_support.transient_internet(self.FTP_HOST, timeout=None): + u = _urlopen_with_retry(self.FTP_HOST) + self.assertIsNone(u.fp.fp._sock.gettimeout()) + + def test_ftp_default_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + with test_support.transient_internet(self.FTP_HOST): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(self.FTP_HOST) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + + def test_ftp_no_timeout(self): + self.assertIsNone(socket.getdefaulttimeout(),) + with test_support.transient_internet(self.FTP_HOST): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(self.FTP_HOST, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(u.fp.fp._sock.gettimeout()) + + def test_ftp_timeout(self): + with test_support.transient_internet(self.FTP_HOST): + u = _urlopen_with_retry(self.FTP_HOST, timeout=60) + self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + + +def test_main(): + test_support.requires("network") + test_support.run_unittest(AuthTests, + OtherNetworkTests, + CloseSocketTest, + TimeoutTest, + ) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/test_wsgiref.py b/src/greentest/2.7/test_wsgiref.py new file mode 100644 index 0000000..2469f67 --- /dev/null +++ b/src/greentest/2.7/test_wsgiref.py @@ -0,0 +1,571 @@ +from unittest import TestCase +from wsgiref.util import setup_testing_defaults +from wsgiref.headers import Headers +from wsgiref.handlers import BaseHandler, BaseCGIHandler +from wsgiref import util +from wsgiref.validate import validator +from wsgiref.simple_server import WSGIServer, WSGIRequestHandler +from wsgiref.simple_server import make_server +from StringIO import StringIO +from SocketServer import BaseServer + +import os +import re +import sys + +from test import test_support + +class MockServer(WSGIServer): + """Non-socket HTTP server""" + + def __init__(self, server_address, RequestHandlerClass): + BaseServer.__init__(self, server_address, RequestHandlerClass) + self.server_bind() + + def server_bind(self): + host, port = self.server_address + self.server_name = host + self.server_port = port + self.setup_environ() + + +class MockHandler(WSGIRequestHandler): + """Non-socket HTTP handler""" + def setup(self): + self.connection = self.request + self.rfile, self.wfile = self.connection + + def finish(self): + pass + + +def hello_app(environ,start_response): + start_response("200 OK", [ + ('Content-Type','text/plain'), + ('Date','Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return ["Hello, world!"] + +def run_amock(app=hello_app, data="GET / HTTP/1.0\n\n"): + server = make_server("", 80, app, MockServer, MockHandler) + inp, out, err, olderr = StringIO(data), StringIO(), StringIO(), sys.stderr + sys.stderr = err + + try: + server.finish_request((inp,out), ("127.0.0.1",8888)) + finally: + sys.stderr = olderr + + return out.getvalue(), err.getvalue() + + +def compare_generic_iter(make_it,match): + """Utility to compare a generic 2.1/2.2+ iterator with an iterable + + If running under Python 2.2+, this tests the iterator using iter()/next(), + as well as __getitem__. 'make_it' must be a function returning a fresh + iterator to be tested (since this may test the iterator twice).""" + + it = make_it() + n = 0 + for item in match: + if not it[n]==item: raise AssertionError + n+=1 + try: + it[n] + except IndexError: + pass + else: + raise AssertionError("Too many items from __getitem__",it) + + try: + iter, StopIteration + except NameError: + pass + else: + # Only test iter mode under 2.2+ + it = make_it() + if not iter(it) is it: raise AssertionError + for item in match: + if not it.next()==item: raise AssertionError + try: + it.next() + except StopIteration: + pass + else: + raise AssertionError("Too many items from .next()",it) + + +class IntegrationTests(TestCase): + + def check_hello(self, out, has_length=True): + self.assertEqual(out, + "HTTP/1.0 200 OK\r\n" + "Server: WSGIServer/0.1 Python/"+sys.version.split()[0]+"\r\n" + "Content-Type: text/plain\r\n" + "Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + + (has_length and "Content-Length: 13\r\n" or "") + + "\r\n" + "Hello, world!" + ) + + def test_plain_hello(self): + out, err = run_amock() + self.check_hello(out) + + def test_request_length(self): + out, err = run_amock(data="GET " + ("x" * 65537) + " HTTP/1.0\n\n") + self.assertEqual(out.splitlines()[0], + "HTTP/1.0 414 Request-URI Too Long") + + def test_validated_hello(self): + out, err = run_amock(validator(hello_app)) + # the middleware doesn't support len(), so content-length isn't there + self.check_hello(out, has_length=False) + + def test_simple_validation_error(self): + def bad_app(environ,start_response): + start_response("200 OK", ('Content-Type','text/plain')) + return ["Hello, world!"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + "A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], + "AssertionError: Headers (('Content-Type', 'text/plain')) must" + " be of type list: " + ) + + +class UtilityTests(TestCase): + + def checkShift(self,sn_in,pi_in,part,sn_out,pi_out): + env = {'SCRIPT_NAME':sn_in,'PATH_INFO':pi_in} + util.setup_testing_defaults(env) + self.assertEqual(util.shift_path_info(env),part) + self.assertEqual(env['PATH_INFO'],pi_out) + self.assertEqual(env['SCRIPT_NAME'],sn_out) + return env + + def checkDefault(self, key, value, alt=None): + # Check defaulting when empty + env = {} + util.setup_testing_defaults(env) + if isinstance(value, StringIO): + self.assertIsInstance(env[key], StringIO) + else: + self.assertEqual(env[key], value) + + # Check existing value + env = {key:alt} + util.setup_testing_defaults(env) + self.assertIs(env[key], alt) + + def checkCrossDefault(self,key,value,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(kw[key],value) + + def checkAppURI(self,uri,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.application_uri(kw),uri) + + def checkReqURI(self,uri,query=1,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.request_uri(kw,query),uri) + + def checkFW(self,text,size,match): + + def make_it(text=text,size=size): + return util.FileWrapper(StringIO(text),size) + + compare_generic_iter(make_it,match) + + it = make_it() + self.assertFalse(it.filelike.closed) + + for item in it: + pass + + self.assertFalse(it.filelike.closed) + + it.close() + self.assertTrue(it.filelike.closed) + + def testSimpleShifts(self): + self.checkShift('','/', '', '/', '') + self.checkShift('','/x', 'x', '/x', '') + self.checkShift('/','', None, '/', '') + self.checkShift('/a','/x/y', 'x', '/a/x', '/y') + self.checkShift('/a','/x/', 'x', '/a/x', '/') + + def testNormalizedShifts(self): + self.checkShift('/a/b', '/../y', '..', '/a', '/y') + self.checkShift('', '/../y', '..', '', '/y') + self.checkShift('/a/b', '//y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '//y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '/./y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '/./y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '///./..//y/.//', '..', '/a', '/y/') + self.checkShift('/a/b', '///', '', '/a/b/', '') + self.checkShift('/a/b', '/.//', '', '/a/b/', '') + self.checkShift('/a/b', '/x//', 'x', '/a/b/x', '/') + self.checkShift('/a/b', '/.', None, '/a/b', '') + + def testDefaults(self): + for key, value in [ + ('SERVER_NAME','127.0.0.1'), + ('SERVER_PORT', '80'), + ('SERVER_PROTOCOL','HTTP/1.0'), + ('HTTP_HOST','127.0.0.1'), + ('REQUEST_METHOD','GET'), + ('SCRIPT_NAME',''), + ('PATH_INFO','/'), + ('wsgi.version', (1,0)), + ('wsgi.run_once', 0), + ('wsgi.multithread', 0), + ('wsgi.multiprocess', 0), + ('wsgi.input', StringIO("")), + ('wsgi.errors', StringIO()), + ('wsgi.url_scheme','http'), + ]: + self.checkDefault(key,value) + + def testCrossDefaults(self): + self.checkCrossDefault('HTTP_HOST',"foo.bar",SERVER_NAME="foo.bar") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="on") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="1") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="yes") + self.checkCrossDefault('wsgi.url_scheme',"http",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"80",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"443",HTTPS="on") + + def testGuessScheme(self): + self.assertEqual(util.guess_scheme({}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"foo"}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"on"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"yes"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"1"}), "https") + + def testAppURIs(self): + self.checkAppURI("http://127.0.0.1/") + self.checkAppURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkAppURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkAppURI("http://spam.example.com:2071/", + HTTP_HOST="spam.example.com:2071", SERVER_PORT="2071") + self.checkAppURI("http://spam.example.com/", + SERVER_NAME="spam.example.com") + self.checkAppURI("http://127.0.0.1/", + HTTP_HOST="127.0.0.1", SERVER_NAME="spam.example.com") + self.checkAppURI("https://127.0.0.1/", HTTPS="on") + self.checkAppURI("http://127.0.0.1:8000/", SERVER_PORT="8000", + HTTP_HOST=None) + + def testReqURIs(self): + self.checkReqURI("http://127.0.0.1/") + self.checkReqURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkReqURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam", + SCRIPT_NAME="/spammity", PATH_INFO="/spam") + self.checkReqURI("http://127.0.0.1/spammity/sp%E4m", + SCRIPT_NAME="/spammity", PATH_INFO="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam;ham", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;ham") + self.checkReqURI("http://127.0.0.1/spammity/spam;cookie=1234,5678", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;cookie=1234,5678") + self.checkReqURI("http://127.0.0.1/spammity/spam?say=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam?s%E4y=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="s%E4y=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam", 0, + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + + def testFileWrapper(self): + self.checkFW("xyz"*50, 120, ["xyz"*40,"xyz"*10]) + + def testHopByHop(self): + for hop in ( + "Connection Keep-Alive Proxy-Authenticate Proxy-Authorization " + "TE Trailers Transfer-Encoding Upgrade" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertTrue(util.is_hop_by_hop(alt)) + + # Not comprehensive, just a few random header names + for hop in ( + "Accept Cache-Control Date Pragma Trailer Via Warning" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertFalse(util.is_hop_by_hop(alt)) + +class HeaderTests(TestCase): + + def testMappingInterface(self): + test = [('x','y')] + self.assertEqual(len(Headers([])),0) + self.assertEqual(len(Headers(test[:])),1) + self.assertEqual(Headers(test[:]).keys(), ['x']) + self.assertEqual(Headers(test[:]).values(), ['y']) + self.assertEqual(Headers(test[:]).items(), test) + self.assertIsNot(Headers(test).items(), test) # must be copy! + + h=Headers([]) + del h['foo'] # should not raise an error + + h['Foo'] = 'bar' + for m in h.has_key, h.__contains__, h.get, h.get_all, h.__getitem__: + self.assertTrue(m('foo')) + self.assertTrue(m('Foo')) + self.assertTrue(m('FOO')) + self.assertFalse(m('bar')) + + self.assertEqual(h['foo'],'bar') + h['foo'] = 'baz' + self.assertEqual(h['FOO'],'baz') + self.assertEqual(h.get_all('foo'),['baz']) + + self.assertEqual(h.get("foo","whee"), "baz") + self.assertEqual(h.get("zoo","whee"), "whee") + self.assertEqual(h.setdefault("foo","whee"), "baz") + self.assertEqual(h.setdefault("zoo","whee"), "whee") + self.assertEqual(h["foo"],"baz") + self.assertEqual(h["zoo"],"whee") + + def testRequireList(self): + self.assertRaises(TypeError, Headers, "foo") + + + def testExtras(self): + h = Headers([]) + self.assertEqual(str(h),'\r\n') + + h.add_header('foo','bar',baz="spam") + self.assertEqual(h['foo'], 'bar; baz="spam"') + self.assertEqual(str(h),'foo: bar; baz="spam"\r\n\r\n') + + h.add_header('Foo','bar',cheese=None) + self.assertEqual(h.get_all('foo'), + ['bar; baz="spam"', 'bar; cheese']) + + self.assertEqual(str(h), + 'foo: bar; baz="spam"\r\n' + 'Foo: bar; cheese\r\n' + '\r\n' + ) + + +class ErrorHandler(BaseCGIHandler): + """Simple handler subclass for testing BaseHandler""" + + # BaseHandler records the OS environment at import time, but envvars + # might have been changed later by other tests, which trips up + # HandlerTests.testEnviron(). + os_environ = dict(os.environ.items()) + + def __init__(self,**kw): + setup_testing_defaults(kw) + BaseCGIHandler.__init__( + self, StringIO(''), StringIO(), StringIO(), kw, + multithread=True, multiprocess=True + ) + +class TestHandler(ErrorHandler): + """Simple handler subclass for testing BaseHandler, w/error passthru""" + + def handle_error(self): + raise # for testing, we want to see what's happening + + +class HandlerTests(TestCase): + + def checkEnvironAttrs(self, handler): + env = handler.environ + for attr in [ + 'version','multithread','multiprocess','run_once','file_wrapper' + ]: + if attr=='file_wrapper' and handler.wsgi_file_wrapper is None: + continue + self.assertEqual(getattr(handler,'wsgi_'+attr),env['wsgi.'+attr]) + + def checkOSEnviron(self,handler): + empty = {}; setup_testing_defaults(empty) + env = handler.environ + from os import environ + for k,v in environ.items(): + if k not in empty: + self.assertEqual(env[k],v) + for k,v in empty.items(): + self.assertIn(k, env) + + def testEnviron(self): + h = TestHandler(X="Y") + h.setup_environ() + self.checkEnvironAttrs(h) + self.checkOSEnviron(h) + self.assertEqual(h.environ["X"],"Y") + + def testCGIEnviron(self): + h = BaseCGIHandler(None,None,None,{}) + h.setup_environ() + for key in 'wsgi.url_scheme', 'wsgi.input', 'wsgi.errors': + self.assertIn(key, h.environ) + + def testScheme(self): + h=TestHandler(HTTPS="on"); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'https') + h=TestHandler(); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'http') + + def testAbstractMethods(self): + h = BaseHandler() + for name in [ + '_flush','get_stdin','get_stderr','add_cgi_vars' + ]: + self.assertRaises(NotImplementedError, getattr(h,name)) + self.assertRaises(NotImplementedError, h._write, "test") + + def testContentLength(self): + # Demo one reason iteration is better than write()... ;) + + def trivial_app1(e,s): + s('200 OK',[]) + return [e['wsgi.url_scheme']] + + def trivial_app2(e,s): + s('200 OK',[])(e['wsgi.url_scheme']) + return [] + + def trivial_app4(e,s): + # Simulate a response to a HEAD request + s('200 OK',[('Content-Length', '12345')]) + return [] + + h = TestHandler() + h.run(trivial_app1) + self.assertEqual(h.stdout.getvalue(), + "Status: 200 OK\r\n" + "Content-Length: 4\r\n" + "\r\n" + "http") + + h = TestHandler() + h.run(trivial_app2) + self.assertEqual(h.stdout.getvalue(), + "Status: 200 OK\r\n" + "\r\n" + "http") + + + h = TestHandler() + h.run(trivial_app4) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 12345\r\n' + b'\r\n') + + def testBasicErrorOutput(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + def error_app(e,s): + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(non_error_app) + self.assertEqual(h.stdout.getvalue(), + "Status: 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n") + self.assertEqual(h.stderr.getvalue(),"") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + "Status: %s\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: %d\r\n" + "\r\n%s" % (h.error_status,len(h.error_body),h.error_body)) + + self.assertNotEqual(h.stderr.getvalue().find("AssertionError"), -1) + + def testErrorAfterOutput(self): + MSG = "Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + "Status: 200 OK\r\n" + "\r\n"+MSG) + self.assertNotEqual(h.stderr.getvalue().find("AssertionError"), -1) + + def testHeaderFormats(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + stdpat = ( + r"HTTP/%s 200 OK\r\n" + r"Date: \w{3}, [ 0123]\d \w{3} \d{4} \d\d:\d\d:\d\d GMT\r\n" + r"%s" r"Content-Length: 0\r\n" r"\r\n" + ) + shortpat = ( + "Status: 200 OK\r\n" "Content-Length: 0\r\n" "\r\n" + ) + + for ssw in "FooBar/1.0", None: + sw = ssw and "Server: %s\r\n" % ssw or "" + + for version in "1.0", "1.1": + for proto in "HTTP/0.9", "HTTP/1.0", "HTTP/1.1": + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = False + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + self.assertEqual(shortpat,h.stdout.getvalue()) + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = True + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + if proto=="HTTP/0.9": + self.assertEqual(h.stdout.getvalue(),"") + else: + self.assertTrue( + re.match(stdpat%(version,sw), h.stdout.getvalue()), + (stdpat%(version,sw), h.stdout.getvalue()) + ) + + def testCloseOnError(self): + side_effects = {'close_called': False} + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + class CrashyIterable(object): + def __iter__(self): + while True: + yield b'blah' + raise AssertionError("This should be caught by handler") + + def close(self): + side_effects['close_called'] = True + return CrashyIterable() + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(side_effects['close_called'], True) + + +def test_main(): + test_support.run_unittest(__name__) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7/version b/src/greentest/2.7/version new file mode 100644 index 0000000..af9c448 --- /dev/null +++ b/src/greentest/2.7/version @@ -0,0 +1 @@ +2.7.11 diff --git a/src/greentest/2.7/wrongcert.pem b/src/greentest/2.7/wrongcert.pem new file mode 100644 index 0000000..5f92f9b --- /dev/null +++ b/src/greentest/2.7/wrongcert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH +FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T +f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB +AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq +1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW +7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg +SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe +19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg +ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666 +lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs +ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv +frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk +2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6 ++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK +24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G +A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9 +OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ +fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E +usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+ +43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw +eSHj5jpC8iZKjCHBn+mAi4cQ514= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/allsans.pem b/src/greentest/2.7pypy/allsans.pem new file mode 100644 index 0000000..3ee4f59 --- /dev/null +++ b/src/greentest/2.7pypy/allsans.pem @@ -0,0 +1,37 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOoy7/QOtTjQ0niE +6uDcTwtkC0R2Tvy1AjVnXohCntZfdzbTGDoYTgXSOLsP8A697jUiJ8VCePGH50xG +Z4DKnAF3a9O3a9nr2pLXb0iY3XOMv+YEBii7CfI+3oxFYgCl0sMgHzDD2ZTVYAsm +DWgLUVsE2gHEccRwrM2tPf2EgR+FAgMBAAECgYEA3qyfyYVSeTrTYxO93x6ZaVMu +A2IZp9zSxMQL9bKiI2GRj+cV2ebSCGbg2btFnD6qBor7FWsmYz+8g6FNN/9sY4az +61rMqMtQvLBe+7L8w70FeTze4qQ4Y1oQri0qD6tBWhDVlpnbI5Py9bkZKD67yVUk +elcEA/5x4PrYXkuqsAECQQD80NjT0mDvaY0JOOaQFSEpMv6QiUA8GGX8Xli7IoKb +tAolPG8rQBa+qSpcWfDMTrWw/aWHuMEEQoP/bVDH9W4FAkEA7SYQbBAKnojZ5A3G +kOHdV7aeivRQxQk/JN8Fb8oKB9Csvpv/BsuGxPKXHdhFa6CBTTsNRtHQw/szPo4l +xMIjgQJAPoMxqibR+0EBM6+TKzteSL6oPXsCnBl4Vk/J5vPgkbmR7KUl4+7j8N8J +b2554TrxKEN/w7CGYZRE6UrRd7ATNQJAWD7Yz41sli+wfPdPU2xo1BHljyl4wMk/ +EPZYbI/PCbdyAH/F935WyQTIjNeEhZc1Zkq6FwdOWw8ns3hrv3rKgQJAHXv1BqUa +czGPIFxX2TNoqtcl6/En4vrxVB1wzsfzkkDAg98kBl7qsF+S3qujSzKikjeaVbI2 +/CyWR2P3yLtOmA== +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDcjCCAtugAwIBAgIJAN5dc9TOWjB7MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTYwODA1 +MTAyMTExWhcNMjYwODAzMTAyMTExWjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO +Q2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0 +aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQDqMu/0DrU40NJ4hOrg3E8LZAtEdk78tQI1Z16IQp7WX3c20xg6GE4F0ji7D/AO +ve41IifFQnjxh+dMRmeAypwBd2vTt2vZ69qS129ImN1zjL/mBAYouwnyPt6MRWIA +pdLDIB8ww9mU1WALJg1oC1FbBNoBxHHEcKzNrT39hIEfhQIDAQABo4IBODCCATQw +ggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBp +ZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMC +AQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUu +b3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMw +IQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGly +bmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAA +AAAAAAAAAAAAAAAAAYgEKgMEBTANBgkqhkiG9w0BAQsFAAOBgQAy16h+F+nOmeiT +VWR0fc8F/j6FcadbLseAUaogcC15OGxCl4UYpLV88HBkABOoGCpP155qwWTwOrdG +iYPGJSusf1OnJEbvzFejZf6u078bPd9/ZL4VWLjv+FPGkjd+N+/OaqMvgj8Lu99f +3Y/C4S7YbHxxwff6C6l2Xli+q6gnuQ== +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/badcert.pem b/src/greentest/2.7pypy/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/greentest/2.7pypy/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/badkey.pem b/src/greentest/2.7pypy/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/greentest/2.7pypy/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/capath/0e4015b9.0 b/src/greentest/2.7pypy/capath/0e4015b9.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/2.7pypy/capath/0e4015b9.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/capath/4e1295a3.0 b/src/greentest/2.7pypy/capath/4e1295a3.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/2.7pypy/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/capath/5ed36f99.0 b/src/greentest/2.7pypy/capath/5ed36f99.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/2.7pypy/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/capath/6e88d7b8.0 b/src/greentest/2.7pypy/capath/6e88d7b8.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/2.7pypy/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/capath/99d0fa06.0 b/src/greentest/2.7pypy/capath/99d0fa06.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/2.7pypy/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/capath/ce7b8643.0 b/src/greentest/2.7pypy/capath/ce7b8643.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/2.7pypy/capath/ce7b8643.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/dh1024.pem b/src/greentest/2.7pypy/dh1024.pem new file mode 100644 index 0000000..a391176 --- /dev/null +++ b/src/greentest/2.7pypy/dh1024.pem @@ -0,0 +1,7 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAIbzw1s9CT8SV5yv6L7esdAdZYZjPi3qWFs61CYTFFQnf2s/d09NYaJt +rrvJhIzWavqnue71qXCf83/J3nz3FEwUU/L0mGyheVbsSHiI64wUo3u50wK5Igo0 +RNs/LD0irs7m0icZ//hijafTU+JOBiuA8zMI+oZfU7BGuc9XrUprAgEC +-----END DH PARAMETERS----- + +Generated with: openssl dhparam -out dh1024.pem 1024 diff --git a/src/greentest/2.7pypy/https_svn_python_org_root.pem b/src/greentest/2.7pypy/https_svn_python_org_root.pem new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/2.7pypy/https_svn_python_org_root.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/keycert.passwd.pem b/src/greentest/2.7pypy/keycert.passwd.pem new file mode 100644 index 0000000..e905748 --- /dev/null +++ b/src/greentest/2.7pypy/keycert.passwd.pem @@ -0,0 +1,33 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/keycert.pem b/src/greentest/2.7pypy/keycert.pem new file mode 100644 index 0000000..64318aa --- /dev/null +++ b/src/greentest/2.7pypy/keycert.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/keycert2.pem b/src/greentest/2.7pypy/keycert2.pem new file mode 100644 index 0000000..c4a18bf --- /dev/null +++ b/src/greentest/2.7pypy/keycert2.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBANcLaMB7T/Wi9DBc +PltGzgt8cxsv55m7PQPHMZvn6Ke8xmNqcmEzib8opRwKGrCV6TltKeFlNSg8dwQK +Tl4ktyTkGCVweRQJ37AkBayvEBml5s+QD4vlhqkJPsL/Nsd+fnqngOGc5+59+C6r +s3XpiLlF5ah/z8q92Mnw54nypw1JAgMBAAECgYBE3t2Mj7GbDLZB6rj5yKJioVfI +BD6bSJEQ7bGgqdQkLFwpKMU7BiN+ekjuwvmrRkesYZ7BFgXBPiQrwhU5J28Tpj5B +EOMYSIOHfzdalhxDGM1q2oK9LDFiCotTaSdEzMYadel5rmKXJ0zcK2Jho0PCuECf +tf/ghRxK+h1Hm0tKgQJBAO6MdGDSmGKYX6/5kPDje7we/lSLorSDkYmV0tmVShsc +JxgaGaapazceA/sHL3Myx7Eenkip+yPYDXEDFvAKNDECQQDmxsT9NOp6mo7ISvky +GFr2vVHsJ745BMWoma4rFjPBVnS8RkgK+b2EpDCdZSrQ9zw2r8sKTgrEyrDiGTEg +wJyZAkA8OOc0flYMJg2aHnYR6kwVjPmGHI5h5gk648EMPx0rROs1sXkiUwkHLCOz +HvhCq+Iv+9vX2lnVjbiu/CmxRdIxAkA1YEfzoKeTD+hyXxTgB04Sv5sRGegfXAEz +i8gC4zG5R/vcCA1lrHmvEiLEZL/QcT6WD3bQvVg0SAU9ZkI8pxARAkA7yqMSvP1l +gJXy44R+rzpLYb1/PtiLkIkaKG3x9TUfPnfD2jY09fPkZlfsRU3/uS09IkhSwimV +d5rWoljEfdou +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICXTCCAcagAwIBAgIJALVQzebTtrXFMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTAeFw0x +NDExMjMxNzAwMDdaFw0yNDExMjAxNzAwMDdaMGIxCzAJBgNVBAYTAlhZMRcwFQYD +VQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZv +dW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEA1wtowHtP9aL0MFw+W0bOC3xzGy/nmbs9A8cxm+fop7zGY2py +YTOJvyilHAoasJXpOW0p4WU1KDx3BApOXiS3JOQYJXB5FAnfsCQFrK8QGaXmz5AP +i+WGqQk+wv82x35+eqeA4Zzn7n34LquzdemIuUXlqH/Pyr3YyfDnifKnDUkCAwEA +AaMbMBkwFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBBQUAA4GB +AKuay3vDKfWzt5+ch/HHBsert84ISot4fUjzXDA/oOgTOEjVcSShHxqNShMOW1oA +QYBpBB/5Kx5RkD/w6imhucxt2WQPRgjX4x4bwMipVH/HvFDp03mG51/Cpi1TyZ74 +El7qa/Pd4lHhOLzMKBA6503fpeYSFUIBxZbGLqylqRK7 +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/keycert3.pem b/src/greentest/2.7pypy/keycert3.pem new file mode 100644 index 0000000..5bfa62c --- /dev/null +++ b/src/greentest/2.7pypy/keycert3.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP +jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM +9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ +aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe +yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j +y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ +AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW +5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL +9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 +1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT +DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh +1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m +JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 +RnJdHOMXWem7/w== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: + 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: + c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: + 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: + f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: + 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: + f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: + af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: + 21:82:a5:3c:88:e5:be:1b:b1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: + e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: + f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: + e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: + d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: + 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: + ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: + 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: + 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: + 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: + 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: + f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: + 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: + a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: + fc:a9:94:71 +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C +tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola +N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 +TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR +iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG +xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo +5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv +mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF +YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh +2EJ36/yplHE= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/keycert4.pem b/src/greentest/2.7pypy/keycert4.pem new file mode 100644 index 0000000..53355c8 --- /dev/null +++ b/src/greentest/2.7pypy/keycert4.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv +L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2 +NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1 +L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L +pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de +R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9 +myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT +drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS +Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx +i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK +Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu +JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN ++/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/ +e83Gq6ffLVfKNQ== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443282 (0xb09264b1f2da21d2) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14: + 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9: + cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a: + b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76: + 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7: + 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a: + d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79: + 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd: + 81:7e:bd:1b:ae:0b:5d:c6:39 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0: + 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f: + 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56: + 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15: + 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1: + 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb: + 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e: + e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18: + d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1: + af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53: + 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68: + 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da: + 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92: + 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa: + 49:12:1e:ce +-----BEGIN CERTIFICATE----- +MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z +dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU +aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0 +ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ +hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v +xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338 +Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP +XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0 +UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz +aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb +oF+6ufu6+kkSHs4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/nokia.pem b/src/greentest/2.7pypy/nokia.pem new file mode 100644 index 0000000..0d044df --- /dev/null +++ b/src/greentest/2.7pypy/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/nullbytecert.pem b/src/greentest/2.7pypy/nullbytecert.pem new file mode 100644 index 0000000..447186c --- /dev/null +++ b/src/greentest/2.7pypy/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/nullcert.pem b/src/greentest/2.7pypy/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/greentest/2.7pypy/pycacert.pem b/src/greentest/2.7pypy/pycacert.pem new file mode 100644 index 0000000..09b1f3e --- /dev/null +++ b/src/greentest/2.7pypy/pycacert.pem @@ -0,0 +1,78 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Jan 2 19:47:07 2023 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: + 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: + e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: + e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: + 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: + 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: + a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: + e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: + 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: + 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: + e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: + c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: + cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: + 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: + 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: + 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: + e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: + c5:4d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + X509v3 Authority Key Identifier: + keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: + 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: + a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: + 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: + 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: + 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: + fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: + 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: + 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: + 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: + ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: + 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: + b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: + 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: + 5e:58:c8:9e +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/revocation.crl b/src/greentest/2.7pypy/revocation.crl new file mode 100644 index 0000000..6d89b08 --- /dev/null +++ b/src/greentest/2.7pypy/revocation.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTEzMTEyMTE3MDg0N1oXDTIzMDkzMDE3MDg0N1qgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQCNJXC2mVKauEeN3LlQ3ZtM5gkH3ExH ++i4bmJjtJn497WwvvoIeUdrmVXgJQR93RtV37hZwN0SXMLlNmUZPH4rHhihayw4m +unCzVj/OhCCY7/TPjKuJ1O/0XhaLBpBVjQN7R/1ujoRKbSia/CD3vcn7Fqxzw7LK +fSRCKRGTj1CZiuxrphtFchwALXSiFDy9mr2ZKhImcyq1PydfgEzU78APpOkMQsIC +UNJ/cf3c9emzf+dUtcMEcejQ3mynBo4eIGg1EW42bz4q4hSjzQlKcBV0muw5qXhc +HOxH2iTFhQ7SrvVuK/dM14rYM4B5mSX3nRC1kNmXpS9j3wJDhuwmjHed +-----END X509 CRL----- diff --git a/src/greentest/2.7pypy/selfsigned_pythontestdotnet.pem b/src/greentest/2.7pypy/selfsigned_pythontestdotnet.pem new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/2.7pypy/selfsigned_pythontestdotnet.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/sha256.pem b/src/greentest/2.7pypy/sha256.pem new file mode 100644 index 0000000..9475576 --- /dev/null +++ b/src/greentest/2.7pypy/sha256.pem @@ -0,0 +1,128 @@ +# Certificate chain for https://sha256.tbs-internet.com + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=Certificats TBS X509/CN=ecom.tbs-x509.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business +-----BEGIN CERTIFICATE----- +MIIGTjCCBTagAwIBAgIQOh3d9dNDPq1cSdJmEiMpqDANBgkqhkiG9w0BAQUFADCB +yTELMAkGA1UEBhMCRlIxETAPBgNVBAgTCENhbHZhZG9zMQ0wCwYDVQQHEwRDYWVu +MRUwEwYDVQQKEwxUQlMgSU5URVJORVQxSDBGBgNVBAsTP1Rlcm1zIGFuZCBDb25k +aXRpb25zOiBodHRwOi8vd3d3LnRicy1pbnRlcm5ldC5jb20vQ0EvcmVwb3NpdG9y +eTEYMBYGA1UECxMPVEJTIElOVEVSTkVUIENBMR0wGwYDVQQDExRUQlMgWDUwOSBD +QSBidXNpbmVzczAeFw0xMTAxMjUwMDAwMDBaFw0xMzAyMDUyMzU5NTlaMIHHMQsw +CQYDVQQGEwJGUjEOMAwGA1UEERMFMTQwMDAxETAPBgNVBAgTCENhbHZhZG9zMQ0w +CwYDVQQHEwRDQUVOMRswGQYDVQQJExIyMiBydWUgZGUgQnJldGFnbmUxFTATBgNV +BAoTDFRCUyBJTlRFUk5FVDEXMBUGA1UECxMOMDAwMiA0NDA0NDM4MTAxHTAbBgNV +BAsTFENlcnRpZmljYXRzIFRCUyBYNTA5MRowGAYDVQQDExFlY29tLnRicy14NTA5 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRrlHUnJ++1lpcg +jtYco7cdmRe+EEfTmwPfCdfV3G1QfsTSvY6FfMpm/83pqHfT+4ANwr18wD9ZrAEN +G16mf9VdCGK12+TP7DmqeZyGIqlFFoahQnmb8EarvE43/1UeQ2CV9XmzwZvpqeli +LfXsFonawrY3H6ZnMwS64St61Z+9gdyuZ/RbsoZBbT5KUjDEG844QRU4OT1IGeEI +eY5NM5RNIh6ZNhVtqeeCxMS7afONkHQrOco73RdSTRck/Hj96Ofl3MHNHryr+AMK +DGFk1kLCZGpPdXtkxXvaDeQoiYDlil26CWc+YK6xyDPMdsWvoG14ZLyCpzMXA7/7 +4YAQRH0CAwEAAaOCAjAwggIsMB8GA1UdIwQYMBaAFBoJBMz5CY+7HqDO1KQUf0vV +I1jNMB0GA1UdDgQWBBQgOU8HsWzbmD4WZP5Wtdw7jca2WDAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIw +TAYDVR0gBEUwQzBBBgsrBgEEAYDlNwIBATAyMDAGCCsGAQUFBwIBFiRodHRwczov +L3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL0NQUzEwdwYDVR0fBHAwbjA3oDWgM4Yx +aHR0cDovL2NybC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNy +bDAzoDGgL4YtaHR0cDovL2NybC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5l +c3MuY3JsMIGwBggrBgEFBQcBAQSBozCBoDA9BggrBgEFBQcwAoYxaHR0cDovL2Ny +dC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQWJ1c2luZXNzLmNydDA5BggrBgEF +BQcwAoYtaHR0cDovL2NydC50YnMteDUwOS5jb20vVEJTWDUwOUNBYnVzaW5lc3Mu +Y3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC50YnMteDUwOS5jb20wMwYDVR0R +BCwwKoIRZWNvbS50YnMteDUwOS5jb22CFXd3dy5lY29tLnRicy14NTA5LmNvbTAN +BgkqhkiG9w0BAQUFAAOCAQEArT4NHfbY87bGAw8lPV4DmHlmuDuVp/y7ltO3Ynse +3Rz8RxW2AzuO0Oy2F0Cu4yWKtMyEyMXyHqWtae7ElRbdTu5w5GwVBLJHClCzC8S9 +SpgMMQTx3Rgn8vjkHuU9VZQlulZyiPK7yunjc7c310S9FRZ7XxOwf8Nnx4WnB+No +WrfApzhhQl31w+RyrNxZe58hCfDDHmevRvwLjQ785ZoQXJDj2j3qAD4aI2yB8lB5 +oaE1jlCJzC7Kmz/Y9jzfmv/zAs1LQTm9ktevv4BTUFaGjv9jxnQ1xnS862ZiouLW +zZYIlYPf4F6JjXGiIQgQRglILUfq3ftJd9/ok9W9ZF8h8w== +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA business + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFPzCCBCegAwIBAgIQDlBz/++iRSmLDeVRHT/hADANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDcwOTE4MTkyMlow +gckxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEdMBsGA1UEAxMUVEJTIFg1MDkg +Q0EgYnVzaW5lc3MwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDB1PAU +qudCcz3tmyGcf+u6EkZqonKKHrV4gZYbvVkIRojmmlhfi/jwvpHvo8bqSt/9Rj5S +jhCDW0pcbI+IPPtD1Jy+CHNSfnMqVDy6CKQ3p5maTzCMG6ZT+XjnvcND5v+FtaiB +xk1iCX6uvt0jeUtdZvYbyytsSDE6c3Y5//wRxOF8tM1JxibwO3pyER26jbbN2gQz +m/EkdGjLdJ4svPk23WDAvQ6G0/z2LcAaJB+XLfqRwfQpHQvfKa1uTi8PivC8qtip +rmNQMMPMjxSK2azX8cKjjTDJiUKaCb4VHlJDWKEsCFRpgJAoAuX8f7Yfs1M4esGo +sWb3PGspK3O22uIlAgMBAAGjggF6MIIBdjAdBgNVHQ4EFgQUGgkEzPkJj7seoM7U +pBR/S9UjWM0wDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQAwGAYD +VR0gBBEwDzANBgsrBgEEAYDlNwIBATB7BgNVHR8EdDByMDigNqA0hjJodHRwOi8v +Y3JsLmNvbW9kb2NhLmNvbS9BZGRUcnVzdEV4dGVybmFsQ0FSb290LmNybDA2oDSg +MoYwaHR0cDovL2NybC5jb21vZG8ubmV0L0FkZFRydXN0RXh0ZXJuYWxDQVJvb3Qu +Y3JsMIGGBggrBgEFBQcBAQR6MHgwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29t +b2RvY2EuY29tL0FkZFRydXN0VVROU2VydmVyQ0EuY3J0MDkGCCsGAQUFBzAChi1o +dHRwOi8vY3J0LmNvbW9kby5uZXQvQWRkVHJ1c3RVVE5TZXJ2ZXJDQS5jcnQwEQYJ +YIZIAYb4QgEBBAQDAgIEMA0GCSqGSIb3DQEBBQUAA4IBAQA7mqrMgk/MrE6QnbNA +h4nRCn2ti4bg4w2C3lB6bSvRPnYwuNw9Jb8vuKkNFzRDxNJXqVDZdfFW5CVQJuyd +nfAx83+wk+spzvFaE1KhFYfN9G9pQfXUfvDRoIcJgPEKUXL1wRiOG+IjU3VVI8pg +IgqHkr7ylln5i5zCiFAPuIJmYUSFg/gxH5xkCNcjJqqrHrHatJr6Qrrke93joupw +oU1njfAcZtYp6fbiK6u2b1pJqwkVBE8RsfLnPhRj+SFbpvjv8Od7o/ieJhFIYQNU +k2jX2u8qZnAiNw93LZW9lpYjtuvMXq8QQppENNja5b53q7UwI+lU7ZGjZ7quuESp +J6/5 +-----END CERTIFICATE----- + 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware +-----BEGIN CERTIFICATE----- +MIIETzCCAzegAwIBAgIQHM5EYpUZep1jUvnyI6m2mDANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNMDUwNjA3MDgwOTEwWhcNMTkwNzA5MTgxOTIyWjBvMQswCQYD +VQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFkZFRydXN0 +IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBFeHRlcm5h +bCBDQSBSb290MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt/caM+by +AAQtOeBOW+0fvGwPzbX6I7bO3psRM5ekKUx9k5+9SryT7QMa44/P5W1QWtaXKZRa +gLBJetsulf24yr83OC0ePpFBrXBWx/BPP+gynnTKyJBU6cZfD3idmkA8Dqxhql4U +j56HoWpQ3NeaTq8Fs6ZxlJxxs1BgCscTnTgHhgKo6ahpJhiQq0ywTyOrOk+E2N/O +n+Fpb7vXQtdrROTHre5tQV9yWnEIN7N5ZaRZoJQ39wAvDcKSctrQOHLbFKhFxF0q +fbe01sTurM0TRLfJK91DACX6YblpalgjEbenM49WdVn1zSnXRrcKK2W200JvFbK4 +e/vv6V1T1TRaJwIDAQABo4G9MIG6MB8GA1UdIwQYMBaAFKFyXyYbKJhDlV0HN9WF +lp1L0sNFMB0GA1UdDgQWBBStvZh6NLQm9/rEJlTvA73gJMtUGjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhCAQEEBAMCAQIwRAYDVR0f +BD0wOzA5oDegNYYzaHR0cDovL2NybC51c2VydHJ1c3QuY29tL1VUTi1VU0VSRmly +c3QtSGFyZHdhcmUuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQByQhANOs4kClrwF8BW +onvUOGCSjRK52zYZgDXYNjDtmr5rJ6NyPFDNn+JxkLpjYetIFMTbSRe679Bt8m7a +gIAoQYFQtxMuyLnJegB2aEbQiIxh/tC21UcFF7ktdnDoTlA6w3pLuvunaI84Of3o +2YBrhzkTbCfaYk5JRlTpudW9DkUkHBsyx3nknPKnplkIGaK0jgn8E0n+SFabYaHk +I9LroYT/+JtLefh9lgBdAgVv0UPbzoGfuDsrk/Zh+UrgbLFpHoVnElhzbkh64Z0X +OGaJunQc68cCZu5HTn/aK7fBGMcVflRCXLVEQpU9PIAdGA8Ynvg684t8GMaKsRl1 +jIGZ +-----END CERTIFICATE----- + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN-USERFirst-Hardware +-----BEGIN CERTIFICATE----- +MIIEdDCCA1ygAwIBAgIQRL4Mi1AAJLQR0zYq/mUK/TANBgkqhkiG9w0BAQUFADCB +lzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3Qt +SGFyZHdhcmUwHhcNOTkwNzA5MTgxMDQyWhcNMTkwNzA5MTgxOTIyWjCBlzELMAkG +A1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2UgQ2l0eTEe +MBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExhodHRwOi8v +d3d3LnVzZXJ0cnVzdC5jb20xHzAdBgNVBAMTFlVUTi1VU0VSRmlyc3QtSGFyZHdh +cmUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx98M4P7Sof885glFn +0G2f0v9Y8+efK+wNiVSZuTiZFvfgIXlIwrthdBKWHTxqctU8EGc6Oe0rE81m65UJ +M6Rsl7HoxuzBdXmcRl6Nq9Bq/bkqVRcQVLMZ8Jr28bFdtqdt++BxF2uiiPsA3/4a +MXcMmgF6sTLjKwEHOG7DpV4jvEWbe1DByTCP2+UretNb+zNAHqDVmBe8i4fDidNd +oI6yqqr2jmmIBsX6iSHzCJ1pLgkzmykNRg+MzEk0sGlRvfkGzWitZky8PqxhvQqI +DsjfPe58BEydCl5rkdbux+0ojatNh4lz0G6k0B4WixThdkQDf2Os5M1JnMWS9Ksy +oUhbAgMBAAGjgbkwgbYwCwYDVR0PBAQDAgHGMA8GA1UdEwEB/wQFMAMBAf8wHQYD +VR0OBBYEFKFyXyYbKJhDlV0HN9WFlp1L0sNFMEQGA1UdHwQ9MDswOaA3oDWGM2h0 +dHA6Ly9jcmwudXNlcnRydXN0LmNvbS9VVE4tVVNFUkZpcnN0LUhhcmR3YXJlLmNy +bDAxBgNVHSUEKjAoBggrBgEFBQcDAQYIKwYBBQUHAwUGCCsGAQUFBwMGBggrBgEF +BQcDBzANBgkqhkiG9w0BAQUFAAOCAQEARxkP3nTGmZev/K0oXnWO6y1n7k57K9cM +//bey1WiCuFMVGWTYGufEpytXoMs61quwOQt9ABjHbjAbPLPSbtNk28Gpgoiskli +CE7/yMgUsogWXecB5BKV5UU0s4tpvc+0hY91UZ59Ojg6FEgSxvunOxqNDYJAB+gE +CJChicsZUN/KHAG8HQQZexB2lzvukJDKxA4fFm517zP4029bHpbj4HR3dHuKom4t +3XbWOTCC8KucUvIqx69JXn7HaOWCgchqJ/kniCrVWFCVH/A7HFe7fRQ5YiuayZSS +KqMiDP+JJn1fIytH1xUdqWqeUQ0qUZ6B+dQ7XnASfxAynB67nfhmqA== +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/ssl_cert.pem b/src/greentest/2.7pypy/ssl_cert.pem new file mode 100644 index 0000000..47a7d7e --- /dev/null +++ b/src/greentest/2.7pypy/ssl_cert.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/2.7pypy/ssl_key.passwd.pem b/src/greentest/2.7pypy/ssl_key.passwd.pem new file mode 100644 index 0000000..2524672 --- /dev/null +++ b/src/greentest/2.7pypy/ssl_key.passwd.pem @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- diff --git a/src/greentest/2.7pypy/ssl_key.pem b/src/greentest/2.7pypy/ssl_key.pem new file mode 100644 index 0000000..3fd3bbd --- /dev/null +++ b/src/greentest/2.7pypy/ssl_key.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- diff --git a/src/greentest/2.7pypy/subprocessdata/sigchild_ignore.py b/src/greentest/2.7pypy/subprocessdata/sigchild_ignore.py new file mode 100644 index 0000000..86320fb --- /dev/null +++ b/src/greentest/2.7pypy/subprocessdata/sigchild_ignore.py @@ -0,0 +1,15 @@ +import signal, subprocess, sys, time +# On Linux this causes os.waitpid to fail with OSError as the OS has already +# reaped our child process. The wait() passing the OSError on to the caller +# and causing us to exit with an error is what we are testing against. +signal.signal(signal.SIGCHLD, signal.SIG_IGN) +subprocess.Popen([sys.executable, '-c', 'print("albatross")']).wait() +# Also ensure poll() handles an errno.ECHILD appropriately. +p = subprocess.Popen([sys.executable, '-c', 'print("albatross")']) +num_polls = 0 +while p.poll() is None: + # Waiting for the process to finish. + time.sleep(0.01) # Avoid being a CPU busy loop. + num_polls += 1 + if num_polls > 3000: + raise RuntimeError('poll should have returned 0 within 30 seconds') diff --git a/src/greentest/2.7pypy/test_asyncore.py b/src/greentest/2.7pypy/test_asyncore.py new file mode 100644 index 0000000..20eceb6 --- /dev/null +++ b/src/greentest/2.7pypy/test_asyncore.py @@ -0,0 +1,744 @@ +import asyncore +import unittest +import select +import os +import socket +import sys +import time +import warnings +import errno +import struct + +from test import test_support +from test.test_support import TESTFN, run_unittest, unlink, HOST +from StringIO import StringIO + +try: + import threading +except ImportError: + threading = None + + +class dummysocket: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def fileno(self): + return 42 + +class dummychannel: + def __init__(self): + self.socket = dummysocket() + + def close(self): + self.socket.close() + +class exitingdummy: + def __init__(self): + pass + + def handle_read_event(self): + raise asyncore.ExitNow() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + +class crashingdummy: + def __init__(self): + self.error_handled = False + + def handle_read_event(self): + raise Exception() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + + def handle_error(self): + self.error_handled = True + +# used when testing senders; just collects what it gets until newline is sent +def capture_server(evt, buf, serv): + try: + serv.listen(5) + conn, addr = serv.accept() + except socket.timeout: + pass + else: + n = 200 + while n > 0: + r, w, e = select.select([conn], [], []) + if r: + data = conn.recv(10) + # keep everything except for the newline terminator + buf.write(data.replace('\n', '')) + if '\n' in data: + break + n -= 1 + time.sleep(0.01) + + conn.close() + finally: + serv.close() + evt.set() + + +class HelperFunctionTests(unittest.TestCase): + def test_readwriteexc(self): + # Check exception handling behavior of read, write and _exception + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore read/write/_exception calls + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.read, tr1) + self.assertRaises(asyncore.ExitNow, asyncore.write, tr1) + self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + asyncore.read(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore.write(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore._exception(tr2) + self.assertEqual(tr2.error_handled, True) + + # asyncore.readwrite uses constants in the select module that + # are not present in Windows systems (see this thread: + # http://mail.python.org/pipermail/python-list/2001-October/109973.html) + # These constants should be present as long as poll is available + + @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') + def test_readwrite(self): + # Check that correct methods are called by readwrite() + + attributes = ('read', 'expt', 'write', 'closed', 'error_handled') + + expected = ( + (select.POLLIN, 'read'), + (select.POLLPRI, 'expt'), + (select.POLLOUT, 'write'), + (select.POLLERR, 'closed'), + (select.POLLHUP, 'closed'), + (select.POLLNVAL, 'closed'), + ) + + class testobj: + def __init__(self): + self.read = False + self.write = False + self.closed = False + self.expt = False + self.error_handled = False + + def handle_read_event(self): + self.read = True + + def handle_write_event(self): + self.write = True + + def handle_close(self): + self.closed = True + + def handle_expt_event(self): + self.expt = True + + def handle_error(self): + self.error_handled = True + + for flag, expectedattr in expected: + tobj = testobj() + self.assertEqual(getattr(tobj, expectedattr), False) + asyncore.readwrite(tobj, flag) + + # Only the attribute modified by the routine we expect to be + # called should be True. + for attr in attributes: + self.assertEqual(getattr(tobj, attr), attr==expectedattr) + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore readwrite call + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + self.assertEqual(tr2.error_handled, False) + asyncore.readwrite(tr2, flag) + self.assertEqual(tr2.error_handled, True) + + def test_closeall(self): + self.closeall_check(False) + + def test_closeall_default(self): + self.closeall_check(True) + + def closeall_check(self, usedefault): + # Check that close_all() closes everything in a given map + + l = [] + testmap = {} + for i in range(10): + c = dummychannel() + l.append(c) + self.assertEqual(c.socket.closed, False) + testmap[i] = c + + if usedefault: + socketmap = asyncore.socket_map + try: + asyncore.socket_map = testmap + asyncore.close_all() + finally: + testmap, asyncore.socket_map = asyncore.socket_map, socketmap + else: + asyncore.close_all(testmap) + + self.assertEqual(len(testmap), 0) + + for c in l: + self.assertEqual(c.socket.closed, True) + + def test_compact_traceback(self): + try: + raise Exception("I don't like spam!") + except: + real_t, real_v, real_tb = sys.exc_info() + r = asyncore.compact_traceback() + else: + self.fail("Expected exception") + + (f, function, line), t, v, info = r + self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py') + self.assertEqual(function, 'test_compact_traceback') + self.assertEqual(t, real_t) + self.assertEqual(v, real_v) + self.assertEqual(info, '[%s|%s|%s]' % (f, function, line)) + + +class DispatcherTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + def test_basic(self): + d = asyncore.dispatcher() + self.assertEqual(d.readable(), True) + self.assertEqual(d.writable(), True) + + def test_repr(self): + d = asyncore.dispatcher() + self.assertEqual(repr(d), '' % id(d)) + + def test_log(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log() (to stderr) + fp = StringIO() + stderr = sys.stderr + l1 = "Lovely spam! Wonderful spam!" + l2 = "I don't like spam!" + try: + sys.stderr = fp + d.log(l1) + d.log(l2) + finally: + sys.stderr = stderr + + lines = fp.getvalue().splitlines() + self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2]) + + def test_log_info(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log_info() (to stdout via print) + fp = StringIO() + stdout = sys.stdout + l1 = "Have you got anything without spam?" + l2 = "Why can't she have egg bacon spam and sausage?" + l3 = "THAT'S got spam in it!" + try: + sys.stdout = fp + d.log_info(l1, 'EGGS') + d.log_info(l2) + d.log_info(l3, 'SPAM') + finally: + sys.stdout = stdout + + lines = fp.getvalue().splitlines() + expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3] + + self.assertEqual(lines, expected) + + def test_unhandled(self): + d = asyncore.dispatcher() + d.ignore_log_types = () + + # capture output of dispatcher.log_info() (to stdout via print) + fp = StringIO() + stdout = sys.stdout + try: + sys.stdout = fp + d.handle_expt() + d.handle_read() + d.handle_write() + d.handle_connect() + d.handle_accept() + finally: + sys.stdout = stdout + + lines = fp.getvalue().splitlines() + expected = ['warning: unhandled incoming priority event', + 'warning: unhandled read event', + 'warning: unhandled write event', + 'warning: unhandled connect event', + 'warning: unhandled accept event'] + self.assertEqual(lines, expected) + + def test_issue_8594(self): + # XXX - this test is supposed to be removed in next major Python + # version + d = asyncore.dispatcher(socket.socket()) + # make sure the error message no longer refers to the socket + # object but the dispatcher instance instead + self.assertRaisesRegexp(AttributeError, 'dispatcher instance', + getattr, d, 'foo') + # cheap inheritance with the underlying socket is supposed + # to still work but a DeprecationWarning is expected + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + family = d.family + self.assertEqual(family, socket.AF_INET) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[0].category, DeprecationWarning)) + + def test_strerror(self): + # refers to bug #8573 + err = asyncore._strerror(errno.EPERM) + if hasattr(os, 'strerror'): + self.assertEqual(err, os.strerror(errno.EPERM)) + err = asyncore._strerror(-1) + self.assertTrue(err != "") + + +class dispatcherwithsend_noread(asyncore.dispatcher_with_send): + def readable(self): + return False + + def handle_connect(self): + pass + +class DispatcherWithSendTests(unittest.TestCase): + usepoll = False + + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + @unittest.skipUnless(threading, 'Threading required for this test.') + @test_support.reap_threads + def test_send(self): + evt = threading.Event() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(3) + port = test_support.bind_port(sock) + + cap = StringIO() + args = (evt, cap, sock) + t = threading.Thread(target=capture_server, args=args) + t.start() + try: + # wait a little longer for the server to initialize (it sometimes + # refuses connections on slow machines without this wait) + time.sleep(0.2) + + data = "Suppose there isn't a 16-ton weight?" + d = dispatcherwithsend_noread() + d.create_socket(socket.AF_INET, socket.SOCK_STREAM) + d.connect((HOST, port)) + + # give time for socket to connect + time.sleep(0.1) + + d.send(data) + d.send(data) + d.send('\n') + + n = 1000 + while d.out_buffer and n > 0: + asyncore.poll() + n -= 1 + + evt.wait() + + self.assertEqual(cap.getvalue(), data*2) + finally: + t.join() + + +class DispatcherWithSendTests_UsePoll(DispatcherWithSendTests): + usepoll = True + +@unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), + 'asyncore.file_wrapper required') +class FileWrapperTest(unittest.TestCase): + def setUp(self): + self.d = "It's not dead, it's sleeping!" + with file(TESTFN, 'w') as h: + h.write(self.d) + + def tearDown(self): + unlink(TESTFN) + + def test_recv(self): + fd = os.open(TESTFN, os.O_RDONLY) + w = asyncore.file_wrapper(fd) + os.close(fd) + + self.assertNotEqual(w.fd, fd) + self.assertNotEqual(w.fileno(), fd) + self.assertEqual(w.recv(13), "It's not dead") + self.assertEqual(w.read(6), ", it's") + w.close() + self.assertRaises(OSError, w.read, 1) + + + def test_send(self): + d1 = "Come again?" + d2 = "I want to buy some cheese." + fd = os.open(TESTFN, os.O_WRONLY | os.O_APPEND) + w = asyncore.file_wrapper(fd) + os.close(fd) + + w.write(d1) + w.send(d2) + w.close() + self.assertEqual(file(TESTFN).read(), self.d + d1 + d2) + + @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'), + 'asyncore.file_dispatcher required') + def test_dispatcher(self): + fd = os.open(TESTFN, os.O_RDONLY) + data = [] + class FileDispatcher(asyncore.file_dispatcher): + def handle_read(self): + data.append(self.recv(29)) + s = FileDispatcher(fd) + os.close(fd) + asyncore.loop(timeout=0.01, use_poll=True, count=2) + self.assertEqual(b"".join(data), self.d) + + +class BaseTestHandler(asyncore.dispatcher): + + def __init__(self, sock=None): + asyncore.dispatcher.__init__(self, sock) + self.flag = False + + def handle_accept(self): + raise Exception("handle_accept not supposed to be called") + + def handle_connect(self): + raise Exception("handle_connect not supposed to be called") + + def handle_expt(self): + raise Exception("handle_expt not supposed to be called") + + def handle_close(self): + raise Exception("handle_close not supposed to be called") + + def handle_error(self): + raise + + +class TCPServer(asyncore.dispatcher): + """A server which listens on an address and dispatches the + connection to a handler. + """ + + def __init__(self, handler=BaseTestHandler, host=HOST, port=0): + asyncore.dispatcher.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.set_reuse_addr() + self.bind((host, port)) + self.listen(5) + self.handler = handler + + @property + def address(self): + return self.socket.getsockname()[:2] + + def handle_accept(self): + pair = self.accept() + if pair is not None: + self.handler(pair[0]) + + def handle_error(self): + raise + + +class BaseClient(BaseTestHandler): + + def __init__(self, address): + BaseTestHandler.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.connect(address) + + def handle_connect(self): + pass + + +class BaseTestAPI(unittest.TestCase): + + def tearDown(self): + asyncore.close_all() + + def loop_waiting_for_flag(self, instance, timeout=5): + timeout = float(timeout) / 100 + count = 100 + while asyncore.socket_map and count > 0: + asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) + if instance.flag: + return + count -= 1 + time.sleep(timeout) + self.fail("flag not set") + + def test_handle_connect(self): + # make sure handle_connect is called on connect() + + class TestClient(BaseClient): + def handle_connect(self): + self.flag = True + + server = TCPServer() + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_handle_accept(self): + # make sure handle_accept() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self): + BaseTestHandler.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.bind((HOST, 0)) + self.listen(5) + self.address = self.socket.getsockname()[:2] + + def handle_accept(self): + self.flag = True + + server = TestListener() + client = BaseClient(server.address) + self.loop_waiting_for_flag(server) + + def test_handle_read(self): + # make sure handle_read is called on data received + + class TestClient(BaseClient): + def handle_read(self): + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.send('x' * 1024) + + server = TCPServer(TestHandler) + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_handle_write(self): + # make sure handle_write is called + + class TestClient(BaseClient): + def handle_write(self): + self.flag = True + + server = TCPServer() + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close(self): + # make sure handle_close is called when the other end closes + # the connection + + class TestClient(BaseClient): + + def handle_read(self): + # in order to make handle_close be called we are supposed + # to make at least one recv() call + self.recv(1024) + + def handle_close(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.close() + + server = TCPServer(TestHandler) + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + @unittest.skipIf(sys.platform.startswith("sunos"), + "OOB support is broken on Solaris") + def test_handle_expt(self): + # Make sure handle_expt is called on OOB data received. + # Note: this might fail on some platforms as OOB data is + # tenuously supported and rarely used. + + class TestClient(BaseClient): + def handle_expt(self): + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.socket.send(chr(244), socket.MSG_OOB) + + server = TCPServer(TestHandler) + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_handle_error(self): + + class TestClient(BaseClient): + def handle_write(self): + 1.0 / 0 + def handle_error(self): + self.flag = True + try: + raise + except ZeroDivisionError: + pass + else: + raise Exception("exception not raised") + + server = TCPServer() + client = TestClient(server.address) + self.loop_waiting_for_flag(client) + + def test_connection_attributes(self): + server = TCPServer() + client = BaseClient(server.address) + + # we start disconnected + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + # this can't be taken for granted across all platforms + #self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # execute some loops so that client connects to server + asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100) + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertTrue(client.connected) + self.assertFalse(client.accepting) + + # disconnect the client + client.close() + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # stop serving + server.close() + self.assertFalse(server.connected) + self.assertFalse(server.accepting) + + def test_create_socket(self): + s = asyncore.dispatcher() + s.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.assertEqual(s.socket.family, socket.AF_INET) + self.assertEqual(s.socket.type, socket.SOCK_STREAM) + + def test_bind(self): + s1 = asyncore.dispatcher() + s1.create_socket(socket.AF_INET, socket.SOCK_STREAM) + s1.bind((HOST, 0)) + s1.listen(5) + port = s1.socket.getsockname()[1] + + s2 = asyncore.dispatcher() + s2.create_socket(socket.AF_INET, socket.SOCK_STREAM) + # EADDRINUSE indicates the socket was correctly bound + self.assertRaises(socket.error, s2.bind, (HOST, port)) + + def test_set_reuse_addr(self): + sock = socket.socket() + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except socket.error: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket()) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.create_socket(socket.AF_INET, socket.SOCK_STREAM) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + finally: + sock.close() + + @unittest.skipUnless(threading, 'Threading required for this test.') + @test_support.reap_threads + def test_quick_connect(self): + # see: http://bugs.python.org/issue10340 + server = TCPServer() + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, count=500)) + t.start() + self.addCleanup(t.join) + + for x in xrange(20): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + try: + s.connect(server.address) + except socket.error: + pass + finally: + s.close() + + +class TestAPI_UseSelect(BaseTestAPI): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UsePoll(BaseTestAPI): + use_poll = True + + +def test_main(): + tests = [HelperFunctionTests, DispatcherTests, DispatcherWithSendTests, + DispatcherWithSendTests_UsePoll, TestAPI_UseSelect, + TestAPI_UsePoll, FileWrapperTest] + run_unittest(*tests) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_ftplib.py b/src/greentest/2.7pypy/test_ftplib.py new file mode 100644 index 0000000..cc1c19b --- /dev/null +++ b/src/greentest/2.7pypy/test_ftplib.py @@ -0,0 +1,865 @@ +"""Test script for ftplib module.""" + +# Modified by Giampaolo Rodola' to test FTP class, IPv6 and TLS +# environment + +import ftplib +import asyncore +import asynchat +import socket +import StringIO +import errno +import os +try: + import ssl +except ImportError: + ssl = None + +from unittest import TestCase, SkipTest, skipUnless +from test import test_support +from test.test_support import HOST, HOSTv6 +threading = test_support.import_module('threading') + +TIMEOUT = 3 +# the dummy data returned by server over the data channel when +# RETR, LIST and NLST commands are issued +RETR_DATA = 'abcde12345\r\n' * 1000 +LIST_DATA = 'foo\r\nbar\r\n' +NLST_DATA = 'foo\r\nbar\r\n' + + +class DummyDTPHandler(asynchat.async_chat): + dtp_conn_closed = False + + def __init__(self, conn, baseclass): + asynchat.async_chat.__init__(self, conn) + self.baseclass = baseclass + self.baseclass.last_received_data = '' + + def handle_read(self): + self.baseclass.last_received_data += self.recv(1024) + + def handle_close(self): + # XXX: this method can be called many times in a row for a single + # connection, including in clear-text (non-TLS) mode. + # (behaviour witnessed with test_data_connection) + if not self.dtp_conn_closed: + self.baseclass.push('226 transfer complete') + self.close() + self.dtp_conn_closed = True + + def handle_error(self): + raise + + +class DummyFTPHandler(asynchat.async_chat): + + dtp_handler = DummyDTPHandler + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + self.set_terminator("\r\n") + self.in_buffer = [] + self.dtp = None + self.last_received_cmd = None + self.last_received_data = '' + self.next_response = '' + self.rest = None + self.next_retr_data = RETR_DATA + self.push('220 welcome') + + def collect_incoming_data(self, data): + self.in_buffer.append(data) + + def found_terminator(self): + line = ''.join(self.in_buffer) + self.in_buffer = [] + if self.next_response: + self.push(self.next_response) + self.next_response = '' + cmd = line.split(' ')[0].lower() + self.last_received_cmd = cmd + space = line.find(' ') + if space != -1: + arg = line[space + 1:] + else: + arg = "" + if hasattr(self, 'cmd_' + cmd): + method = getattr(self, 'cmd_' + cmd) + method(arg) + else: + self.push('550 command "%s" not understood.' %cmd) + + def handle_error(self): + raise + + def push(self, data): + asynchat.async_chat.push(self, data + '\r\n') + + def cmd_port(self, arg): + addr = map(int, arg.split(',')) + ip = '%d.%d.%d.%d' %tuple(addr[:4]) + port = (addr[4] * 256) + addr[5] + s = socket.create_connection((ip, port), timeout=10) + self.dtp = self.dtp_handler(s, baseclass=self) + self.push('200 active data connection established') + + def cmd_pasv(self, arg): + sock = socket.socket() + sock.bind((self.socket.getsockname()[0], 0)) + sock.listen(5) + sock.settimeout(10) + ip, port = sock.getsockname()[:2] + ip = ip.replace('.', ',') + p1, p2 = divmod(port, 256) + self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2)) + conn, addr = sock.accept() + self.dtp = self.dtp_handler(conn, baseclass=self) + + def cmd_eprt(self, arg): + af, ip, port = arg.split(arg[0])[1:-1] + port = int(port) + s = socket.create_connection((ip, port), timeout=10) + self.dtp = self.dtp_handler(s, baseclass=self) + self.push('200 active data connection established') + + def cmd_epsv(self, arg): + sock = socket.socket(socket.AF_INET6) + sock.bind((self.socket.getsockname()[0], 0)) + sock.listen(5) + sock.settimeout(10) + port = sock.getsockname()[1] + self.push('229 entering extended passive mode (|||%d|)' %port) + conn, addr = sock.accept() + self.dtp = self.dtp_handler(conn, baseclass=self) + + def cmd_echo(self, arg): + # sends back the received string (used by the test suite) + self.push(arg) + + def cmd_user(self, arg): + self.push('331 username ok') + + def cmd_pass(self, arg): + self.push('230 password ok') + + def cmd_acct(self, arg): + self.push('230 acct ok') + + def cmd_rnfr(self, arg): + self.push('350 rnfr ok') + + def cmd_rnto(self, arg): + self.push('250 rnto ok') + + def cmd_dele(self, arg): + self.push('250 dele ok') + + def cmd_cwd(self, arg): + self.push('250 cwd ok') + + def cmd_size(self, arg): + self.push('250 1000') + + def cmd_mkd(self, arg): + self.push('257 "%s"' %arg) + + def cmd_rmd(self, arg): + self.push('250 rmd ok') + + def cmd_pwd(self, arg): + self.push('257 "pwd ok"') + + def cmd_type(self, arg): + self.push('200 type ok') + + def cmd_quit(self, arg): + self.push('221 quit ok') + self.close() + + def cmd_stor(self, arg): + self.push('125 stor ok') + + def cmd_rest(self, arg): + self.rest = arg + self.push('350 rest ok') + + def cmd_retr(self, arg): + self.push('125 retr ok') + if self.rest is not None: + offset = int(self.rest) + else: + offset = 0 + self.dtp.push(self.next_retr_data[offset:]) + self.dtp.close_when_done() + self.rest = None + + def cmd_list(self, arg): + self.push('125 list ok') + self.dtp.push(LIST_DATA) + self.dtp.close_when_done() + + def cmd_nlst(self, arg): + self.push('125 nlst ok') + self.dtp.push(NLST_DATA) + self.dtp.close_when_done() + + def cmd_setlongretr(self, arg): + # For testing. Next RETR will return long line. + self.next_retr_data = 'x' * int(arg) + self.push('125 setlongretr ok') + + +class DummyFTPServer(asyncore.dispatcher, threading.Thread): + + handler = DummyFTPHandler + + def __init__(self, address, af=socket.AF_INET): + threading.Thread.__init__(self) + asyncore.dispatcher.__init__(self) + self.create_socket(af, socket.SOCK_STREAM) + self.bind(address) + self.listen(5) + self.active = False + self.active_lock = threading.Lock() + self.host, self.port = self.socket.getsockname()[:2] + self.handler_instance = None + + def start(self): + assert not self.active + self.__flag = threading.Event() + threading.Thread.start(self) + self.__flag.wait() + + def run(self): + self.active = True + self.__flag.set() + while self.active and asyncore.socket_map: + self.active_lock.acquire() + asyncore.loop(timeout=0.1, count=1) + self.active_lock.release() + asyncore.close_all(ignore_all=True) + + def stop(self): + assert self.active + self.active = False + self.join() + + def handle_accept(self): + conn, addr = self.accept() + self.handler_instance = self.handler(conn) + + def handle_connect(self): + self.close() + handle_read = handle_connect + + def writable(self): + return 0 + + def handle_error(self): + raise + + +if ssl is not None: + + CERTFILE = os.path.join(os.path.dirname(__file__), "keycert3.pem") + CAFILE = os.path.join(os.path.dirname(__file__), "pycacert.pem") + + class SSLConnection(object, asyncore.dispatcher): + """An asyncore.dispatcher subclass supporting TLS/SSL.""" + + _ssl_accepting = False + _ssl_closing = False + + def secure_connection(self): + socket = ssl.wrap_socket(self.socket, suppress_ragged_eofs=False, + certfile=CERTFILE, server_side=True, + do_handshake_on_connect=False, + ssl_version=ssl.PROTOCOL_SSLv23) + self.del_channel() + self.set_socket(socket) + self._ssl_accepting = True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + elif err.args[0] == ssl.SSL_ERROR_EOF: + return self.handle_close() + raise + except socket.error as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def _do_ssl_shutdown(self): + self._ssl_closing = True + try: + self.socket = self.socket.unwrap() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + except socket.error as err: + # Any "socket error" corresponds to a SSL_ERROR_SYSCALL return + # from OpenSSL's SSL_shutdown(), corresponding to a + # closed socket condition. See also: + # http://www.mail-archive.com/openssl-users@openssl.org/msg60710.html + pass + self._ssl_closing = False + if getattr(self, '_ccc', False) is False: + super(SSLConnection, self).close() + else: + pass + + def handle_read_event(self): + if self._ssl_accepting: + self._do_ssl_handshake() + elif self._ssl_closing: + self._do_ssl_shutdown() + else: + super(SSLConnection, self).handle_read_event() + + def handle_write_event(self): + if self._ssl_accepting: + self._do_ssl_handshake() + elif self._ssl_closing: + self._do_ssl_shutdown() + else: + super(SSLConnection, self).handle_write_event() + + def send(self, data): + try: + return super(SSLConnection, self).send(data) + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN, + ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return 0 + raise + + def recv(self, buffer_size): + try: + return super(SSLConnection, self).recv(buffer_size) + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return b'' + if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN): + self.handle_close() + return b'' + raise + + def handle_error(self): + raise + + def close(self): + if (isinstance(self.socket, ssl.SSLSocket) and + self.socket._sslobj is not None): + self._do_ssl_shutdown() + else: + super(SSLConnection, self).close() + + + class DummyTLS_DTPHandler(SSLConnection, DummyDTPHandler): + """A DummyDTPHandler subclass supporting TLS/SSL.""" + + def __init__(self, conn, baseclass): + DummyDTPHandler.__init__(self, conn, baseclass) + if self.baseclass.secure_data_channel: + self.secure_connection() + + + class DummyTLS_FTPHandler(SSLConnection, DummyFTPHandler): + """A DummyFTPHandler subclass supporting TLS/SSL.""" + + dtp_handler = DummyTLS_DTPHandler + + def __init__(self, conn): + DummyFTPHandler.__init__(self, conn) + self.secure_data_channel = False + + def cmd_auth(self, line): + """Set up secure control channel.""" + self.push('234 AUTH TLS successful') + self.secure_connection() + + def cmd_pbsz(self, line): + """Negotiate size of buffer for secure data transfer. + For TLS/SSL the only valid value for the parameter is '0'. + Any other value is accepted but ignored. + """ + self.push('200 PBSZ=0 successful.') + + def cmd_prot(self, line): + """Setup un/secure data channel.""" + arg = line.upper() + if arg == 'C': + self.push('200 Protection set to Clear') + self.secure_data_channel = False + elif arg == 'P': + self.push('200 Protection set to Private') + self.secure_data_channel = True + else: + self.push("502 Unrecognized PROT type (use C or P).") + + + class DummyTLS_FTPServer(DummyFTPServer): + handler = DummyTLS_FTPHandler + + +class TestFTPClass(TestCase): + + def setUp(self): + self.server = DummyFTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP(timeout=10) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + + def test_getwelcome(self): + self.assertEqual(self.client.getwelcome(), '220 welcome') + + def test_sanitize(self): + self.assertEqual(self.client.sanitize('foo'), repr('foo')) + self.assertEqual(self.client.sanitize('pass 12345'), repr('pass *****')) + self.assertEqual(self.client.sanitize('PASS 12345'), repr('PASS *****')) + + def test_exceptions(self): + self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 400') + self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 499') + self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 500') + self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 599') + self.assertRaises(ftplib.error_proto, self.client.sendcmd, 'echo 999') + + def test_all_errors(self): + exceptions = (ftplib.error_reply, ftplib.error_temp, ftplib.error_perm, + ftplib.error_proto, ftplib.Error, IOError, EOFError) + for x in exceptions: + try: + raise x('exception not included in all_errors set') + except ftplib.all_errors: + pass + + def test_set_pasv(self): + # passive mode is supposed to be enabled by default + self.assertTrue(self.client.passiveserver) + self.client.set_pasv(True) + self.assertTrue(self.client.passiveserver) + self.client.set_pasv(False) + self.assertFalse(self.client.passiveserver) + + def test_voidcmd(self): + self.client.voidcmd('echo 200') + self.client.voidcmd('echo 299') + self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') + self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') + + def test_login(self): + self.client.login() + + def test_acct(self): + self.client.acct('passwd') + + def test_rename(self): + self.client.rename('a', 'b') + self.server.handler_instance.next_response = '200' + self.assertRaises(ftplib.error_reply, self.client.rename, 'a', 'b') + + def test_delete(self): + self.client.delete('foo') + self.server.handler_instance.next_response = '199' + self.assertRaises(ftplib.error_reply, self.client.delete, 'foo') + + def test_size(self): + self.client.size('foo') + + def test_mkd(self): + dir = self.client.mkd('/foo') + self.assertEqual(dir, '/foo') + + def test_rmd(self): + self.client.rmd('foo') + + def test_cwd(self): + dir = self.client.cwd('/foo') + self.assertEqual(dir, '250 cwd ok') + + def test_pwd(self): + dir = self.client.pwd() + self.assertEqual(dir, 'pwd ok') + + def test_quit(self): + self.assertEqual(self.client.quit(), '221 quit ok') + # Ensure the connection gets closed; sock attribute should be None + self.assertEqual(self.client.sock, None) + + def test_retrbinary(self): + received = [] + self.client.retrbinary('retr', received.append) + self.assertEqual(''.join(received), RETR_DATA) + + def test_retrbinary_rest(self): + for rest in (0, 10, 20): + received = [] + self.client.retrbinary('retr', received.append, rest=rest) + self.assertEqual(''.join(received), RETR_DATA[rest:], + msg='rest test case %d %d %d' % (rest, + len(''.join(received)), + len(RETR_DATA[rest:]))) + + def test_retrlines(self): + received = [] + self.client.retrlines('retr', received.append) + self.assertEqual(''.join(received), RETR_DATA.replace('\r\n', '')) + + def test_storbinary(self): + f = StringIO.StringIO(RETR_DATA) + self.client.storbinary('stor', f) + self.assertEqual(self.server.handler_instance.last_received_data, RETR_DATA) + # test new callback arg + flag = [] + f.seek(0) + self.client.storbinary('stor', f, callback=lambda x: flag.append(None)) + self.assertTrue(flag) + + def test_storbinary_rest(self): + f = StringIO.StringIO(RETR_DATA) + for r in (30, '30'): + f.seek(0) + self.client.storbinary('stor', f, rest=r) + self.assertEqual(self.server.handler_instance.rest, str(r)) + + def test_storlines(self): + f = StringIO.StringIO(RETR_DATA.replace('\r\n', '\n')) + self.client.storlines('stor', f) + self.assertEqual(self.server.handler_instance.last_received_data, RETR_DATA) + # test new callback arg + flag = [] + f.seek(0) + self.client.storlines('stor foo', f, callback=lambda x: flag.append(None)) + self.assertTrue(flag) + + def test_nlst(self): + self.client.nlst() + self.assertEqual(self.client.nlst(), NLST_DATA.split('\r\n')[:-1]) + + def test_dir(self): + l = [] + self.client.dir(lambda x: l.append(x)) + self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) + + def test_makeport(self): + self.client.makeport() + # IPv4 is in use, just make sure send_eprt has not been used + self.assertEqual(self.server.handler_instance.last_received_cmd, 'port') + + def test_makepasv(self): + host, port = self.client.makepasv() + conn = socket.create_connection((host, port), 10) + conn.close() + # IPv4 is in use, just make sure send_epsv has not been used + self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv') + + def test_line_too_long(self): + self.assertRaises(ftplib.Error, self.client.sendcmd, + 'x' * self.client.maxline * 2) + + def test_retrlines_too_long(self): + self.client.sendcmd('SETLONGRETR %d' % (self.client.maxline * 2)) + received = [] + self.assertRaises(ftplib.Error, + self.client.retrlines, 'retr', received.append) + + def test_storlines_too_long(self): + f = StringIO.StringIO('x' * self.client.maxline * 2) + self.assertRaises(ftplib.Error, self.client.storlines, 'stor', f) + + +@skipUnless(socket.has_ipv6, "IPv6 not enabled") +class TestIPv6Environment(TestCase): + + @classmethod + def setUpClass(cls): + try: + DummyFTPServer((HOST, 0), af=socket.AF_INET6) + except socket.error: + raise SkipTest("IPv6 not enabled") + + def setUp(self): + self.server = DummyFTPServer((HOSTv6, 0), af=socket.AF_INET6) + self.server.start() + self.client = ftplib.FTP() + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + + def test_af(self): + self.assertEqual(self.client.af, socket.AF_INET6) + + def test_makeport(self): + self.client.makeport() + self.assertEqual(self.server.handler_instance.last_received_cmd, 'eprt') + + def test_makepasv(self): + host, port = self.client.makepasv() + conn = socket.create_connection((host, port), 10) + conn.close() + self.assertEqual(self.server.handler_instance.last_received_cmd, 'epsv') + + def test_transfer(self): + def retr(): + received = [] + self.client.retrbinary('retr', received.append) + self.assertEqual(''.join(received), RETR_DATA) + self.client.set_pasv(True) + retr() + self.client.set_pasv(False) + retr() + + +@skipUnless(ssl, "SSL not available") +class TestTLS_FTPClassMixin(TestFTPClass): + """Repeat TestFTPClass tests starting the TLS layer for both control + and data connections first. + """ + + def setUp(self): + self.server = DummyTLS_FTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP_TLS(timeout=10) + self.client.connect(self.server.host, self.server.port) + # enable TLS + self.client.auth() + self.client.prot_p() + + +@skipUnless(ssl, "SSL not available") +class TestTLS_FTPClass(TestCase): + """Specific TLS_FTP class tests.""" + + def setUp(self): + self.server = DummyTLS_FTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP_TLS(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + + def test_control_connection(self): + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.auth() + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + + def test_data_connection(self): + # clear text + sock = self.client.transfercmd('list') + self.assertNotIsInstance(sock, ssl.SSLSocket) + sock.close() + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + # secured, after PROT P + self.client.prot_p() + sock = self.client.transfercmd('list') + self.assertIsInstance(sock, ssl.SSLSocket) + sock.close() + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + # PROT C is issued, the connection must be in cleartext again + self.client.prot_c() + sock = self.client.transfercmd('list') + self.assertNotIsInstance(sock, ssl.SSLSocket) + sock.close() + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + def test_login(self): + # login() is supposed to implicitly secure the control connection + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.login() + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + # make sure that AUTH TLS doesn't get issued again + self.client.login() + + def test_auth_issued_twice(self): + self.client.auth() + self.assertRaises(ValueError, self.client.auth) + + def test_auth_ssl(self): + try: + self.client.ssl_version = ssl.PROTOCOL_SSLv23 + self.client.auth() + self.assertRaises(ValueError, self.client.auth) + finally: + self.client.ssl_version = ssl.PROTOCOL_TLSv1 + + def test_context(self): + self.client.quit() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE, + context=ctx) + self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, + context=ctx) + self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, + keyfile=CERTFILE, context=ctx) + + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.auth() + self.assertIs(self.client.sock.context, ctx) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + + self.client.prot_p() + sock = self.client.transfercmd('list') + try: + self.assertIs(sock.context, ctx) + self.assertIsInstance(sock, ssl.SSLSocket) + finally: + sock.close() + + def test_check_hostname(self): + self.client.quit() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.check_hostname = True + ctx.load_verify_locations(CAFILE) + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) + + # 127.0.0.1 doesn't match SAN + self.client.connect(self.server.host, self.server.port) + with self.assertRaises(ssl.CertificateError): + self.client.auth() + # exception quits connection + + self.client.connect(self.server.host, self.server.port) + self.client.prot_p() + with self.assertRaises(ssl.CertificateError): + self.client.transfercmd("list").close() + self.client.quit() + + self.client.connect("localhost", self.server.port) + self.client.auth() + self.client.quit() + + self.client.connect("localhost", self.server.port) + self.client.prot_p() + self.client.transfercmd("list").close() + + +class TestTimeouts(TestCase): + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(10) + self.port = test_support.bind_port(self.sock) + threading.Thread(target=self.server, args=(self.evt,self.sock)).start() + # Wait for the server to be ready. + self.evt.wait() + self.evt.clear() + ftplib.FTP.port = self.port + + def tearDown(self): + self.evt.wait() + + def server(self, evt, serv): + # This method sets the evt 3 times: + # 1) when the connection is ready to be accepted. + # 2) when it is safe for the caller to close the connection + # 3) when we have closed the socket + serv.listen(5) + # (1) Signal the caller that we are ready to accept the connection. + evt.set() + try: + conn, addr = serv.accept() + except socket.timeout: + pass + else: + conn.send("1 Hola mundo\n") + # (2) Signal the caller that it is safe to close the socket. + evt.set() + conn.close() + finally: + serv.close() + # (3) Signal the caller that we are done. + evt.set() + + def testTimeoutDefault(self): + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + ftp = ftplib.FTP(HOST) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutNone(self): + # no timeout -- do not use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + ftp = ftplib.FTP(HOST, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(ftp.sock.gettimeout()) + self.evt.wait() + ftp.close() + + def testTimeoutValue(self): + # a value + ftp = ftplib.FTP(HOST, timeout=30) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutConnect(self): + ftp = ftplib.FTP() + ftp.connect(HOST, timeout=30) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutDifferentOrder(self): + ftp = ftplib.FTP(timeout=30) + ftp.connect(HOST) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutDirectAccess(self): + ftp = ftplib.FTP() + ftp.timeout = 30 + ftp.connect(HOST) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + +def test_main(): + tests = [TestFTPClass, TestTimeouts, + TestIPv6Environment, + TestTLS_FTPClassMixin, TestTLS_FTPClass] + + thread_info = test_support.threading_setup() + try: + test_support.run_unittest(*tests) + finally: + test_support.threading_cleanup(*thread_info) + + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7pypy/test_httplib.py b/src/greentest/2.7pypy/test_httplib.py new file mode 100644 index 0000000..7e8b058 --- /dev/null +++ b/src/greentest/2.7pypy/test_httplib.py @@ -0,0 +1,992 @@ +import httplib +import itertools +import array +import StringIO +import socket +import errno +import os +import tempfile + +import unittest +TestCase = unittest.TestCase + +from test import test_support + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') +# Self-signed cert file for self-signed.pythontest.net +CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') + +HOST = test_support.HOST + +class FakeSocket: + def __init__(self, text, fileclass=StringIO.StringIO, host=None, port=None): + self.text = text + self.fileclass = fileclass + self.data = '' + self.file_closed = False + self.host = host + self.port = port + + def sendall(self, data): + self.data += ''.join(data) + + def makefile(self, mode, bufsize=None): + if mode != 'r' and mode != 'rb': + raise httplib.UnimplementedFileMode() + # keep the file around so we can check how much was read from it + self.file = self.fileclass(self.text) + self.file.close = self.file_close #nerf close () + return self.file + + def file_close(self): + self.file_closed = True + + def close(self): + pass + +class EPipeSocket(FakeSocket): + + def __init__(self, text, pipe_trigger): + # When sendall() is called with pipe_trigger, raise EPIPE. + FakeSocket.__init__(self, text) + self.pipe_trigger = pipe_trigger + + def sendall(self, data): + if self.pipe_trigger in data: + raise socket.error(errno.EPIPE, "gotcha") + self.data += data + + def close(self): + pass + +class NoEOFStringIO(StringIO.StringIO): + """Like StringIO, but raises AssertionError on EOF. + + This is used below to test that httplib doesn't try to read + more from the underlying file than it should. + """ + def read(self, n=-1): + data = StringIO.StringIO.read(self, n) + if data == '': + raise AssertionError('caller tried to read past EOF') + return data + + def readline(self, length=None): + data = StringIO.StringIO.readline(self, length) + if data == '': + raise AssertionError('caller tried to read past EOF') + return data + + +class HeaderTests(TestCase): + def test_auto_headers(self): + # Some headers are added automatically, but should not be added by + # .request() if they are explicitly set. + + class HeaderCountingBuffer(list): + def __init__(self): + self.count = {} + def append(self, item): + kv = item.split(':') + if len(kv) > 1: + # item is a 'Key: Value' header string + lcKey = kv[0].lower() + self.count.setdefault(lcKey, 0) + self.count[lcKey] += 1 + list.append(self, item) + + for explicit_header in True, False: + for header in 'Content-length', 'Host', 'Accept-encoding': + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket('blahblahblah') + conn._buffer = HeaderCountingBuffer() + + body = 'spamspamspam' + headers = {} + if explicit_header: + headers[header] = str(len(body)) + conn.request('POST', '/', body, headers) + self.assertEqual(conn._buffer.count[header.lower()], 1) + + def test_content_length_0(self): + + class ContentLengthChecker(list): + def __init__(self): + list.__init__(self) + self.content_length = None + def append(self, item): + kv = item.split(':', 1) + if len(kv) > 1 and kv[0].lower() == 'content-length': + self.content_length = kv[1].strip() + list.append(self, item) + + # Here, we're testing that methods expecting a body get a + # content-length set to zero if the body is empty (either None or '') + bodies = (None, '') + methods_with_body = ('PUT', 'POST', 'PATCH') + for method, body in itertools.product(methods_with_body, bodies): + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', body) + self.assertEqual( + conn._buffer.content_length, '0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # For these methods, we make sure that content-length is not set when + # the body is None because it might cause unexpected behaviour on the + # server. + methods_without_body = ( + 'GET', 'CONNECT', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', + ) + for method in methods_without_body: + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', None) + self.assertEqual( + conn._buffer.content_length, None, + 'Header Content-Length set for empty body on {}'.format(method) + ) + + # If the body is set to '', that's considered to be "present but + # empty" rather than "missing", so content length would be set, even + # for methods that don't expect a body. + for method in methods_without_body: + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', '') + self.assertEqual( + conn._buffer.content_length, '0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # If the body is set, make sure Content-Length is set. + for method in itertools.chain(methods_without_body, methods_with_body): + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', ' ') + self.assertEqual( + conn._buffer.content_length, '1', + 'Header Content-Length incorrect on {}'.format(method) + ) + + def test_putheader(self): + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.putrequest('GET','/') + conn.putheader('Content-length',42) + self.assertIn('Content-length: 42', conn._buffer) + + conn.putheader('Foo', ' bar ') + self.assertIn(b'Foo: bar ', conn._buffer) + conn.putheader('Bar', '\tbaz\t') + self.assertIn(b'Bar: \tbaz\t', conn._buffer) + conn.putheader('Authorization', 'Bearer mytoken') + self.assertIn(b'Authorization: Bearer mytoken', conn._buffer) + conn.putheader('IterHeader', 'IterA', 'IterB') + self.assertIn(b'IterHeader: IterA\r\n\tIterB', conn._buffer) + conn.putheader('LatinHeader', b'\xFF') + self.assertIn(b'LatinHeader: \xFF', conn._buffer) + conn.putheader('Utf8Header', b'\xc3\x80') + self.assertIn(b'Utf8Header: \xc3\x80', conn._buffer) + conn.putheader('C1-Control', b'next\x85line') + self.assertIn(b'C1-Control: next\x85line', conn._buffer) + conn.putheader('Embedded-Fold-Space', 'is\r\n allowed') + self.assertIn(b'Embedded-Fold-Space: is\r\n allowed', conn._buffer) + conn.putheader('Embedded-Fold-Tab', 'is\r\n\tallowed') + self.assertIn(b'Embedded-Fold-Tab: is\r\n\tallowed', conn._buffer) + conn.putheader('Key Space', 'value') + self.assertIn(b'Key Space: value', conn._buffer) + conn.putheader('KeySpace ', 'value') + self.assertIn(b'KeySpace : value', conn._buffer) + conn.putheader(b'Nonbreak\xa0Space', 'value') + self.assertIn(b'Nonbreak\xa0Space: value', conn._buffer) + conn.putheader(b'\xa0NonbreakSpace', 'value') + self.assertIn(b'\xa0NonbreakSpace: value', conn._buffer) + + def test_ipv6host_header(self): + # Default host header on IPv6 transaction should be wrapped by [] if + # it is an IPv6 address + expected = 'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \ + 'Accept-Encoding: identity\r\n\r\n' + conn = httplib.HTTPConnection('[2001::]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + expected = 'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ + 'Accept-Encoding: identity\r\n\r\n' + conn = httplib.HTTPConnection('[2001:102A::]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + def test_malformed_headers_coped_with(self): + # Issue 19996 + body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + + self.assertEqual(resp.getheader('First'), 'val') + self.assertEqual(resp.getheader('Second'), 'val') + + def test_malformed_truncation(self): + # Other malformed header lines, especially without colons, used to + # cause the rest of the header section to be truncated + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'Public-Key-Pins: \n' + b'pin-sha256="xxx=";\n' + b'report-uri="https://..."\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertIsNotNone(resp.getheader('Public-Key-Pins')) + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + def test_blank_line_forms(self): + # Test that both CRLF and LF blank lines can terminate the header + # section and start the body + for blank in (b'\r\n', b'\n'): + resp = b'HTTP/1.1 200 OK\r\n' b'Transfer-Encoding: chunked\r\n' + resp += blank + resp += b'4\r\nbody\r\n0\r\n\r\n' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + resp = b'HTTP/1.0 200 OK\r\n' + blank + b'body' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'body') + + # A blank line ending in CR is not treated as the end of the HTTP + # header section, therefore header fields following it should be + # parsed if possible + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'\r' + b'Name: value\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + # No header fields nor blank line + resp = b'HTTP/1.0 200 OK\r\n' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'') + + def test_from_line(self): + # The parser handles "From" lines specially, so test this does not + # affect parsing the rest of the header section + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'From start\r\n' + b' continued\r\n' + b'Name: value\r\n' + b'From middle\r\n' + b' continued\r\n' + b'Transfer-Encoding: chunked\r\n' + b'From end\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertIsNotNone(resp.getheader('Name')) + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + resp = ( + b'HTTP/1.0 200 OK\r\n' + b'From alone\r\n' + b'\r\n' + b'body' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'body') + + def test_parse_all_octets(self): + # Ensure no valid header field octet breaks the parser + body = ( + b'HTTP/1.1 200 OK\r\n' + b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters + b'VCHAR: ' + bytearray(range(0x21, 0x7E + 1)) + b'\r\n' + b'obs-text: ' + bytearray(range(0x80, 0xFF + 1)) + b'\r\n' + b'obs-fold: text\r\n' + b' folded with space\r\n' + b'\tfolded with tab\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.getheader('Content-Length'), '0') + self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value') + vchar = ''.join(map(chr, range(0x21, 0x7E + 1))) + self.assertEqual(resp.getheader('VCHAR'), vchar) + self.assertIsNotNone(resp.getheader('obs-text')) + folded = resp.getheader('obs-fold') + self.assertTrue(folded.startswith('text')) + self.assertIn(' folded with space', folded) + self.assertTrue(folded.endswith('folded with tab')) + + def test_invalid_headers(self): + conn = httplib.HTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/') + + # http://tools.ietf.org/html/rfc7230#section-3.2.4, whitespace is no + # longer allowed in header names + cases = ( + (b'Invalid\r\nName', b'ValidValue'), + (b'Invalid\rName', b'ValidValue'), + (b'Invalid\nName', b'ValidValue'), + (b'\r\nInvalidName', b'ValidValue'), + (b'\rInvalidName', b'ValidValue'), + (b'\nInvalidName', b'ValidValue'), + (b' InvalidName', b'ValidValue'), + (b'\tInvalidName', b'ValidValue'), + (b'Invalid:Name', b'ValidValue'), + (b':InvalidName', b'ValidValue'), + (b'ValidName', b'Invalid\r\nValue'), + (b'ValidName', b'Invalid\rValue'), + (b'ValidName', b'Invalid\nValue'), + (b'ValidName', b'InvalidValue\r\n'), + (b'ValidName', b'InvalidValue\r'), + (b'ValidName', b'InvalidValue\n'), + ) + for name, value in cases: + with self.assertRaisesRegexp(ValueError, 'Invalid header'): + conn.putheader(name, value) + + +class BasicTest(TestCase): + def test_status_lines(self): + # Test HTTP status lines + + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(0), '') # Issue #20007 + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(), 'Text') + self.assertTrue(resp.isclosed()) + + body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + self.assertRaises(httplib.BadStatusLine, resp.begin) + + def test_bad_status_repr(self): + exc = httplib.BadStatusLine('') + self.assertEqual(repr(exc), '''BadStatusLine("\'\'",)''') + + def test_partial_reads(self): + # if we have a length, the system knows when to close itself + # same behaviour than when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), 'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), 'xt') + self.assertTrue(resp.isclosed()) + + def test_partial_reads_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), 'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), 'xt') + self.assertEqual(resp.read(1), '') + self.assertTrue(resp.isclosed()) + + def test_partial_reads_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), 'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), 'xt') + self.assertEqual(resp.read(1), '') + self.assertTrue(resp.isclosed()) + + def test_host_port(self): + # Check invalid host_port + + # Note that httplib does not accept user:password@ in the host-port. + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(httplib.InvalidURL, httplib.HTTP, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", "fe80::207:e9ff:fe9b", + 8000), + ("www.python.org:80", "www.python.org", 80), + ("www.python.org", "www.python.org", 80), + ("www.python.org:", "www.python.org", 80), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80)): + http = httplib.HTTP(hp) + c = http._conn + if h != c.host: + self.fail("Host incorrectly parsed: %s != %s" % (h, c.host)) + if p != c.port: + self.fail("Port incorrectly parsed: %s != %s" % (p, c.host)) + + def test_response_headers(self): + # test response with multiple message headers with the same field name. + text = ('HTTP/1.1 200 OK\r\n' + 'Set-Cookie: Customer="WILE_E_COYOTE";' + ' Version="1"; Path="/acme"\r\n' + 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' + ' Path="/acme"\r\n' + '\r\n' + 'No body\r\n') + hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"' + ', ' + 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"') + s = FakeSocket(text) + r = httplib.HTTPResponse(s) + r.begin() + cookies = r.getheader("Set-Cookie") + if cookies != hdr: + self.fail("multiple headers not combined properly") + + def test_read_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFStringIO) + resp = httplib.HTTPResponse(sock, method="HEAD") + resp.begin() + if resp.read() != "": + self.fail("Did not expect response from HEAD request") + + def test_too_many_headers(self): + headers = '\r\n'.join('Header%d: foo' % i for i in xrange(200)) + '\r\n' + text = ('HTTP/1.1 200 OK\r\n' + headers) + s = FakeSocket(text) + r = httplib.HTTPResponse(s) + self.assertRaises(httplib.HTTPException, r.begin) + + def test_send_file(self): + expected = 'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + 'Accept-Encoding: identity\r\nContent-Length:' + + body = open(__file__, 'rb') + conn = httplib.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertTrue(sock.data.startswith(expected)) + self.assertIn('def test_send_file', sock.data) + + def test_send_tempfile(self): + expected = ('GET /foo HTTP/1.1\r\nHost: example.com\r\n' + 'Accept-Encoding: identity\r\nContent-Length: 9\r\n\r\n' + 'fake\ndata') + + with tempfile.TemporaryFile() as body: + body.write('fake\ndata') + body.seek(0) + + conn = httplib.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertEqual(sock.data, expected) + + def test_send(self): + expected = 'this is a test this is only a test' + conn = httplib.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + conn.send(expected) + self.assertEqual(expected, sock.data) + sock.data = '' + conn.send(array.array('c', expected)) + self.assertEqual(expected, sock.data) + sock.data = '' + conn.send(StringIO.StringIO(expected)) + self.assertEqual(expected, sock.data) + + def test_chunked(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + '0\r\n') + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), 'hello world') + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except httplib.IncompleteRead, i: + self.assertEqual(i.partial, 'hello world') + self.assertEqual(repr(i),'IncompleteRead(11 bytes read)') + self.assertEqual(str(i),'IncompleteRead(11 bytes read)') + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + '0\r\n') + resp = httplib.HTTPResponse(sock, method="HEAD") + resp.begin() + self.assertEqual(resp.read(), '') + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + + def test_negative_content_length(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\n' + 'Content-Length: -1\r\n\r\nHello\r\n') + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), 'Hello\r\n') + self.assertTrue(resp.isclosed()) + + def test_incomplete_read(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') + resp = httplib.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except httplib.IncompleteRead as i: + self.assertEqual(i.partial, 'Hello\r\n') + self.assertEqual(repr(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertEqual(str(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertTrue(resp.isclosed()) + else: + self.fail('IncompleteRead expected') + + def test_epipe(self): + sock = EPipeSocket( + "HTTP/1.0 401 Authorization Required\r\n" + "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"example\"\r\n", + b"Content-Length") + conn = httplib.HTTPConnection("example.com") + conn.sock = sock + self.assertRaises(socket.error, + lambda: conn.request("PUT", "/url", "body")) + resp = conn.getresponse() + self.assertEqual(401, resp.status) + self.assertEqual("Basic realm=\"example\"", + resp.getheader("www-authenticate")) + + def test_filenoattr(self): + # Just test the fileno attribute in the HTTPResponse Object. + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + self.assertTrue(hasattr(resp,'fileno'), + 'HTTPResponse should expose a fileno attribute') + + # Test lines overflowing the max line size (_MAXLINE in httplib) + + def test_overflowing_status_line(self): + self.skipTest("disabled for HTTP 0.9 support") + body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n" + resp = httplib.HTTPResponse(FakeSocket(body)) + self.assertRaises((httplib.LineTooLong, httplib.BadStatusLine), resp.begin) + + def test_overflowing_header_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(body)) + self.assertRaises(httplib.LineTooLong, resp.begin) + + def test_overflowing_chunked_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + + '0' * 65536 + 'a\r\n' + 'hello world\r\n' + '0\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertRaises(httplib.LineTooLong, resp.read) + + def test_early_eof(self): + # Test httpresponse with no \r\n termination, + body = "HTTP/1.1 200 Ok" + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(), '') + self.assertTrue(resp.isclosed()) + + def test_error_leak(self): + # Test that the socket is not leaked if getresponse() fails + conn = httplib.HTTPConnection('example.com') + response = [] + class Response(httplib.HTTPResponse): + def __init__(self, *pos, **kw): + response.append(self) # Avoid garbage collector closing the socket + httplib.HTTPResponse.__init__(self, *pos, **kw) + conn.response_class = Response + conn.sock = FakeSocket('') # Emulate server dropping connection + conn.request('GET', '/') + self.assertRaises(httplib.BadStatusLine, conn.getresponse) + self.assertTrue(response) + #self.assertTrue(response[0].closed) + self.assertTrue(conn.sock.file_closed) + + def test_proxy_tunnel_without_status_line(self): + # Issue 17849: If a proxy tunnel is created that does not return + # a status code, fail. + body = 'hello world' + conn = httplib.HTTPConnection('example.com', strict=False) + conn.set_tunnel('foo') + conn.sock = FakeSocket(body) + with self.assertRaisesRegexp(socket.error, "Invalid response"): + conn._tunnel() + +class OfflineTest(TestCase): + def test_responses(self): + self.assertEqual(httplib.responses[httplib.NOT_FOUND], "Not Found") + + +class TestServerMixin: + """A limited socket server mixin. + + This is used by test cases for testing http connection end points. + """ + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = test_support.bind_port(self.serv) + self.source_port = test_support.find_unused_port() + self.serv.listen(5) + self.conn = None + + def tearDown(self): + if self.conn: + self.conn.close() + self.conn = None + self.serv.close() + self.serv = None + +class SourceAddressTest(TestServerMixin, TestCase): + def testHTTPConnectionSourceAddress(self): + self.conn = httplib.HTTPConnection(HOST, self.port, + source_address=('', self.source_port)) + self.conn.connect() + self.assertEqual(self.conn.sock.getsockname()[1], self.source_port) + + @unittest.skipIf(not hasattr(httplib, 'HTTPSConnection'), + 'httplib.HTTPSConnection not defined') + def testHTTPSConnectionSourceAddress(self): + self.conn = httplib.HTTPSConnection(HOST, self.port, + source_address=('', self.source_port)) + # We don't test anything here other than the constructor not barfing as + # this code doesn't deal with setting up an active running SSL server + # for an ssl_wrapped connect() to actually return from. + + +class HTTPTest(TestServerMixin, TestCase): + def testHTTPConnection(self): + self.conn = httplib.HTTP(host=HOST, port=self.port, strict=None) + self.conn.connect() + self.assertEqual(self.conn._conn.host, HOST) + self.assertEqual(self.conn._conn.port, self.port) + + def testHTTPWithConnectHostPort(self): + testhost = 'unreachable.test.domain' + testport = '80' + self.conn = httplib.HTTP(host=testhost, port=testport) + self.conn.connect(host=HOST, port=self.port) + self.assertNotEqual(self.conn._conn.host, testhost) + self.assertNotEqual(self.conn._conn.port, testport) + self.assertEqual(self.conn._conn.host, HOST) + self.assertEqual(self.conn._conn.port, self.port) + + +class TimeoutTest(TestCase): + PORT = None + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + TimeoutTest.PORT = test_support.bind_port(self.serv) + self.serv.listen(5) + + def tearDown(self): + self.serv.close() + self.serv = None + + def testTimeoutAttribute(self): + '''This will prove that the timeout gets through + HTTPConnection and into the socket. + ''' + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = httplib.HTTPConnection(HOST, TimeoutTest.PORT) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + # no timeout -- do not use global socket default + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = httplib.HTTPConnection(HOST, TimeoutTest.PORT, + timeout=None) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), None) + httpConn.close() + + # a value + httpConn = httplib.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30) + httpConn.connect() + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + +class HTTPSTest(TestCase): + + def setUp(self): + if not hasattr(httplib, 'HTTPSConnection'): + self.skipTest('ssl support required') + + def make_server(self, certfile): + from test.ssl_servers import make_https_server + return make_https_server(self, certfile=certfile) + + def test_attributes(self): + # simple test to check it's storing the timeout + h = httplib.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) + self.assertEqual(h.timeout, 30) + + def test_networked(self): + # Default settings: requires a valid cert from a trusted CA + import ssl + test_support.requires('network') + with test_support.transient_internet('self-signed.pythontest.net'): + h = httplib.HTTPSConnection('self-signed.pythontest.net', 443) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_networked_noverification(self): + # Switch off cert verification + import ssl + test_support.requires('network') + with test_support.transient_internet('self-signed.pythontest.net'): + context = ssl._create_stdlib_context() + h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, + context=context) + h.request('GET', '/') + resp = h.getresponse() + self.assertIn('nginx', resp.getheader('server')) + + @test_support.system_must_validate_cert + def test_networked_trusted_by_default_cert(self): + # Default settings: requires a valid cert from a trusted CA + test_support.requires('network') + with test_support.transient_internet('www.python.org'): + h = httplib.HTTPSConnection('www.python.org', 443) + h.request('GET', '/') + resp = h.getresponse() + content_type = resp.getheader('content-type') + self.assertIn('text/html', content_type) + + def test_networked_good_cert(self): + # We feed the server's cert as a validating cert + import ssl + test_support.requires('network') + with test_support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_selfsigned_pythontestdotnet) + h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h.request('GET', '/') + resp = h.getresponse() + server_string = resp.getheader('server') + self.assertIn('nginx', server_string) + + def test_networked_bad_cert(self): + # We feed a "CA" cert that is unrelated to the server's cert + import ssl + test_support.requires('network') + with test_support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = httplib.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_unknown_cert(self): + # The custom cert isn't known to the default trust bundle + import ssl + server = self.make_server(CERT_localhost) + h = httplib.HTTPSConnection('localhost', server.port) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_good_hostname(self): + # The (valid) cert validates the HTTP hostname + import ssl + server = self.make_server(CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = httplib.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + + def test_local_bad_hostname(self): + # The (valid) cert doesn't validate the HTTP hostname + import ssl + server = self.make_server(CERT_fakehostname) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(CERT_fakehostname) + h = httplib.HTTPSConnection('localhost', server.port, context=context) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + h.close() + # With context.check_hostname=False, the mismatching is ignored + context.check_hostname = False + h = httplib.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(httplib.InvalidURL, httplib.HTTPSConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:443", "www.python.org", 443), + ("www.python.org:", "www.python.org", 443), + ("www.python.org", "www.python.org", 443), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 443), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", + 443)): + c = httplib.HTTPSConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + +class TunnelTests(TestCase): + def test_connect(self): + response_text = ( + 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n' # Reply to HEAD + 'Content-Length: 42\r\n\r\n' + ) + + def create_connection(address, timeout=None, source_address=None): + return FakeSocket(response_text, host=address[0], port=address[1]) + + conn = httplib.HTTPConnection('proxy.com') + conn._create_connection = create_connection + + # Once connected, we should not be able to tunnel anymore + conn.connect() + self.assertRaises(RuntimeError, conn.set_tunnel, 'destination.com') + + # But if close the connection, we are good. + conn.close() + conn.set_tunnel('destination.com') + conn.request('HEAD', '/', '') + + self.assertEqual(conn.sock.host, 'proxy.com') + self.assertEqual(conn.sock.port, 80) + self.assertIn('CONNECT destination.com', conn.sock.data) + # issue22095 + self.assertNotIn('Host: destination.com:None', conn.sock.data) + self.assertIn('Host: destination.com', conn.sock.data) + + self.assertNotIn('Host: proxy.com', conn.sock.data) + + conn.close() + + conn.request('PUT', '/', '') + self.assertEqual(conn.sock.host, 'proxy.com') + self.assertEqual(conn.sock.port, 80) + self.assertTrue('CONNECT destination.com' in conn.sock.data) + self.assertTrue('Host: destination.com' in conn.sock.data) + + +@test_support.reap_threads +def test_main(verbose=None): + test_support.run_unittest(HeaderTests, OfflineTest, BasicTest, TimeoutTest, + HTTPTest, HTTPSTest, SourceAddressTest, + TunnelTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7pypy/test_httpservers.py b/src/greentest/2.7pypy/test_httpservers.py new file mode 100644 index 0000000..6e0ddfd --- /dev/null +++ b/src/greentest/2.7pypy/test_httpservers.py @@ -0,0 +1,694 @@ +"""Unittests for the various HTTPServer modules. + +Written by Cody A.W. Somerville , +Josip Dzolonga, and Michael Otteneder for the 2007/08 GHOP contest. +""" + +import os +import sys +import re +import base64 +import ntpath +import shutil +import urllib +import httplib +import tempfile +import unittest +import CGIHTTPServer + + +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +from SimpleHTTPServer import SimpleHTTPRequestHandler +from CGIHTTPServer import CGIHTTPRequestHandler +from StringIO import StringIO +from test import test_support + + +threading = test_support.import_module('threading') + + +class NoLogRequestHandler: + def log_message(self, *args): + # don't write log messages to stderr + pass + +class SocketlessRequestHandler(SimpleHTTPRequestHandler): + def __init__(self): + self.get_called = False + self.protocol_version = "HTTP/1.1" + + def do_GET(self): + self.get_called = True + self.send_response(200) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(b'Data\r\n') + + def log_message(self, fmt, *args): + pass + + +class TestServerThread(threading.Thread): + def __init__(self, test_object, request_handler): + threading.Thread.__init__(self) + self.request_handler = request_handler + self.test_object = test_object + + def run(self): + self.server = HTTPServer(('', 0), self.request_handler) + self.test_object.PORT = self.server.socket.getsockname()[1] + self.test_object.server_started.set() + self.test_object = None + try: + self.server.serve_forever(0.05) + finally: + self.server.server_close() + + def stop(self): + self.server.shutdown() + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test_support.threading_setup() + os.environ = test_support.EnvironmentVarGuard() + self.server_started = threading.Event() + self.thread = TestServerThread(self, self.request_handler) + self.thread.start() + self.server_started.wait() + + def tearDown(self): + self.thread.stop() + os.environ.__exit__() + test_support.threading_cleanup(*self._threads) + + def request(self, uri, method='GET', body=None, headers={}): + self.connection = httplib.HTTPConnection('localhost', self.PORT) + self.connection.request(method, uri, body, headers) + return self.connection.getresponse() + +class BaseHTTPRequestHandlerTestCase(unittest.TestCase): + """Test the functionality of the BaseHTTPServer focussing on + BaseHTTPRequestHandler. + """ + + HTTPResponseMatch = re.compile('HTTP/1.[0-9]+ 200 OK') + + def setUp (self): + self.handler = SocketlessRequestHandler() + + def send_typical_request(self, message): + input_msg = StringIO(message) + output = StringIO() + self.handler.rfile = input_msg + self.handler.wfile = output + self.handler.handle_one_request() + output.seek(0) + return output.readlines() + + def verify_get_called(self): + self.assertTrue(self.handler.get_called) + + def verify_expected_headers(self, headers): + for fieldName in 'Server: ', 'Date: ', 'Content-Type: ': + self.assertEqual(sum(h.startswith(fieldName) for h in headers), 1) + + def verify_http_server_response(self, response): + match = self.HTTPResponseMatch.search(response) + self.assertIsNotNone(match) + + def test_http_1_1(self): + result = self.send_typical_request('GET / HTTP/1.1\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], 'Data\r\n') + + def test_http_1_0(self): + result = self.send_typical_request('GET / HTTP/1.0\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], 'Data\r\n') + + def test_http_0_9(self): + result = self.send_typical_request('GET / HTTP/0.9\r\n\r\n') + self.assertEqual(len(result), 1) + self.assertEqual(result[0], 'Data\r\n') + self.verify_get_called() + + def test_with_continue_1_0(self): + result = self.send_typical_request('GET / HTTP/1.0\r\nExpect: 100-continue\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], 'Data\r\n') + + def test_request_length(self): + # Issue #10714: huge request lines are discarded, to avoid Denial + # of Service attacks. + result = self.send_typical_request(b'GET ' + b'x' * 65537) + self.assertEqual(result[0], b'HTTP/1.1 414 Request-URI Too Long\r\n') + self.assertFalse(self.handler.get_called) + + +class BaseHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + default_request_version = 'HTTP/1.1' + + def do_TEST(self): + self.send_response(204) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'close') + self.end_headers() + + def do_KEEP(self): + self.send_response(204) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'keep-alive') + self.end_headers() + + def do_KEYERROR(self): + self.send_error(999) + + def do_CUSTOM(self): + self.send_response(999) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'close') + self.end_headers() + + def do_SEND_ERROR(self): + self.send_error(int(self.path[1:])) + + def do_HEAD(self): + self.send_error(int(self.path[1:])) + + def setUp(self): + BaseTestCase.setUp(self) + self.con = httplib.HTTPConnection('localhost', self.PORT) + self.con.connect() + + def test_command(self): + self.con.request('GET', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_request_line_trimming(self): + self.con._http_vsn_str = 'HTTP/1.1\n' + self.con.putrequest('XYZBOGUS', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_version_bogus(self): + self.con._http_vsn_str = 'FUBAR' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 400) + + def test_version_digits(self): + self.con._http_vsn_str = 'HTTP/9.9.9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 400) + + def test_version_none_get(self): + self.con._http_vsn_str = '' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_version_none(self): + # Test that a valid method is rejected when not HTTP/1.x + self.con._http_vsn_str = '' + self.con.putrequest('CUSTOM', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 400) + + def test_version_invalid(self): + self.con._http_vsn = 99 + self.con._http_vsn_str = 'HTTP/9.9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 505) + + def test_send_blank(self): + self.con._http_vsn_str = '' + self.con.putrequest('', '') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 400) + + def test_header_close(self): + self.con.putrequest('GET', '/') + self.con.putheader('Connection', 'close') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_head_keep_alive(self): + self.con._http_vsn_str = 'HTTP/1.1' + self.con.putrequest('GET', '/') + self.con.putheader('Connection', 'keep-alive') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, 501) + + def test_handler(self): + self.con.request('TEST', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 204) + + def test_return_header_keep_alive(self): + self.con.request('KEEP', '/') + res = self.con.getresponse() + self.assertEqual(res.getheader('Connection'), 'keep-alive') + self.con.request('TEST', '/') + self.addCleanup(self.con.close) + + def test_internal_key_error(self): + self.con.request('KEYERROR', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 999) + + def test_return_custom_status(self): + self.con.request('CUSTOM', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 999) + + def test_send_error(self): + allow_transfer_encoding_codes = (205, 304) + for code in (101, 102, 204, 205, 304): + self.con.request('SEND_ERROR', '/{}'.format(code)) + res = self.con.getresponse() + self.assertEqual(code, res.status) + self.assertEqual(None, res.getheader('Content-Length')) + self.assertEqual(None, res.getheader('Content-Type')) + if code not in allow_transfer_encoding_codes: + self.assertEqual(None, res.getheader('Transfer-Encoding')) + + data = res.read() + self.assertEqual(b'', data) + + def test_head_via_send_error(self): + allow_transfer_encoding_codes = (205, 304) + for code in (101, 200, 204, 205, 304): + self.con.request('HEAD', '/{}'.format(code)) + res = self.con.getresponse() + self.assertEqual(code, res.status) + if code == 200: + self.assertEqual(None, res.getheader('Content-Length')) + self.assertIn('text/html', res.getheader('Content-Type')) + else: + self.assertEqual(None, res.getheader('Content-Length')) + self.assertEqual(None, res.getheader('Content-Type')) + if code not in allow_transfer_encoding_codes: + self.assertEqual(None, res.getheader('Transfer-Encoding')) + + data = res.read() + self.assertEqual(b'', data) + + +class SimpleHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, SimpleHTTPRequestHandler): + pass + + def setUp(self): + BaseTestCase.setUp(self) + self.cwd = os.getcwd() + basetempdir = tempfile.gettempdir() + os.chdir(basetempdir) + self.data = 'We are the knights who say Ni!' + self.tempdir = tempfile.mkdtemp(dir=basetempdir) + self.tempdir_name = os.path.basename(self.tempdir) + self.base_url = '/' + self.tempdir_name + temp = open(os.path.join(self.tempdir, 'test'), 'wb') + temp.write(self.data) + temp.close() + + def tearDown(self): + try: + os.chdir(self.cwd) + try: + shutil.rmtree(self.tempdir) + except OSError: + pass + finally: + BaseTestCase.tearDown(self) + + def check_status_and_reason(self, response, status, data=None): + body = response.read() + self.assertTrue(response) + self.assertEqual(response.status, status) + self.assertIsNotNone(response.reason) + if data: + self.assertEqual(data, body) + + def test_get(self): + #constructs the path relative to the root directory of the HTTPServer + response = self.request(self.base_url + '/test') + self.check_status_and_reason(response, 200, data=self.data) + # check for trailing "/" which should return 404. See Issue17324 + response = self.request(self.base_url + '/test/') + self.check_status_and_reason(response, 404) + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, 200) + response = self.request(self.base_url) + self.check_status_and_reason(response, 301) + response = self.request(self.base_url + '/?hi=2') + self.check_status_and_reason(response, 200) + response = self.request(self.base_url + '?hi=1') + self.check_status_and_reason(response, 301) + self.assertEqual(response.getheader("Location"), + self.base_url + "/?hi=1") + response = self.request('/ThisDoesNotExist') + self.check_status_and_reason(response, 404) + response = self.request('/' + 'ThisDoesNotExist' + '/') + self.check_status_and_reason(response, 404) + with open(os.path.join(self.tempdir_name, 'index.html'), 'w') as fp: + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, 200) + # chmod() doesn't work as expected on Windows, and filesystem + # permissions are ignored by root on Unix. + if os.name == 'posix' and os.geteuid() != 0: + os.chmod(self.tempdir, 0) + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, 404) + os.chmod(self.tempdir, 0755) + + def test_head(self): + response = self.request( + self.base_url + '/test', method='HEAD') + self.check_status_and_reason(response, 200) + self.assertEqual(response.getheader('content-length'), + str(len(self.data))) + self.assertEqual(response.getheader('content-type'), + 'application/octet-stream') + + def test_invalid_requests(self): + response = self.request('/', method='FOO') + self.check_status_and_reason(response, 501) + # requests must be case sensitive,so this should fail too + response = self.request('/', method='custom') + self.check_status_and_reason(response, 501) + response = self.request('/', method='GETs') + self.check_status_and_reason(response, 501) + + def test_path_without_leading_slash(self): + response = self.request(self.tempdir_name + '/test') + self.check_status_and_reason(response, 200, data=self.data) + response = self.request(self.tempdir_name + '/test/') + self.check_status_and_reason(response, 404) + response = self.request(self.tempdir_name + '/') + self.check_status_and_reason(response, 200) + response = self.request(self.tempdir_name) + self.check_status_and_reason(response, 301) + response = self.request(self.tempdir_name + '/?hi=2') + self.check_status_and_reason(response, 200) + response = self.request(self.tempdir_name + '?hi=1') + self.check_status_and_reason(response, 301) + self.assertEqual(response.getheader("Location"), + self.tempdir_name + "/?hi=1") + + +cgi_file1 = """\ +#!%s + +print "Content-type: text/html" +print +print "Hello World" +""" + +cgi_file2 = """\ +#!%s +import cgi + +print "Content-type: text/html" +print + +form = cgi.FieldStorage() +print "%%s, %%s, %%s" %% (form.getfirst("spam"), form.getfirst("eggs"), + form.getfirst("bacon")) +""" + +cgi_file4 = """\ +#!%s +import os + +print("Content-type: text/html") +print("") + +print(os.environ["%s"]) +""" + + +@unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, + "This test can't be run reliably as root (issue #13308).") +class CGIHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): + pass + + def setUp(self): + BaseTestCase.setUp(self) + self.parent_dir = tempfile.mkdtemp() + self.cgi_dir = os.path.join(self.parent_dir, 'cgi-bin') + self.cgi_child_dir = os.path.join(self.cgi_dir, 'child-dir') + os.mkdir(self.cgi_dir) + os.mkdir(self.cgi_child_dir) + + # The shebang line should be pure ASCII: use symlink if possible. + # See issue #7668. + if hasattr(os, 'symlink'): + self.pythonexe = os.path.join(self.parent_dir, 'python') + os.symlink(sys.executable, self.pythonexe) + else: + self.pythonexe = sys.executable + + self.nocgi_path = os.path.join(self.parent_dir, 'nocgi.py') + with open(self.nocgi_path, 'w') as fp: + fp.write(cgi_file1 % self.pythonexe) + os.chmod(self.nocgi_path, 0777) + + self.file1_path = os.path.join(self.cgi_dir, 'file1.py') + with open(self.file1_path, 'w') as file1: + file1.write(cgi_file1 % self.pythonexe) + os.chmod(self.file1_path, 0777) + + self.file2_path = os.path.join(self.cgi_dir, 'file2.py') + with open(self.file2_path, 'w') as file2: + file2.write(cgi_file2 % self.pythonexe) + os.chmod(self.file2_path, 0777) + + self.file3_path = os.path.join(self.cgi_child_dir, 'file3.py') + with open(self.file3_path, 'w') as file3: + file3.write(cgi_file1 % self.pythonexe) + os.chmod(self.file3_path, 0777) + + self.file4_path = os.path.join(self.cgi_dir, 'file4.py') + with open(self.file4_path, 'w') as file4: + file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING')) + os.chmod(self.file4_path, 0o777) + + self.cwd = os.getcwd() + os.chdir(self.parent_dir) + + def tearDown(self): + try: + os.chdir(self.cwd) + if self.pythonexe != sys.executable: + os.remove(self.pythonexe) + os.remove(self.nocgi_path) + os.remove(self.file1_path) + os.remove(self.file2_path) + os.remove(self.file3_path) + os.remove(self.file4_path) + os.rmdir(self.cgi_child_dir) + os.rmdir(self.cgi_dir) + os.rmdir(self.parent_dir) + finally: + BaseTestCase.tearDown(self) + + def test_url_collapse_path(self): + # verify tail is the last portion and head is the rest on proper urls + test_vectors = { + '': '//', + '..': IndexError, + '/.//..': IndexError, + '/': '//', + '//': '//', + '/\\': '//\\', + '/.//': '//', + 'cgi-bin/file1.py': '/cgi-bin/file1.py', + '/cgi-bin/file1.py': '/cgi-bin/file1.py', + 'a': '//a', + '/a': '//a', + '//a': '//a', + './a': '//a', + './C:/': '/C:/', + '/a/b': '/a/b', + '/a/b/': '/a/b/', + '/a/b/.': '/a/b/', + '/a/b/c/..': '/a/b/', + '/a/b/c/../d': '/a/b/d', + '/a/b/c/../d/e/../f': '/a/b/d/f', + '/a/b/c/../d/e/../../f': '/a/b/f', + '/a/b/c/../d/e/.././././..//f': '/a/b/f', + '../a/b/c/../d/e/.././././..//f': IndexError, + '/a/b/c/../d/e/../../../f': '/a/f', + '/a/b/c/../d/e/../../../../f': '//f', + '/a/b/c/../d/e/../../../../../f': IndexError, + '/a/b/c/../d/e/../../../../f/..': '//', + '/a/b/c/../d/e/../../../../f/../.': '//', + } + for path, expected in test_vectors.iteritems(): + if isinstance(expected, type) and issubclass(expected, Exception): + self.assertRaises(expected, + CGIHTTPServer._url_collapse_path, path) + else: + actual = CGIHTTPServer._url_collapse_path(path) + self.assertEqual(expected, actual, + msg='path = %r\nGot: %r\nWanted: %r' % + (path, actual, expected)) + + def test_headers_and_content(self): + res = self.request('/cgi-bin/file1.py') + self.assertEqual(('Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_issue19435(self): + res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh') + self.assertEqual(res.status, 404) + + def test_post(self): + params = urllib.urlencode({'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) + headers = {'Content-type' : 'application/x-www-form-urlencoded'} + res = self.request('/cgi-bin/file2.py', 'POST', params, headers) + + self.assertEqual(res.read(), '1, python, 123456\n') + + def test_invaliduri(self): + res = self.request('/cgi-bin/invalid') + res.read() + self.assertEqual(res.status, 404) + + def test_authorization(self): + headers = {'Authorization' : 'Basic %s' % + base64.b64encode('username:pass')} + res = self.request('/cgi-bin/file1.py', 'GET', headers=headers) + self.assertEqual(('Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_no_leading_slash(self): + # http://bugs.python.org/issue2254 + res = self.request('cgi-bin/file1.py') + self.assertEqual(('Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_os_environ_is_not_altered(self): + signature = "Test CGI Server" + os.environ['SERVER_SOFTWARE'] = signature + res = self.request('/cgi-bin/file1.py') + self.assertEqual((b'Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) + + def test_urlquote_decoding_in_cgi_check(self): + res = self.request('/cgi-bin%2ffile1.py') + self.assertEqual((b'Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_nested_cgi_path_issue21323(self): + res = self.request('/cgi-bin/child-dir/file3.py') + self.assertEqual((b'Hello World\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_query_with_multiple_question_mark(self): + res = self.request('/cgi-bin/file4.py?a=b?c=d') + self.assertEqual( + (b'a=b?c=d\n', 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_query_with_continuous_slashes(self): + res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//') + self.assertEqual( + (b'k=aa%2F%2Fbb&//q//p//=//a//b//\n', + 'text/html', 200), + (res.read(), res.getheader('Content-type'), res.status)) + + +class SimpleHTTPRequestHandlerTestCase(unittest.TestCase): + """ Test url parsing """ + def setUp(self): + self.translated = os.getcwd() + self.translated = os.path.join(self.translated, 'filename') + self.handler = SocketlessRequestHandler() + + def test_query_arguments(self): + path = self.handler.translate_path('/filename') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('/filename?foo=bar') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('/filename?a=b&spam=eggs#zot') + self.assertEqual(path, self.translated) + + def test_start_with_double_slash(self): + path = self.handler.translate_path('//filename') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('//filename?foo=bar') + self.assertEqual(path, self.translated) + + def test_windows_colon(self): + import SimpleHTTPServer + with test_support.swap_attr(SimpleHTTPServer.os, 'path', ntpath): + path = self.handler.translate_path('c:c:c:foo/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('\\c:../filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('c:\\c:..\\foo/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('c:c:foo\\c:c:bar/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + +def test_main(verbose=None): + # XXX: gevent: On windows with pypy2, some of these + # tests are incredibly slow or hang in shutdown for unknown + # reasons + import gevent.testing as greentest + MySimpleHTTPRequestHandlerTestCase = SimpleHTTPRequestHandlerTestCase + MySimpleHTTPServerTestCase = SimpleHTTPServerTestCase + MyCGIHTTPServerTestCase = CGIHTTPServerTestCase + if greentest.PYPY and greentest.WIN: + class MySimpleHTTPRequestHandlerTestCase(unittest.TestCase): + def setUp(self): + raise unittest.SkipTest("gevent: Hangs") + def test_empty(self): + return + MySimpleHTTPServerTestCase = MySimpleHTTPRequestHandlerTestCase + MyCGIHTTPServerTestCase = MySimpleHTTPRequestHandlerTestCase + try: + cwd = os.getcwd() + test_support.run_unittest(BaseHTTPRequestHandlerTestCase, + MySimpleHTTPRequestHandlerTestCase, + BaseHTTPServerTestCase, + MySimpleHTTPServerTestCase, + MyCGIHTTPServerTestCase + ) + finally: + os.chdir(cwd) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7pypy/test_queue.py b/src/greentest/2.7pypy/test_queue.py new file mode 100644 index 0000000..34a4aef --- /dev/null +++ b/src/greentest/2.7pypy/test_queue.py @@ -0,0 +1,325 @@ +# Some simple queue module tests, plus some failure conditions +# to ensure the Queue locks remain stable. +import Queue +import time +import unittest +from test import test_support +threading = test_support.import_module('threading') + +QUEUE_SIZE = 5 + +# A thread to run a function that unclogs a blocked Queue. +class _TriggerThread(threading.Thread): + def __init__(self, fn, args): + self.fn = fn + self.args = args + self.startedEvent = threading.Event() + threading.Thread.__init__(self) + + def run(self): + # The sleep isn't necessary, but is intended to give the blocking + # function in the main thread a chance at actually blocking before + # we unclog it. But if the sleep is longer than the timeout-based + # tests wait in their blocking functions, those tests will fail. + # So we give them much longer timeout values compared to the + # sleep here (I aimed at 10 seconds for blocking functions -- + # they should never actually wait that long - they should make + # progress as soon as we call self.fn()). + time.sleep(0.1) + self.startedEvent.set() + self.fn(*self.args) + + +# Execute a function that blocks, and in a separate thread, a function that +# triggers the release. Returns the result of the blocking function. Caution: +# block_func must guarantee to block until trigger_func is called, and +# trigger_func must guarantee to change queue state so that block_func can make +# enough progress to return. In particular, a block_func that just raises an +# exception regardless of whether trigger_func is called will lead to +# timing-dependent sporadic failures, and one of those went rarely seen but +# undiagnosed for years. Now block_func must be unexceptional. If block_func +# is supposed to raise an exception, call do_exceptional_blocking_test() +# instead. + +class BlockingTestMixin: + + def tearDown(self): + self.t = None + + def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): + self.t = _TriggerThread(trigger_func, trigger_args) + self.t.start() + self.result = block_func(*block_args) + # If block_func returned before our thread made the call, we failed! + if not self.t.startedEvent.is_set(): + self.fail("blocking function '%r' appeared not to block" % + block_func) + self.t.join(10) # make sure the thread terminates + if self.t.is_alive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) + return self.result + + # Call this instead if block_func is supposed to raise an exception. + def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, + trigger_args, expected_exception_class): + self.t = _TriggerThread(trigger_func, trigger_args) + self.t.start() + try: + try: + block_func(*block_args) + except expected_exception_class: + raise + else: + self.fail("expected exception of kind %r" % + expected_exception_class) + finally: + self.t.join(10) # make sure the thread terminates + if self.t.is_alive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) + if not self.t.startedEvent.is_set(): + self.fail("trigger thread ended but event never set") + + +class BaseQueueTest(BlockingTestMixin): + def setUp(self): + self.cum = 0 + self.cumlock = threading.Lock() + + def simple_queue_test(self, q): + if not q.empty(): + raise RuntimeError, "Call this function with an empty queue" + # I guess we better check things actually queue correctly a little :) + q.put(111) + q.put(333) + q.put(222) + target_order = dict(Queue = [111, 333, 222], + LifoQueue = [222, 333, 111], + PriorityQueue = [111, 222, 333]) + actual_order = [q.get(), q.get(), q.get()] + self.assertEqual(actual_order, target_order[q.__class__.__name__], + "Didn't seem to queue the correct data!") + for i in range(QUEUE_SIZE-1): + q.put(i) + self.assertTrue(not q.empty(), "Queue should not be empty") + self.assertTrue(not q.full(), "Queue should not be full") + last = 2 * QUEUE_SIZE + full = 3 * 2 * QUEUE_SIZE + q.put(last) + self.assertTrue(q.full(), "Queue should be full") + try: + q.put(full, block=0) + self.fail("Didn't appear to block with a full queue") + except Queue.Full: + pass + try: + q.put(full, timeout=0.01) + self.fail("Didn't appear to time-out with a full queue") + except Queue.Full: + pass + # Test a blocking put + self.do_blocking_test(q.put, (full,), q.get, ()) + self.do_blocking_test(q.put, (full, True, 10), q.get, ()) + # Empty it + for i in range(QUEUE_SIZE): + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + try: + q.get(block=0) + self.fail("Didn't appear to block with an empty queue") + except Queue.Empty: + pass + try: + q.get(timeout=0.01) + self.fail("Didn't appear to time-out with an empty queue") + except Queue.Empty: + pass + # Test a blocking get + self.do_blocking_test(q.get, (), q.put, ('empty',)) + self.do_blocking_test(q.get, (True, 10), q.put, ('empty',)) + + + def worker(self, q): + while True: + x = q.get() + if x is None: + q.task_done() + return + with self.cumlock: + self.cum += x + q.task_done() + + def queue_join_test(self, q): + self.cum = 0 + for i in (0,1): + threading.Thread(target=self.worker, args=(q,)).start() + for i in xrange(100): + q.put(i) + q.join() + self.assertEqual(self.cum, sum(range(100)), + "q.join() did not block until all tasks were done") + for i in (0,1): + q.put(None) # instruct the threads to close + q.join() # verify that you can join twice + + def test_queue_task_done(self): + # Test to make sure a queue task completed successfully. + q = self.type2test() + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + + def test_queue_join(self): + # Test that a queue join()s successfully, and before anything else + # (done twice for insurance). + q = self.type2test() + self.queue_join_test(q) + self.queue_join_test(q) + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + + def test_simple_queue(self): + # Do it a couple of times on the same queue. + # Done twice to make sure works with same instance reused. + q = self.type2test(QUEUE_SIZE) + self.simple_queue_test(q) + self.simple_queue_test(q) + + +class QueueTest(BaseQueueTest, unittest.TestCase): + type2test = Queue.Queue + +class LifoQueueTest(BaseQueueTest, unittest.TestCase): + type2test = Queue.LifoQueue + +class PriorityQueueTest(BaseQueueTest, unittest.TestCase): + type2test = Queue.PriorityQueue + + + +# A Queue subclass that can provoke failure at a moment's notice :) +class FailingQueueException(Exception): + pass + +class FailingQueue(Queue.Queue): + def __init__(self, *args): + self.fail_next_put = False + self.fail_next_get = False + Queue.Queue.__init__(self, *args) + def _put(self, item): + if self.fail_next_put: + self.fail_next_put = False + raise FailingQueueException, "You Lose" + return Queue.Queue._put(self, item) + def _get(self): + if self.fail_next_get: + self.fail_next_get = False + raise FailingQueueException, "You Lose" + return Queue.Queue._get(self) + +class FailingQueueTest(BlockingTestMixin, unittest.TestCase): + + def failing_queue_test(self, q): + if not q.empty(): + raise RuntimeError, "Call this function with an empty queue" + for i in range(QUEUE_SIZE-1): + q.put(i) + # Test a failing non-blocking put. + q.fail_next_put = True + try: + q.put("oops", block=0) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + q.fail_next_put = True + try: + q.put("oops", timeout=0.1) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + q.put("last") + self.assertTrue(q.full(), "Queue should be full") + # Test a failing blocking put + q.fail_next_put = True + try: + self.do_blocking_test(q.put, ("full",), q.get, ()) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # Check the Queue isn't damaged. + # put failed, but get succeeded - re-add + q.put("last") + # Test a failing timeout put + q.fail_next_put = True + try: + self.do_exceptional_blocking_test(q.put, ("full", True, 10), q.get, (), + FailingQueueException) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # Check the Queue isn't damaged. + # put failed, but get succeeded - re-add + q.put("last") + self.assertTrue(q.full(), "Queue should be full") + q.get() + self.assertTrue(not q.full(), "Queue should not be full") + q.put("last") + self.assertTrue(q.full(), "Queue should be full") + # Test a blocking put + self.do_blocking_test(q.put, ("full",), q.get, ()) + # Empty it + for i in range(QUEUE_SIZE): + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + q.put("first") + q.fail_next_get = True + try: + q.get() + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + self.assertTrue(not q.empty(), "Queue should not be empty") + q.fail_next_get = True + try: + q.get(timeout=0.1) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + self.assertTrue(not q.empty(), "Queue should not be empty") + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + q.fail_next_get = True + try: + self.do_exceptional_blocking_test(q.get, (), q.put, ('empty',), + FailingQueueException) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # put succeeded, but get failed. + self.assertTrue(not q.empty(), "Queue should not be empty") + q.get() + self.assertTrue(q.empty(), "Queue should be empty") + + def test_failing_queue(self): + # Test to make sure a queue is functioning correctly. + # Done twice to the same instance. + q = FailingQueue(QUEUE_SIZE) + self.failing_queue_test(q) + self.failing_queue_test(q) + + +def test_main(): + test_support.run_unittest(QueueTest, LifoQueueTest, PriorityQueueTest, + FailingQueueTest) + + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_select.py b/src/greentest/2.7pypy/test_select.py new file mode 100644 index 0000000..e8e9968 --- /dev/null +++ b/src/greentest/2.7pypy/test_select.py @@ -0,0 +1,77 @@ +from test import test_support +import unittest +import select +import os +import sys + +@unittest.skipIf(sys.platform[:3] in ('win', 'os2', 'riscos'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + def test_select(self): + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if test_support.verbose: + print 'timeout =', tout + rfd, wfd, xfd = select.select([p], [], [], tout) + if (rfd, wfd, xfd) == ([], [], []): + continue + if (rfd, wfd, xfd) == ([p], [], []): + line = p.readline() + if test_support.verbose: + print repr(line) + if not line: + if test_support.verbose: + print 'EOF' + break + continue + self.fail('Unexpected return values from select():', rfd, wfd, xfd) + p.close() + + # Issue 16230: Crash on select resized list + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + result = select.select([], a, []) + # CPython: 'a' ends up with 5 items, because each fileno() + # removes an item and at the middle the iteration stops. + # PyPy: 'a' ends up empty, because the iteration is done on + # a copy of the original list: fileno() is called 10 times. + if test_support.check_impl_detail(cpython=True): + self.assertEqual(len(result[1]), 5) + self.assertEqual(len(a), 5) + if test_support.check_impl_detail(pypy=True): + self.assertEqual(len(result[1]), 10) + self.assertEqual(len(a), 0) + +def test_main(): + test_support.run_unittest(SelectTestCase) + test_support.reap_children() + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_signal.py b/src/greentest/2.7pypy/test_signal.py new file mode 100644 index 0000000..7483f64 --- /dev/null +++ b/src/greentest/2.7pypy/test_signal.py @@ -0,0 +1,501 @@ +import unittest +from test import test_support +from contextlib import closing +import gc +import pickle +import select +import signal +import subprocess +import traceback +import sys, os, time, errno + +if sys.platform in ('os2', 'riscos'): + raise unittest.SkipTest("Can't test signal on %s" % sys.platform) + + +class HandlerBCalled(Exception): + pass + + +def exit_subprocess(): + """Use os._exit(0) to exit the current subprocess. + + Otherwise, the test catches the SystemExit and continues executing + in parallel with the original test, so you wind up with an + exponential number of tests running concurrently. + """ + os._exit(0) + + +def ignoring_eintr(__func, *args, **kwargs): + try: + return __func(*args, **kwargs) + except EnvironmentError as e: + if e.errno != errno.EINTR: + raise + return None + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class InterProcessSignalTests(unittest.TestCase): + MAX_DURATION = 20 # Entire test should last at most 20 sec. + + def setUp(self): + self.using_gc = gc.isenabled() + gc.disable() + + def tearDown(self): + if self.using_gc: + gc.enable() + + def format_frame(self, frame, limit=None): + return ''.join(traceback.format_stack(frame, limit=limit)) + + def handlerA(self, signum, frame): + self.a_called = True + if test_support.verbose: + print "handlerA invoked from signal %s at:\n%s" % ( + signum, self.format_frame(frame, limit=1)) + + def handlerB(self, signum, frame): + self.b_called = True + if test_support.verbose: + print "handlerB invoked from signal %s at:\n%s" % ( + signum, self.format_frame(frame, limit=1)) + raise HandlerBCalled(signum, self.format_frame(frame)) + + def wait(self, child): + """Wait for child to finish, ignoring EINTR.""" + while True: + try: + child.wait() + return + except OSError as e: + if e.errno != errno.EINTR: + raise + + def run_test(self): + # Install handlers. This function runs in a sub-process, so we + # don't worry about re-setting the default handlers. + signal.signal(signal.SIGHUP, self.handlerA) + signal.signal(signal.SIGUSR1, self.handlerB) + signal.signal(signal.SIGUSR2, signal.SIG_IGN) + signal.signal(signal.SIGALRM, signal.default_int_handler) + + # Variables the signals will modify: + self.a_called = False + self.b_called = False + + # Let the sub-processes know who to send signals to. + pid = os.getpid() + if test_support.verbose: + print "test runner's pid is", pid + + child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)]) + if child: + self.wait(child) + if not self.a_called: + time.sleep(1) # Give the signal time to be delivered. + self.assertTrue(self.a_called) + self.assertFalse(self.b_called) + self.a_called = False + + # Make sure the signal isn't delivered while the previous + # Popen object is being destroyed, because __del__ swallows + # exceptions. + del child + try: + child = subprocess.Popen(['kill', '-USR1', str(pid)]) + # This wait should be interrupted by the signal's exception. + self.wait(child) + time.sleep(1) # Give the signal time to be delivered. + self.fail('HandlerBCalled exception not raised') + except HandlerBCalled: + self.assertTrue(self.b_called) + self.assertFalse(self.a_called) + if test_support.verbose: + print "HandlerBCalled exception caught" + + child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)]) + if child: + self.wait(child) # Nothing should happen. + + try: + signal.alarm(1) + # The race condition in pause doesn't matter in this case, + # since alarm is going to raise a KeyboardException, which + # will skip the call. + signal.pause() + # But if another signal arrives before the alarm, pause + # may return early. + time.sleep(1) + except KeyboardInterrupt: + if test_support.verbose: + print "KeyboardInterrupt (the alarm() went off)" + except: + self.fail("Some other exception woke us from pause: %s" % + traceback.format_exc()) + else: + self.fail("pause returned of its own accord, and the signal" + " didn't arrive after another second.") + + # Issue 3864. Unknown if this affects earlier versions of freebsd also. + @unittest.skipIf(sys.platform=='freebsd6', + 'inter process signals not reliable (do not mix well with threading) ' + 'on freebsd6') + def test_main(self): + # This function spawns a child process to insulate the main + # test-running process from all the signals. It then + # communicates with that child process over a pipe and + # re-raises information about any exceptions the child + # raises. The real work happens in self.run_test(). + os_done_r, os_done_w = os.pipe() + with closing(os.fdopen(os_done_r)) as done_r, \ + closing(os.fdopen(os_done_w, 'w')) as done_w: + child = os.fork() + if child == 0: + # In the child process; run the test and report results + # through the pipe. + try: + done_r.close() + # Have to close done_w again here because + # exit_subprocess() will skip the enclosing with block. + with closing(done_w): + try: + self.run_test() + except: + pickle.dump(traceback.format_exc(), done_w) + else: + pickle.dump(None, done_w) + except: + print 'Uh oh, raised from pickle.' + traceback.print_exc() + finally: + exit_subprocess() + + done_w.close() + # Block for up to MAX_DURATION seconds for the test to finish. + r, w, x = select.select([done_r], [], [], self.MAX_DURATION) + if done_r in r: + tb = pickle.load(done_r) + if tb: + self.fail(tb) + else: + os.kill(child, signal.SIGKILL) + self.fail('Test deadlocked after %d seconds.' % + self.MAX_DURATION) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class BasicSignalTests(unittest.TestCase): + def trivial_signal_handler(self, *args): + pass + + def test_out_of_range_signal_number_raises_error(self): + self.assertRaises(ValueError, signal.getsignal, 4242) + + self.assertRaises(ValueError, signal.signal, 4242, + self.trivial_signal_handler) + + def test_setting_signal_handler_to_none_raises_error(self): + self.assertRaises(TypeError, signal.signal, + signal.SIGUSR1, None) + + def test_getsignal(self): + hup = signal.signal(signal.SIGHUP, self.trivial_signal_handler) + self.assertEqual(signal.getsignal(signal.SIGHUP), + self.trivial_signal_handler) + signal.signal(signal.SIGHUP, hup) + self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + + +@unittest.skipUnless(sys.platform == "win32", "Windows specific") +class WindowsSignalTests(unittest.TestCase): + def test_issue9324(self): + # Updated for issue #10003, adding SIGBREAK + handler = lambda x, y: None + for sig in (signal.SIGABRT, signal.SIGBREAK, signal.SIGFPE, + signal.SIGILL, signal.SIGINT, signal.SIGSEGV, + signal.SIGTERM): + # Set and then reset a handler for signals that work on windows + signal.signal(sig, signal.signal(sig, handler)) + + with self.assertRaises(ValueError): + signal.signal(-1, handler) + + with self.assertRaises(ValueError): + signal.signal(7, handler) + + +class WakeupFDTests(unittest.TestCase): + + def test_invalid_fd(self): + fd = test_support.make_bad_fd() + self.assertRaises(ValueError, signal.set_wakeup_fd, fd) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class WakeupSignalTests(unittest.TestCase): + TIMEOUT_FULL = 10 + TIMEOUT_HALF = 5 + + def test_wakeup_fd_early(self): + import select + + signal.alarm(1) + before_time = time.time() + # We attempt to get a signal during the sleep, + # before select is called + time.sleep(self.TIMEOUT_FULL) + mid_time = time.time() + self.assertTrue(mid_time - before_time < self.TIMEOUT_HALF) + select.select([self.read], [], [], self.TIMEOUT_FULL) + after_time = time.time() + self.assertTrue(after_time - mid_time < self.TIMEOUT_HALF) + + def test_wakeup_fd_during(self): + import select + + signal.alarm(1) + before_time = time.time() + # We attempt to get a signal during the select call + self.assertRaises(select.error, select.select, + [self.read], [], [], self.TIMEOUT_FULL) + after_time = time.time() + self.assertTrue(after_time - before_time < self.TIMEOUT_HALF) + + def setUp(self): + import fcntl + + self.alrm = signal.signal(signal.SIGALRM, lambda x,y:None) + self.read, self.write = os.pipe() + flags = fcntl.fcntl(self.write, fcntl.F_GETFL, 0) + flags = flags | os.O_NONBLOCK + fcntl.fcntl(self.write, fcntl.F_SETFL, flags) + self.old_wakeup = signal.set_wakeup_fd(self.write) + + def tearDown(self): + signal.set_wakeup_fd(self.old_wakeup) + os.close(self.read) + os.close(self.write) + signal.signal(signal.SIGALRM, self.alrm) + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class SiginterruptTest(unittest.TestCase): + + def setUp(self): + """Install a no-op signal handler that can be set to allow + interrupts or not, and arrange for the original signal handler to be + re-installed when the test is finished. + """ + self.signum = signal.SIGUSR1 + oldhandler = signal.signal(self.signum, lambda x,y: None) + self.addCleanup(signal.signal, self.signum, oldhandler) + + def readpipe_interrupted(self): + """Perform a read during which a signal will arrive. Return True if the + read is interrupted by the signal and raises an exception. Return False + if it returns normally. + """ + # Create a pipe that can be used for the read. Also clean it up + # when the test is over, since nothing else will (but see below for + # the write end). + r, w = os.pipe() + self.addCleanup(os.close, r) + + # Create another process which can send a signal to this one to try + # to interrupt the read. + ppid = os.getpid() + pid = os.fork() + + if pid == 0: + # Child code: sleep to give the parent enough time to enter the + # read() call (there's a race here, but it's really tricky to + # eliminate it); then signal the parent process. Also, sleep + # again to make it likely that the signal is delivered to the + # parent process before the child exits. If the child exits + # first, the write end of the pipe will be closed and the test + # is invalid. + try: + time.sleep(0.2) + os.kill(ppid, self.signum) + time.sleep(0.2) + finally: + # No matter what, just exit as fast as possible now. + exit_subprocess() + else: + # Parent code. + # Make sure the child is eventually reaped, else it'll be a + # zombie for the rest of the test suite run. + self.addCleanup(os.waitpid, pid, 0) + + # Close the write end of the pipe. The child has a copy, so + # it's not really closed until the child exits. We need it to + # close when the child exits so that in the non-interrupt case + # the read eventually completes, otherwise we could just close + # it *after* the test. + os.close(w) + + # Try the read and report whether it is interrupted or not to + # the caller. + try: + d = os.read(r, 1) + return False + except OSError, err: + if err.errno != errno.EINTR: + raise + return True + + def test_without_siginterrupt(self): + """If a signal handler is installed and siginterrupt is not called + at all, when that signal arrives, it interrupts a syscall that's in + progress. + """ + i = self.readpipe_interrupted() + self.assertTrue(i) + # Arrival of the signal shouldn't have changed anything. + i = self.readpipe_interrupted() + self.assertTrue(i) + + def test_siginterrupt_on(self): + """If a signal handler is installed and siginterrupt is called with + a true value for the second argument, when that signal arrives, it + interrupts a syscall that's in progress. + """ + signal.siginterrupt(self.signum, 1) + i = self.readpipe_interrupted() + self.assertTrue(i) + # Arrival of the signal shouldn't have changed anything. + i = self.readpipe_interrupted() + self.assertTrue(i) + + def test_siginterrupt_off(self): + """If a signal handler is installed and siginterrupt is called with + a false value for the second argument, when that signal arrives, it + does not interrupt a syscall that's in progress. + """ + signal.siginterrupt(self.signum, 0) + i = self.readpipe_interrupted() + self.assertFalse(i) + # Arrival of the signal shouldn't have changed anything. + i = self.readpipe_interrupted() + self.assertFalse(i) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class ItimerTest(unittest.TestCase): + def setUp(self): + self.hndl_called = False + self.hndl_count = 0 + self.itimer = None + self.old_alarm = signal.signal(signal.SIGALRM, self.sig_alrm) + + def tearDown(self): + signal.signal(signal.SIGALRM, self.old_alarm) + if self.itimer is not None: # test_itimer_exc doesn't change this attr + # just ensure that itimer is stopped + signal.setitimer(self.itimer, 0) + + def sig_alrm(self, *args): + self.hndl_called = True + if test_support.verbose: + print("SIGALRM handler invoked", args) + + def sig_vtalrm(self, *args): + self.hndl_called = True + + if self.hndl_count > 3: + # it shouldn't be here, because it should have been disabled. + raise signal.ItimerError("setitimer didn't disable ITIMER_VIRTUAL " + "timer.") + elif self.hndl_count == 3: + # disable ITIMER_VIRTUAL, this function shouldn't be called anymore + signal.setitimer(signal.ITIMER_VIRTUAL, 0) + if test_support.verbose: + print("last SIGVTALRM handler call") + + self.hndl_count += 1 + + if test_support.verbose: + print("SIGVTALRM handler invoked", args) + + def sig_prof(self, *args): + self.hndl_called = True + signal.setitimer(signal.ITIMER_PROF, 0) + + if test_support.verbose: + print("SIGPROF handler invoked", args) + + def test_itimer_exc(self): + # XXX I'm assuming -1 is an invalid itimer, but maybe some platform + # defines it ? + self.assertRaises(signal.ItimerError, signal.setitimer, -1, 0) + # Negative times are treated as zero on some platforms. + if 0: + self.assertRaises(signal.ItimerError, + signal.setitimer, signal.ITIMER_REAL, -1) + + def test_itimer_real(self): + self.itimer = signal.ITIMER_REAL + signal.setitimer(self.itimer, 1.0) + if test_support.verbose: + print("\ncall pause()...") + signal.pause() + + self.assertEqual(self.hndl_called, True) + + # Issue 3864. Unknown if this affects earlier versions of freebsd also. + @unittest.skipIf(sys.platform in ('freebsd6', 'netbsd5'), + 'itimer not reliable (does not mix well with threading) on some BSDs.') + def test_itimer_virtual(self): + self.itimer = signal.ITIMER_VIRTUAL + signal.signal(signal.SIGVTALRM, self.sig_vtalrm) + signal.setitimer(self.itimer, 0.3, 0.2) + + start_time = time.time() + while time.time() - start_time < 60.0: + # use up some virtual time by doing real work + _ = pow(12345, 67890, 10000019) + if signal.getitimer(self.itimer) == (0.0, 0.0): + break # sig_vtalrm handler stopped this itimer + else: # Issue 8424 + self.skipTest("timeout: likely cause: machine too slow or load too " + "high") + + # virtual itimer should be (0.0, 0.0) now + self.assertEqual(signal.getitimer(self.itimer), (0.0, 0.0)) + # and the handler should have been called + self.assertEqual(self.hndl_called, True) + + # Issue 3864. Unknown if this affects earlier versions of freebsd also. + @unittest.skipIf(sys.platform=='freebsd6', + 'itimer not reliable (does not mix well with threading) on freebsd6') + def test_itimer_prof(self): + self.itimer = signal.ITIMER_PROF + signal.signal(signal.SIGPROF, self.sig_prof) + signal.setitimer(self.itimer, 0.2, 0.2) + + start_time = time.time() + while time.time() - start_time < 60.0: + # do some work + _ = pow(12345, 67890, 10000019) + if signal.getitimer(self.itimer) == (0.0, 0.0): + break # sig_prof handler stopped this itimer + else: # Issue 8424 + self.skipTest("timeout: likely cause: machine too slow or load too " + "high") + + # profiling itimer should be (0.0, 0.0) now + self.assertEqual(signal.getitimer(self.itimer), (0.0, 0.0)) + # and the handler should have been called + self.assertEqual(self.hndl_called, True) + +def test_main(): + test_support.run_unittest(BasicSignalTests, InterProcessSignalTests, + WakeupFDTests, WakeupSignalTests, + SiginterruptTest, ItimerTest, + WindowsSignalTests) + + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_smtplib.py b/src/greentest/2.7pypy/test_smtplib.py new file mode 100644 index 0000000..1bb6690 --- /dev/null +++ b/src/greentest/2.7pypy/test_smtplib.py @@ -0,0 +1,560 @@ +import asyncore +import email.utils +import socket +import smtpd +import smtplib +import StringIO +import sys +import time +import select + +import unittest +from test import test_support + +try: + import threading +except ImportError: + threading = None + +HOST = test_support.HOST + +def server(evt, buf, serv): + serv.listen(5) + evt.set() + try: + conn, addr = serv.accept() + except socket.timeout: + pass + else: + n = 500 + while buf and n > 0: + r, w, e = select.select([], [conn], []) + if w: + sent = conn.send(buf) + buf = buf[sent:] + + n -= 1 + + conn.close() + finally: + serv.close() + evt.set() + +@unittest.skipUnless(threading, 'Threading required for this test.') +class GeneralTests(unittest.TestCase): + + def setUp(self): + self._threads = test_support.threading_setup() + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = test_support.bind_port(self.sock) + servargs = (self.evt, "220 Hola mundo\n", self.sock) + self.thread = threading.Thread(target=server, args=servargs) + self.thread.start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + self.thread.join() + test_support.threading_cleanup(*self._threads) + + def testBasic1(self): + # connects + smtp = smtplib.SMTP(HOST, self.port) + smtp.close() + + def testBasic2(self): + # connects, include port in host name + smtp = smtplib.SMTP("%s:%s" % (HOST, self.port)) + smtp.close() + + def testLocalHostName(self): + # check that supplied local_hostname is used + smtp = smtplib.SMTP(HOST, self.port, local_hostname="testhost") + self.assertEqual(smtp.local_hostname, "testhost") + smtp.close() + + def testTimeoutDefault(self): + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + smtp = smtplib.SMTP(HOST, self.port) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(smtp.sock.gettimeout(), 30) + smtp.close() + + def testTimeoutNone(self): + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + smtp = smtplib.SMTP(HOST, self.port, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(smtp.sock.gettimeout()) + smtp.close() + + def testTimeoutValue(self): + smtp = smtplib.SMTP(HOST, self.port, timeout=30) + self.assertEqual(smtp.sock.gettimeout(), 30) + smtp.close() + + +# Test server thread using the specified SMTP server class +def debugging_server(serv, serv_evt, client_evt): + serv_evt.set() + + try: + if hasattr(select, 'poll'): + poll_fun = asyncore.poll2 + else: + poll_fun = asyncore.poll + + n = 1000 + while asyncore.socket_map and n > 0: + poll_fun(0.01, asyncore.socket_map) + + # when the client conversation is finished, it will + # set client_evt, and it's then ok to kill the server + if client_evt.is_set(): + serv.close() + break + + n -= 1 + + except socket.timeout: + pass + finally: + if not client_evt.is_set(): + # allow some time for the client to read the result + time.sleep(0.5) + serv.close() + asyncore.close_all() + serv_evt.set() + +MSG_BEGIN = '---------- MESSAGE FOLLOWS ----------\n' +MSG_END = '------------ END MESSAGE ------------\n' + +# NOTE: Some SMTP objects in the tests below are created with a non-default +# local_hostname argument to the constructor, since (on some systems) the FQDN +# lookup caused by the default local_hostname sometimes takes so long that the +# test server times out, causing the test to fail. + +# Test behavior of smtpd.DebuggingServer +@unittest.skipUnless(threading, 'Threading required for this test.') +class DebuggingServerTests(unittest.TestCase): + + def setUp(self): + # temporarily replace sys.stdout to capture DebuggingServer output + self.old_stdout = sys.stdout + self.output = StringIO.StringIO() + sys.stdout = self.output + + self._threads = test_support.threading_setup() + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = smtpd.DebuggingServer((HOST, 0), ('nowhere', -1)) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + self.thread.join() + test_support.threading_cleanup(*self._threads) + # restore sys.stdout + sys.stdout = self.old_stdout + + def testBasic(self): + # connect + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.quit() + + def testNOOP(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + expected = (250, 'Ok') + self.assertEqual(smtp.noop(), expected) + smtp.quit() + + def testRSET(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + expected = (250, 'Ok') + self.assertEqual(smtp.rset(), expected) + smtp.quit() + + def testNotImplemented(self): + # EHLO isn't implemented in DebuggingServer + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + expected = (502, 'Error: command "EHLO" not implemented') + self.assertEqual(smtp.ehlo(), expected) + smtp.quit() + + def testVRFY(self): + # VRFY isn't implemented in DebuggingServer + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + expected = (502, 'Error: command "VRFY" not implemented') + self.assertEqual(smtp.vrfy('nobody@nowhere.com'), expected) + self.assertEqual(smtp.verify('nobody@nowhere.com'), expected) + smtp.quit() + + def testSecondHELO(self): + # check that a second HELO returns a message that it's a duplicate + # (this behavior is specific to smtpd.SMTPChannel) + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.helo() + expected = (503, 'Duplicate HELO/EHLO') + self.assertEqual(smtp.helo(), expected) + smtp.quit() + + def testHELP(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + self.assertEqual(smtp.help(), 'Error: command "HELP" not implemented') + smtp.quit() + + def testSend(self): + # connect and send mail + m = 'A test message' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.sendmail('John', 'Sally', m) + # XXX(nnorwitz): this test is flaky and dies with a bad file descriptor + # in asyncore. This sleep might help, but should really be fixed + # properly by using an Event variable. + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + +class NonConnectingTests(unittest.TestCase): + + def testNotConnected(self): + # Test various operations on an unconnected SMTP object that + # should raise exceptions (at present the attempt in SMTP.send + # to reference the nonexistent 'sock' attribute of the SMTP object + # causes an AttributeError) + smtp = smtplib.SMTP() + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.ehlo) + self.assertRaises(smtplib.SMTPServerDisconnected, + smtp.send, 'test msg') + + def testNonnumericPort(self): + # check that non-numeric port raises socket.error + self.assertRaises(socket.error, smtplib.SMTP, + "localhost", "bogus") + self.assertRaises(socket.error, smtplib.SMTP, + "localhost:bogus") + + +# test response of client to a non-successful HELO message +@unittest.skipUnless(threading, 'Threading required for this test.') +class BadHELOServerTests(unittest.TestCase): + + def setUp(self): + self.old_stdout = sys.stdout + self.output = StringIO.StringIO() + sys.stdout = self.output + + self._threads = test_support.threading_setup() + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = test_support.bind_port(self.sock) + servargs = (self.evt, "199 no hello for you!\n", self.sock) + self.thread = threading.Thread(target=server, args=servargs) + self.thread.start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + self.thread.join() + test_support.threading_cleanup(*self._threads) + sys.stdout = self.old_stdout + + def testFailingHELO(self): + self.assertRaises(smtplib.SMTPConnectError, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class TooLongLineTests(unittest.TestCase): + respdata = '250 OK' + ('.' * smtplib._MAXLINE * 2) + '\n' + + def setUp(self): + self.old_stdout = sys.stdout + self.output = StringIO.StringIO() + sys.stdout = self.output + + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = test_support.bind_port(self.sock) + servargs = (self.evt, self.respdata, self.sock) + threading.Thread(target=server, args=servargs).start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + sys.stdout = self.old_stdout + + def testLineTooLong(self): + self.assertRaises(smtplib.SMTPResponseException, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + +sim_users = {'Mr.A@somewhere.com':'John A', + 'Ms.B@somewhere.com':'Sally B', + 'Mrs.C@somewhereesle.com':'Ruth C', + } + +sim_auth = ('Mr.A@somewhere.com', 'somepassword') +sim_cram_md5_challenge = ('PENCeUxFREJoU0NnbmhNWitOMjNGNn' + 'dAZWx3b29kLmlubm9zb2Z0LmNvbT4=') +sim_auth_credentials = { + 'login': 'TXIuQUBzb21ld2hlcmUuY29t', + 'plain': 'AE1yLkFAc29tZXdoZXJlLmNvbQBzb21lcGFzc3dvcmQ=', + 'cram-md5': ('TXIUQUBZB21LD2HLCMUUY29TIDG4OWQ0MJ' + 'KWZGQ4ODNMNDA4NTGXMDRLZWMYZJDMODG1'), + } +sim_auth_login_password = 'C29TZXBHC3N3B3JK' + +sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'], + 'list-2':['Ms.B@somewhere.com',], + } + +# Simulated SMTP channel & server +class SimSMTPChannel(smtpd.SMTPChannel): + + def __init__(self, extra_features, *args, **kw): + self._extrafeatures = ''.join( + [ "250-{0}\r\n".format(x) for x in extra_features ]) + smtpd.SMTPChannel.__init__(self, *args, **kw) + + def smtp_EHLO(self, arg): + resp = ('250-testhost\r\n' + '250-EXPN\r\n' + '250-SIZE 20000000\r\n' + '250-STARTTLS\r\n' + '250-DELIVERBY\r\n') + resp = resp + self._extrafeatures + '250 HELP' + self.push(resp) + + def smtp_VRFY(self, arg): + # For max compatibility smtplib should be sending the raw address. + if arg in sim_users: + self.push('250 %s %s' % (sim_users[arg], smtplib.quoteaddr(arg))) + else: + self.push('550 No such user: %s' % arg) + + def smtp_EXPN(self, arg): + list_name = arg.lower() + if list_name in sim_lists: + user_list = sim_lists[list_name] + for n, user_email in enumerate(user_list): + quoted_addr = smtplib.quoteaddr(user_email) + if n < len(user_list) - 1: + self.push('250-%s %s' % (sim_users[user_email], quoted_addr)) + else: + self.push('250 %s %s' % (sim_users[user_email], quoted_addr)) + else: + self.push('550 No access for you!') + + def smtp_AUTH(self, arg): + if arg.strip().lower()=='cram-md5': + self.push('334 {0}'.format(sim_cram_md5_challenge)) + return + mech, auth = arg.split() + mech = mech.lower() + if mech not in sim_auth_credentials: + self.push('504 auth type unimplemented') + return + if mech == 'plain' and auth==sim_auth_credentials['plain']: + self.push('235 plain auth ok') + elif mech=='login' and auth==sim_auth_credentials['login']: + self.push('334 Password:') + else: + self.push('550 No access for you!') + + def handle_error(self): + raise + + +class SimSMTPServer(smtpd.SMTPServer): + + def __init__(self, *args, **kw): + self._extra_features = [] + smtpd.SMTPServer.__init__(self, *args, **kw) + + def handle_accept(self): + conn, addr = self.accept() + self._SMTPchannel = SimSMTPChannel(self._extra_features, + self, conn, addr) + + def process_message(self, peer, mailfrom, rcpttos, data): + pass + + def add_feature(self, feature): + self._extra_features.append(feature) + + def handle_error(self): + raise + + +# Test various SMTP & ESMTP commands/behaviors that require a simulated server +# (i.e., something with more features than DebuggingServer) +@unittest.skipUnless(threading, 'Threading required for this test.') +class SMTPSimTests(unittest.TestCase): + + def setUp(self): + self._threads = test_support.threading_setup() + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = SimSMTPServer((HOST, 0), ('nowhere', -1)) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + self.thread.join() + test_support.threading_cleanup(*self._threads) + + def testBasic(self): + # smoke test + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.quit() + + def testEHLO(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + # no features should be present before the EHLO + self.assertEqual(smtp.esmtp_features, {}) + + # features expected from the test server + expected_features = {'expn':'', + 'size': '20000000', + 'starttls': '', + 'deliverby': '', + 'help': '', + } + + smtp.ehlo() + self.assertEqual(smtp.esmtp_features, expected_features) + for k in expected_features: + self.assertTrue(smtp.has_extn(k)) + self.assertFalse(smtp.has_extn('unsupported-feature')) + smtp.quit() + + def testVRFY(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + for email, name in sim_users.items(): + expected_known = (250, '%s %s' % (name, smtplib.quoteaddr(email))) + self.assertEqual(smtp.vrfy(email), expected_known) + + u = 'nobody@nowhere.com' + expected_unknown = (550, 'No such user: %s' % u) + self.assertEqual(smtp.vrfy(u), expected_unknown) + smtp.quit() + + def testEXPN(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + for listname, members in sim_lists.items(): + users = [] + for m in members: + users.append('%s %s' % (sim_users[m], smtplib.quoteaddr(m))) + expected_known = (250, '\n'.join(users)) + self.assertEqual(smtp.expn(listname), expected_known) + + u = 'PSU-Members-List' + expected_unknown = (550, 'No access for you!') + self.assertEqual(smtp.expn(u), expected_unknown) + smtp.quit() + + def testAUTH_PLAIN(self): + self.serv.add_feature("AUTH PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + expected_auth_ok = (235, b'plain auth ok') + self.assertEqual(smtp.login(sim_auth[0], sim_auth[1]), expected_auth_ok) + + # SimSMTPChannel doesn't fully support LOGIN or CRAM-MD5 auth because they + # require a synchronous read to obtain the credentials...so instead smtpd + # sees the credential sent by smtplib's login method as an unknown command, + # which results in smtplib raising an auth error. Fortunately the error + # message contains the encoded credential, so we can partially check that it + # was generated correctly (partially, because the 'word' is uppercased in + # the error message). + + def testAUTH_LOGIN(self): + self.serv.add_feature("AUTH LOGIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + try: smtp.login(sim_auth[0], sim_auth[1]) + except smtplib.SMTPAuthenticationError as err: + if sim_auth_login_password not in str(err): + raise "expected encoded password not found in error message" + + def testAUTH_CRAM_MD5(self): + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + try: smtp.login(sim_auth[0], sim_auth[1]) + except smtplib.SMTPAuthenticationError as err: + if sim_auth_credentials['cram-md5'] not in str(err): + raise "expected encoded credentials not found in error message" + + #TODO: add tests for correct AUTH method fallback now that the + #test infrastructure can support it. + + def test_quit_resets_greeting(self): + smtp = smtplib.SMTP(HOST, self.port, + local_hostname='localhost', + timeout=15) + code, message = smtp.ehlo() + self.assertEqual(code, 250) + self.assertIn('size', smtp.esmtp_features) + smtp.quit() + self.assertNotIn('size', smtp.esmtp_features) + smtp.connect(HOST, self.port) + self.assertNotIn('size', smtp.esmtp_features) + smtp.ehlo_or_helo_if_needed() + self.assertIn('size', smtp.esmtp_features) + smtp.quit() + + +def test_main(verbose=None): + test_support.run_unittest(GeneralTests, DebuggingServerTests, + NonConnectingTests, + BadHELOServerTests, SMTPSimTests, + TooLongLineTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7pypy/test_socket.py b/src/greentest/2.7pypy/test_socket.py new file mode 100644 index 0000000..d0a7d68 --- /dev/null +++ b/src/greentest/2.7pypy/test_socket.py @@ -0,0 +1,1812 @@ +import unittest +from test import test_support + +import errno +import itertools +import socket +import select +import time +import traceback +import Queue +import sys +import os +import array +import contextlib +import signal +import math +import weakref +try: + import _socket +except ImportError: + _socket = None + + +def try_address(host, port=0, family=socket.AF_INET): + """Try to bind a socket on the given host:port and return True + if that has been possible.""" + try: + sock = socket.socket(family, socket.SOCK_STREAM) + sock.bind((host, port)) + except (socket.error, socket.gaierror): + return False + else: + sock.close() + return True + +HOST = test_support.HOST +MSG = b'Michael Gilfix was here\n' +SUPPORTS_IPV6 = socket.has_ipv6 and try_address('::1', family=socket.AF_INET6) + +try: + import thread + import threading +except ImportError: + thread = None + threading = None + +HOST = test_support.HOST +MSG = 'Michael Gilfix was here\n' + +class SocketTCPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = test_support.bind_port(self.serv) + self.serv.listen(1) + + def tearDown(self): + self.serv.close() + self.serv = None + +class SocketUDPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.port = test_support.bind_port(self.serv) + + def tearDown(self): + self.serv.close() + self.serv = None + +class ThreadableTest: + """Threadable Test class + + The ThreadableTest class makes it easy to create a threaded + client/server pair from an existing unit test. To create a + new threaded class from an existing unit test, use multiple + inheritance: + + class NewClass (OldClass, ThreadableTest): + pass + + This class defines two new fixture functions with obvious + purposes for overriding: + + clientSetUp () + clientTearDown () + + Any new test functions within the class must then define + tests in pairs, where the test name is preceded with a + '_' to indicate the client portion of the test. Ex: + + def testFoo(self): + # Server portion + + def _testFoo(self): + # Client portion + + Any exceptions raised by the clients during their tests + are caught and transferred to the main thread to alert + the testing framework. + + Note, the server setup function cannot call any blocking + functions that rely on the client thread during setup, + unless serverExplicitReady() is called just before + the blocking call (such as in setting up a client/server + connection and performing the accept() in setUp(). + """ + + def __init__(self): + # Swap the true setup function + self.__setUp = self.setUp + self.__tearDown = self.tearDown + self.setUp = self._setUp + self.tearDown = self._tearDown + + def serverExplicitReady(self): + """This method allows the server to explicitly indicate that + it wants the client thread to proceed. This is useful if the + server is about to execute a blocking routine that is + dependent upon the client thread during its setup routine.""" + self.server_ready.set() + + def _setUp(self): + self.server_ready = threading.Event() + self.client_ready = threading.Event() + self.done = threading.Event() + self.queue = Queue.Queue(1) + + # Do some munging to start the client test. + methodname = self.id() + i = methodname.rfind('.') + methodname = methodname[i+1:] + test_method = getattr(self, '_' + methodname) + self.client_thread = thread.start_new_thread( + self.clientRun, (test_method,)) + + self.__setUp() + if not self.server_ready.is_set(): + self.server_ready.set() + self.client_ready.wait() + + def _tearDown(self): + self.__tearDown() + self.done.wait() + + if not self.queue.empty(): + msg = self.queue.get() + self.fail(msg) + + def clientRun(self, test_func): + self.server_ready.wait() + self.clientSetUp() + self.client_ready.set() + if not callable(test_func): + raise TypeError("test_func must be a callable function.") + try: + test_func() + except Exception, strerror: + self.queue.put(strerror) + self.clientTearDown() + + def clientSetUp(self): + raise NotImplementedError("clientSetUp must be implemented.") + + def clientTearDown(self): + self.done.set() + thread.exit() + +class ThreadedTCPSocketTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketUDPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class SocketConnectedTest(ThreadedTCPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + ThreadedTCPSocketTest.setUp(self) + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + ThreadedTCPSocketTest.tearDown(self) + + def clientSetUp(self): + ThreadedTCPSocketTest.clientSetUp(self) + self.cli.connect((HOST, self.port)) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + ThreadedTCPSocketTest.clientTearDown(self) + +class SocketPairTest(unittest.TestCase, ThreadableTest): + + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.serv, self.cli = socket.socketpair() + + def tearDown(self): + self.serv.close() + self.serv = None + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +####################################################################### +## Begin Tests + +class GeneralModuleTests(unittest.TestCase): + + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_csocket_repr(self): + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + try: + expected = ('' + % (s.fileno(), s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + finally: + s.close() + expected = ('' + % (s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + + def test_weakref(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + p = weakref.proxy(s) + self.assertEqual(p.fileno(), s.fileno()) + s.close() + s = None + test_support.gc_collect() + try: + p.fileno() + except ReferenceError: + pass + else: + self.fail('Socket proxy still exists') + + def test_weakref__sock(self): + s = socket.socket()._sock + w = weakref.ref(s) + self.assertIs(w(), s) + del s + test_support.gc_collect() + self.assertIsNone(w()) + + def testSocketError(self): + # Testing socket module exceptions + def raise_error(*args, **kwargs): + raise socket.error + def raise_herror(*args, **kwargs): + raise socket.herror + def raise_gaierror(*args, **kwargs): + raise socket.gaierror + self.assertRaises(socket.error, raise_error, + "Error raising socket exception.") + self.assertRaises(socket.error, raise_herror, + "Error raising socket exception.") + self.assertRaises(socket.error, raise_gaierror, + "Error raising socket exception.") + + def testSendtoErrors(self): + # Testing that sendto doesn't mask failures. See #10169. + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind(('', 0)) + sockname = s.getsockname() + # 2 args + with self.assertRaises(UnicodeEncodeError): + s.sendto(u'\u2620', sockname) + with self.assertRaises(TypeError) as cm: + s.sendto(5j, sockname) + self.assertIn('complex', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', None) + self.assertIn('NoneType', str(cm.exception)) + # 3 args + with self.assertRaises(UnicodeEncodeError): + s.sendto(u'\u2620', 0, sockname) + with self.assertRaises(TypeError) as cm: + s.sendto(5j, 0, sockname) + self.assertIn('complex', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', 0, None) + if test_support.check_impl_detail(): + self.assertIn('not NoneType', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', 'bar', sockname) + self.assertIn('integer', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', None, None) + if test_support.check_impl_detail(): + self.assertIn('an integer is required', str(cm.exception)) + # wrong number of args + with self.assertRaises(TypeError) as cm: + s.sendto('foo') + self.assertIn(' given)', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto('foo', 0, sockname, 4) + self.assertIn(' given)', str(cm.exception)) + + + def testCrucialConstants(self): + # Testing for mission critical constants + socket.AF_INET + socket.SOCK_STREAM + socket.SOCK_DGRAM + socket.SOCK_RAW + socket.SOCK_RDM + socket.SOCK_SEQPACKET + socket.SOL_SOCKET + socket.SO_REUSEADDR + + def testHostnameRes(self): + # Testing hostname resolution mechanisms + hostname = socket.gethostname() + try: + ip = socket.gethostbyname(hostname) + except socket.error: + # Probably name lookup wasn't set up right; skip this test + self.skipTest('name lookup failure') + self.assertTrue(ip.find('.') >= 0, "Error resolving host to ip.") + try: + hname, aliases, ipaddrs = socket.gethostbyaddr(ip) + except socket.error: + # Probably a similar problem as above; skip this test + self.skipTest('address lookup failure') + all_host_names = [hostname, hname] + aliases + fqhn = socket.getfqdn(ip) + if not fqhn in all_host_names: + self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') + def testRefCountGetNameInfo(self): + # Testing reference count for getnameinfo + try: + # On some versions, this loses a reference + orig = sys.getrefcount(__name__) + socket.getnameinfo(__name__,0) + except TypeError: + self.assertEqual(sys.getrefcount(__name__), orig, + "socket.getnameinfo loses a reference") + + def testInterpreterCrash(self): + # Making sure getnameinfo doesn't crash the interpreter + try: + # On some versions, this crashes the interpreter. + socket.getnameinfo(('x', 0, 0, 0), 0) + except socket.error: + pass + + def testNtoH(self): + # This just checks that htons etc. are their own inverse, + # when looking at the lower 16 or 32 bits. + sizes = {socket.htonl: 32, socket.ntohl: 32, + socket.htons: 16, socket.ntohs: 16} + for func, size in sizes.items(): + mask = (1L<= _testcapi.ULONG_MAX: + self.skipTest('needs UINT_MAX < ULONG_MAX') + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.serv.setblocking(_testcapi.UINT_MAX + 1) + self.assertIsNone(self.serv.gettimeout()) + + _testSetBlocking_overflow = test_support.cpython_only(_testSetBlocking) + + def testAccept(self): + # Testing non-blocking accept + self.serv.setblocking(0) + try: + conn, addr = self.serv.accept() + except socket.error: + pass + else: + self.fail("Error trying to do non-blocking accept.") + read, write, err = select.select([self.serv], [], []) + if self.serv in read: + conn, addr = self.serv.accept() + conn.close() + else: + self.fail("Error trying to do accept after select.") + + def _testAccept(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + + def testConnect(self): + # Testing non-blocking connect + conn, addr = self.serv.accept() + conn.close() + + def _testConnect(self): + self.cli.settimeout(10) + self.cli.connect((HOST, self.port)) + + def testRecv(self): + # Testing non-blocking recv + conn, addr = self.serv.accept() + conn.setblocking(0) + try: + msg = conn.recv(len(MSG)) + except socket.error: + pass + else: + self.fail("Error trying to do non-blocking recv.") + read, write, err = select.select([conn], [], []) + if conn in read: + msg = conn.recv(len(MSG)) + conn.close() + self.assertEqual(msg, MSG) + else: + self.fail("Error during select call to non-blocking socket.") + + def _testRecv(self): + self.cli.connect((HOST, self.port)) + time.sleep(0.1) + self.cli.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class FileObjectClassTestCase(SocketConnectedTest): + + bufsize = -1 # Use default buffer size + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def setUp(self): + SocketConnectedTest.setUp(self) + self.serv_file = self.cli_conn.makefile('rb', self.bufsize) + + def tearDown(self): + self.serv_file.close() + self.assertTrue(self.serv_file.closed) + SocketConnectedTest.tearDown(self) + self.serv_file = None + + def clientSetUp(self): + SocketConnectedTest.clientSetUp(self) + self.cli_file = self.serv_conn.makefile('wb') + + def clientTearDown(self): + self.cli_file.close() + self.assertTrue(self.cli_file.closed) + self.cli_file = None + SocketConnectedTest.clientTearDown(self) + + def testSmallRead(self): + # Performing small file read test + first_seg = self.serv_file.read(len(MSG)-3) + second_seg = self.serv_file.read(3) + msg = first_seg + second_seg + self.assertEqual(msg, MSG) + + def _testSmallRead(self): + self.cli_file.write(MSG) + self.cli_file.flush() + + def testFullRead(self): + # read until EOF + msg = self.serv_file.read() + self.assertEqual(msg, MSG) + + def _testFullRead(self): + self.cli_file.write(MSG) + self.cli_file.close() + + def testUnbufferedRead(self): + # Performing unbuffered file read test + buf = '' + while 1: + char = self.serv_file.read(1) + if not char: + break + buf += char + self.assertEqual(buf, MSG) + + def _testUnbufferedRead(self): + self.cli_file.write(MSG) + self.cli_file.flush() + + def testReadline(self): + # Performing file readline test + line = self.serv_file.readline() + self.assertEqual(line, MSG) + + def _testReadline(self): + self.cli_file.write(MSG) + self.cli_file.flush() + + def testReadlineAfterRead(self): + a_baloo_is = self.serv_file.read(len("A baloo is")) + self.assertEqual("A baloo is", a_baloo_is) + _a_bear = self.serv_file.read(len(" a bear")) + self.assertEqual(" a bear", _a_bear) + line = self.serv_file.readline() + self.assertEqual("\n", line) + line = self.serv_file.readline() + self.assertEqual("A BALOO IS A BEAR.\n", line) + line = self.serv_file.readline() + self.assertEqual(MSG, line) + + def _testReadlineAfterRead(self): + self.cli_file.write("A baloo is a bear\n") + self.cli_file.write("A BALOO IS A BEAR.\n") + self.cli_file.write(MSG) + self.cli_file.flush() + + def testReadlineAfterReadNoNewline(self): + end_of_ = self.serv_file.read(len("End Of ")) + self.assertEqual("End Of ", end_of_) + line = self.serv_file.readline() + self.assertEqual("Line", line) + + def _testReadlineAfterReadNoNewline(self): + self.cli_file.write("End Of Line") + + def testClosedAttr(self): + self.assertTrue(not self.serv_file.closed) + + def _testClosedAttr(self): + self.assertTrue(not self.cli_file.closed) + + +class FileObjectInterruptedTestCase(unittest.TestCase): + """Test that the file object correctly handles EINTR internally.""" + + class MockSocket(object): + def __init__(self, recv_funcs=()): + # A generator that returns callables that we'll call for each + # call to recv(). + self._recv_step = iter(recv_funcs) + + def recv(self, size): + return self._recv_step.next()() + + def _reuse(self): pass + def _drop(self): pass + + @staticmethod + def _raise_eintr(): + raise socket.error(errno.EINTR) + + def _test_readline(self, size=-1, **kwargs): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : "This is the first line\nAnd the sec", + self._raise_eintr, + lambda : "ond line is here\n", + lambda : "", + ]) + fo = socket._fileobject(mock_sock, **kwargs) + self.assertEqual(fo.readline(size), "This is the first line\n") + self.assertEqual(fo.readline(size), "And the second line is here\n") + + def _test_read(self, size=-1, **kwargs): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : "This is the first line\nAnd the sec", + self._raise_eintr, + lambda : "ond line is here\n", + lambda : "", + ]) + fo = socket._fileobject(mock_sock, **kwargs) + self.assertEqual(fo.read(size), "This is the first line\n" + "And the second line is here\n") + + def test_default(self): + self._test_readline() + self._test_readline(size=100) + self._test_read() + self._test_read(size=100) + + def test_with_1k_buffer(self): + self._test_readline(bufsize=1024) + self._test_readline(size=100, bufsize=1024) + self._test_read(bufsize=1024) + self._test_read(size=100, bufsize=1024) + + def _test_readline_no_buffer(self, size=-1): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : "aa", + lambda : "\n", + lambda : "BB", + self._raise_eintr, + lambda : "bb", + lambda : "", + ]) + fo = socket._fileobject(mock_sock, bufsize=0) + self.assertEqual(fo.readline(size), "aa\n") + self.assertEqual(fo.readline(size), "BBbb") + + def test_no_buffer(self): + self._test_readline_no_buffer() + self._test_readline_no_buffer(size=4) + self._test_read(bufsize=0) + self._test_read(size=100, bufsize=0) + + +class UnbufferedFileObjectClassTestCase(FileObjectClassTestCase): + + """Repeat the tests from FileObjectClassTestCase with bufsize==0. + + In this case (and in this case only), it should be possible to + create a file object, read a line from it, create another file + object, read another line from it, without loss of data in the + first file object's buffer. Note that httplib relies on this + when reading multiple requests from the same socket.""" + + bufsize = 0 # Use unbuffered mode + + def testUnbufferedReadline(self): + # Read a line, create a new file object, read another line with it + line = self.serv_file.readline() # first line + self.assertEqual(line, "A. " + MSG) # first line + self.serv_file = self.cli_conn.makefile('rb', 0) + line = self.serv_file.readline() # second line + self.assertEqual(line, "B. " + MSG) # second line + + def _testUnbufferedReadline(self): + self.cli_file.write("A. " + MSG) + self.cli_file.write("B. " + MSG) + self.cli_file.flush() + +class LineBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 1 # Default-buffered for reading; line-buffered for writing + + class SocketMemo(object): + """A wrapper to keep track of sent data, needed to examine write behaviour""" + def __init__(self, sock): + self._sock = sock + self.sent = [] + + def send(self, data, flags=0): + n = self._sock.send(data, flags) + self.sent.append(data[:n]) + return n + + def sendall(self, data, flags=0): + self._sock.sendall(data, flags) + self.sent.append(data) + + def __getattr__(self, attr): + return getattr(self._sock, attr) + + def getsent(self): + return [e.tobytes() if isinstance(e, memoryview) else e for e in self.sent] + + def setUp(self): + FileObjectClassTestCase.setUp(self) + self.serv_file._sock = self.SocketMemo(self.serv_file._sock) + + def testLinebufferedWrite(self): + # Write two lines, in small chunks + msg = MSG.strip() + print >> self.serv_file, msg, + print >> self.serv_file, msg + + # second line: + print >> self.serv_file, msg, + print >> self.serv_file, msg, + print >> self.serv_file, msg + + # third line + print >> self.serv_file, '' + + self.serv_file.flush() + + msg1 = "%s %s\n"%(msg, msg) + msg2 = "%s %s %s\n"%(msg, msg, msg) + msg3 = "\n" + self.assertEqual(self.serv_file._sock.getsent(), [msg1, msg2, msg3]) + + def _testLinebufferedWrite(self): + msg = MSG.strip() + msg1 = "%s %s\n"%(msg, msg) + msg2 = "%s %s %s\n"%(msg, msg, msg) + msg3 = "\n" + l1 = self.cli_file.readline() + self.assertEqual(l1, msg1) + l2 = self.cli_file.readline() + self.assertEqual(l2, msg2) + l3 = self.cli_file.readline() + self.assertEqual(l3, msg3) + + +class SmallBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 2 # Exercise the buffering code + + +class NetworkConnectionTest(object): + """Prove network connection.""" + def clientSetUp(self): + # We're inherited below by BasicTCPTest2, which also inherits + # BasicTCPTest, which defines self.port referenced below. + self.cli = socket.create_connection((HOST, self.port)) + self.serv_conn = self.cli + +class BasicTCPTest2(NetworkConnectionTest, BasicTCPTest): + """Tests that NetworkConnection does not break existing TCP functionality. + """ + +class NetworkConnectionNoServer(unittest.TestCase): + class MockSocket(socket.socket): + def connect(self, *args): + raise socket.timeout('timed out') + + @contextlib.contextmanager + def mocked_socket_module(self): + """Return a socket which times out on connect""" + old_socket = socket.socket + socket.socket = self.MockSocket + try: + yield + finally: + socket.socket = old_socket + + def test_connect(self): + port = test_support.find_unused_port() + cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(cli.close) + with self.assertRaises(socket.error) as cm: + cli.connect((HOST, port)) + self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + + def test_create_connection(self): + # Issue #9792: errors raised by create_connection() should have + # a proper errno attribute. + port = test_support.find_unused_port() + with self.assertRaises(socket.error) as cm: + socket.create_connection((HOST, port)) + + # Issue #16257: create_connection() calls getaddrinfo() against + # 'localhost'. This may result in an IPV6 addr being returned + # as well as an IPV4 one: + # >>> socket.getaddrinfo('localhost', port, 0, SOCK_STREAM) + # >>> [(2, 2, 0, '', ('127.0.0.1', 41230)), + # (26, 2, 0, '', ('::1', 41230, 0, 0))] + # + # create_connection() enumerates through all the addresses returned + # and if it doesn't successfully bind to any of them, it propagates + # the last exception it encountered. + # + # On Solaris, ENETUNREACH is returned in this circumstance instead + # of ECONNREFUSED. So, if that errno exists, add it to our list of + # expected errnos. + expected_errnos = [ errno.ECONNREFUSED, ] + if hasattr(errno, 'ENETUNREACH'): + expected_errnos.append(errno.ENETUNREACH) + + self.assertIn(cm.exception.errno, expected_errnos) + + def test_create_connection_timeout(self): + # Issue #9792: create_connection() should not recast timeout errors + # as generic socket errors. + with self.mocked_socket_module(): + with self.assertRaises(socket.timeout): + socket.create_connection((HOST, 1234)) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.source_port = test_support.find_unused_port() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def _justAccept(self): + conn, addr = self.serv.accept() + conn.close() + + testFamily = _justAccept + def _testFamily(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.family, 2) + + testSourceAddress = _justAccept + def _testSourceAddress(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30, + source_address=('', self.source_port)) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.getsockname()[1], self.source_port) + # The port number being used is sufficient to show that the bind() + # call happened. + + testTimeoutDefault = _justAccept + def _testTimeoutDefault(self): + # passing no explicit timeout uses socket's global default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(42) + try: + self.cli = socket.create_connection((HOST, self.port)) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), 42) + + testTimeoutNone = _justAccept + def _testTimeoutNone(self): + # None timeout means the same as sock.settimeout(None) + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + self.cli = socket.create_connection((HOST, self.port), timeout=None) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), None) + + testTimeoutValueNamed = _justAccept + def _testTimeoutValueNamed(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.assertEqual(self.cli.gettimeout(), 30) + + testTimeoutValueNonamed = _justAccept + def _testTimeoutValueNonamed(self): + self.cli = socket.create_connection((HOST, self.port), 30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.gettimeout(), 30) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionBehaviourTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def testInsideTimeout(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + time.sleep(3) + conn.send("done!") + testOutsideTimeout = testInsideTimeout + + def _testInsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port)) + data = sock.recv(5) + self.assertEqual(data, "done!") + + def _testOutsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port), timeout=1) + self.assertRaises(socket.timeout, lambda: sock.recv(5)) + + +class Urllib2FileobjectTest(unittest.TestCase): + + # urllib2.HTTPHandler has "borrowed" socket._fileobject, and requires that + # it close the socket if the close c'tor argument is true + + def testClose(self): + class MockSocket: + closed = False + def flush(self): pass + def close(self): self.closed = True + def _reuse(self): pass + def _drop(self): pass + + # must not close unless we request it: the original use of _fileobject + # by module socket requires that the underlying socket not be closed until + # the _socketobject that created the _fileobject is closed + s = MockSocket() + f = socket._fileobject(s) + f.close() + self.assertTrue(not s.closed) + + s = MockSocket() + f = socket._fileobject(s, close=True) + f.close() + self.assertTrue(s.closed) + +class TCPTimeoutTest(SocketTCPTest): + + def testTCPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.accept() + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (TCP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of error (TCP)") + except socket.error: + ok = True + except: + self.fail("caught unexpected exception (TCP)") + if not ok: + self.fail("accept() returned success when we did not expect it") + + @unittest.skipUnless(hasattr(signal, 'alarm'), + 'test needs signal.alarm()') + def testInterruptedTimeout(self): + # XXX I don't know how to do this test on MSWindows or any other + # plaform that doesn't support signal.alarm() or os.kill(), though + # the bug should have existed on all platforms. + self.serv.settimeout(5.0) # must be longer than alarm + class Alarm(Exception): + pass + def alarm_handler(signal, frame): + raise Alarm + old_alarm = signal.signal(signal.SIGALRM, alarm_handler) + try: + signal.alarm(2) # POSIX allows alarm to be up to 1 second early + try: + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of Alarm") + except Alarm: + pass + except: + self.fail("caught other exception instead of Alarm:" + " %s(%s):\n%s" % + (sys.exc_info()[:2] + (traceback.format_exc(),))) + else: + self.fail("nothing caught") + finally: + signal.alarm(0) # shut off alarm + except Alarm: + self.fail("got Alarm in wrong place") + finally: + # no alarm can be pending. Safe to restore old handler. + signal.signal(signal.SIGALRM, old_alarm) + +class UDPTimeoutTest(SocketUDPTest): + + def testUDPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.recv(1024) + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (UDP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.recv(1024) + except socket.timeout: + self.fail("caught timeout instead of error (UDP)") + except socket.error: + ok = True + except: + self.fail("caught unexpected exception (UDP)") + if not ok: + self.fail("recv() returned success when we did not expect it") + +class TestExceptions(unittest.TestCase): + + def testExceptionTree(self): + self.assertTrue(issubclass(socket.error, Exception)) + self.assertTrue(issubclass(socket.herror, socket.error)) + self.assertTrue(issubclass(socket.gaierror, socket.error)) + self.assertTrue(issubclass(socket.timeout, socket.error)) + +@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +class TestLinuxAbstractNamespace(unittest.TestCase): + + UNIX_PATH_MAX = 108 + + def testLinuxAbstractNamespace(self): + address = "\x00python-test-hello\x00\xff" + s1 = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s1.bind(address) + s1.listen(1) + s2 = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s2.connect(s1.getsockname()) + s1.accept() + self.assertEqual(s1.getsockname(), address) + self.assertEqual(s2.getpeername(), address) + + def testMaxName(self): + address = "\x00" + "h" * (self.UNIX_PATH_MAX - 1) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.bind(address) + self.assertEqual(s.getsockname(), address) + + def testNameOverflow(self): + address = "\x00" + "h" * self.UNIX_PATH_MAX + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.assertRaises(socket.error, s.bind, address) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BufferIOTest(SocketConnectedTest): + """ + Test the buffer versions of socket.recv() and socket.send(). + """ + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecvIntoArray(self): + buf = array.array('c', ' '*1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf.tostring()[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvIntoArray(self): + with test_support.check_py3k_warnings(): + buf = buffer(MSG) + self.serv_conn.send(buf) + + def testRecvIntoBytearray(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoBytearray = _testRecvIntoArray + + def testRecvIntoMemoryview(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoMemoryview = _testRecvIntoArray + + def testRecvFromIntoArray(self): + buf = array.array('c', ' '*1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf.tostring()[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvFromIntoArray(self): + with test_support.check_py3k_warnings(): + buf = buffer(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoBytearray(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoBytearray = _testRecvFromIntoArray + + def testRecvFromIntoMemoryview(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoMemoryview = _testRecvFromIntoArray + + def testRecvFromIntoSmallBuffer(self): + # See issue #20246. + buf = bytearray(8) + self.assertRaises(ValueError, self.cli_conn.recvfrom_into, buf, 1024) + + def _testRecvFromIntoSmallBuffer(self): + with test_support.check_py3k_warnings(): + buf = buffer(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoEmptyBuffer(self): + buf = bytearray() + self.cli_conn.recvfrom_into(buf) + self.cli_conn.recvfrom_into(buf, 0) + + _testRecvFromIntoEmptyBuffer = _testRecvFromIntoArray + + +TIPC_STYPE = 2000 +TIPC_LOWER = 200 +TIPC_UPPER = 210 + +def isTipcAvailable(): + """Check if the TIPC module is loaded + + The TIPC module is not loaded automatically on Ubuntu and probably + other Linux distros. + """ + if not hasattr(socket, "AF_TIPC"): + return False + if not os.path.isfile("/proc/modules"): + return False + with open("/proc/modules") as f: + for line in f: + if line.startswith("tipc "): + return True + return False + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCTest(unittest.TestCase): + def testRDM(self): + srv = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + cli = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + srv.bind(srvaddr) + + sendaddr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + (TIPC_UPPER - TIPC_LOWER) / 2, 0) + cli.sendto(MSG, sendaddr) + + msg, recvaddr = srv.recvfrom(1024) + + self.assertEqual(cli.getsockname(), recvaddr) + self.assertEqual(msg, MSG) + + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCThreadableTest(unittest.TestCase, ThreadableTest): + def __init__(self, methodName = 'runTest'): + unittest.TestCase.__init__(self, methodName = methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.srv = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + self.srv.bind(srvaddr) + self.srv.listen(5) + self.serverExplicitReady() + self.conn, self.connaddr = self.srv.accept() + + def clientSetUp(self): + # There is a hittable race between serverExplicitReady() and the + # accept() call; sleep a little while to avoid it, otherwise + # we could get an exception + time.sleep(0.1) + self.cli = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + addr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + (TIPC_UPPER - TIPC_LOWER) / 2, 0) + self.cli.connect(addr) + self.cliaddr = self.cli.getsockname() + + def testStream(self): + msg = self.conn.recv(1024) + self.assertEqual(msg, MSG) + self.assertEqual(self.cliaddr, self.connaddr) + + def _testStream(self): + self.cli.send(MSG) + self.cli.close() + + +def test_main(): + tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest, + TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, + UDPTimeoutTest ] + + tests.extend([ + NonBlockingTCPTests, + FileObjectClassTestCase, + FileObjectInterruptedTestCase, + UnbufferedFileObjectClassTestCase, + LineBufferedFileObjectClassTestCase, + SmallBufferedFileObjectClassTestCase, + Urllib2FileobjectTest, + NetworkConnectionNoServer, + NetworkConnectionAttributesTest, + NetworkConnectionBehaviourTest, + ]) + tests.append(BasicSocketPairTest) + tests.append(TestLinuxAbstractNamespace) + tests.extend([TIPCTest, TIPCThreadableTest]) + + thread_info = test_support.threading_setup() + test_support.run_unittest(*tests) + test_support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_socketserver.py b/src/greentest/2.7pypy/test_socketserver.py new file mode 100644 index 0000000..d645d20 --- /dev/null +++ b/src/greentest/2.7pypy/test_socketserver.py @@ -0,0 +1,361 @@ +""" +Test suite for SocketServer.py. +""" + +import contextlib +import imp +import os +import select +import signal +import socket +import select +import errno +import tempfile +import unittest +import SocketServer + +import test.test_support +from test.test_support import reap_children, reap_threads, verbose +try: + import threading +except ImportError: + threading = None + +test.test_support.requires("network") + +TEST_STR = "hello world\n" +HOST = test.test_support.HOST + +HAVE_UNIX_SOCKETS = hasattr(socket, "AF_UNIX") +requires_unix_sockets = unittest.skipUnless(HAVE_UNIX_SOCKETS, + 'requires Unix sockets') +HAVE_FORKING = hasattr(os, "fork") and os.name != "os2" +requires_forking = unittest.skipUnless(HAVE_FORKING, 'requires forking') + +def signal_alarm(n): + """Call signal.alarm when it exists (i.e. not on Windows).""" + if hasattr(signal, 'alarm'): + signal.alarm(n) + +# Remember real select() to avoid interferences with mocking +_real_select = select.select + +def receive(sock, n, timeout=20): + r, w, x = _real_select([sock], [], [], timeout) + if sock in r: + return sock.recv(n) + else: + raise RuntimeError, "timed out on %r" % (sock,) + +if HAVE_UNIX_SOCKETS: + class ForkingUnixStreamServer(SocketServer.ForkingMixIn, + SocketServer.UnixStreamServer): + pass + + class ForkingUnixDatagramServer(SocketServer.ForkingMixIn, + SocketServer.UnixDatagramServer): + pass + + +@contextlib.contextmanager +def simple_subprocess(testcase): + pid = os.fork() + if pid == 0: + # Don't raise an exception; it would be caught by the test harness. + os._exit(72) + yield None + pid2, status = os.waitpid(pid, 0) + testcase.assertEqual(pid2, pid) + testcase.assertEqual(72 << 8, status) + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class SocketServerTest(unittest.TestCase): + """Test all socket servers.""" + + def setUp(self): + signal_alarm(60) # Kill deadlocks after 60 seconds. + self.port_seed = 0 + self.test_files = [] + + def tearDown(self): + signal_alarm(0) # Didn't deadlock. + reap_children() + + for fn in self.test_files: + try: + os.remove(fn) + except os.error: + pass + self.test_files[:] = [] + + def pickaddr(self, proto): + if proto == socket.AF_INET: + return (HOST, 0) + else: + # XXX: We need a way to tell AF_UNIX to pick its own name + # like AF_INET provides port==0. + dir = None + if os.name == 'os2': + dir = '\socket' + fn = tempfile.mktemp(prefix='unix_socket.', dir=dir) + if os.name == 'os2': + # AF_UNIX socket names on OS/2 require a specific prefix + # which can't include a drive letter and must also use + # backslashes as directory separators + if fn[1] == ':': + fn = fn[2:] + if fn[0] in (os.sep, os.altsep): + fn = fn[1:] + if os.sep == '/': + fn = fn.replace(os.sep, os.altsep) + else: + fn = fn.replace(os.altsep, os.sep) + self.test_files.append(fn) + return fn + + def make_server(self, addr, svrcls, hdlrbase): + class MyServer(svrcls): + def handle_error(self, request, client_address): + self.close_request(request) + self.server_close() + raise + + class MyHandler(hdlrbase): + def handle(self): + line = self.rfile.readline() + self.wfile.write(line) + + if verbose: print "creating server" + server = MyServer(addr, MyHandler) + self.assertEqual(server.server_address, server.socket.getsockname()) + return server + + @reap_threads + def run_server(self, svrcls, hdlrbase, testfunc): + server = self.make_server(self.pickaddr(svrcls.address_family), + svrcls, hdlrbase) + # We had the OS pick a port, so pull the real address out of + # the server. + addr = server.server_address + if verbose: + print "server created" + print "ADDR =", addr + print "CLASS =", svrcls + t = threading.Thread( + name='%s serving' % svrcls, + target=server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval':0.01}) + t.daemon = True # In case this function raises. + t.start() + if verbose: print "server running" + for i in range(3): + if verbose: print "test client", i + testfunc(svrcls.address_family, addr) + if verbose: print "waiting for server" + server.shutdown() + t.join() + server.server_close() + self.assertRaises(socket.error, server.socket.fileno) + if verbose: print "done" + + def stream_examine(self, proto, addr): + s = socket.socket(proto, socket.SOCK_STREAM) + s.connect(addr) + s.sendall(TEST_STR) + buf = data = receive(s, 100) + while data and '\n' not in buf: + data = receive(s, 100) + buf += data + self.assertEqual(buf, TEST_STR) + s.close() + + def dgram_examine(self, proto, addr): + s = socket.socket(proto, socket.SOCK_DGRAM) + if HAVE_UNIX_SOCKETS and proto == socket.AF_UNIX: + s.bind(self.pickaddr(proto)) + s.sendto(TEST_STR, addr) + buf = data = receive(s, 100) + while data and '\n' not in buf: + data = receive(s, 100) + buf += data + self.assertEqual(buf, TEST_STR) + s.close() + + def test_TCPServer(self): + self.run_server(SocketServer.TCPServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + def test_ThreadingTCPServer(self): + self.run_server(SocketServer.ThreadingTCPServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + @requires_forking + def test_ForkingTCPServer(self): + with simple_subprocess(self): + self.run_server(SocketServer.ForkingTCPServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + def test_UnixStreamServer(self): + self.run_server(SocketServer.UnixStreamServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + def test_ThreadingUnixStreamServer(self): + self.run_server(SocketServer.ThreadingUnixStreamServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + @requires_forking + def test_ForkingUnixStreamServer(self): + with simple_subprocess(self): + self.run_server(ForkingUnixStreamServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + + def test_UDPServer(self): + self.run_server(SocketServer.UDPServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + def test_ThreadingUDPServer(self): + self.run_server(SocketServer.ThreadingUDPServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + @requires_forking + def test_ForkingUDPServer(self): + with simple_subprocess(self): + self.run_server(SocketServer.ForkingUDPServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + @contextlib.contextmanager + def mocked_select_module(self): + """Mocks the select.select() call to raise EINTR for first call""" + old_select = select.select + + class MockSelect: + def __init__(self): + self.called = 0 + + def __call__(self, *args): + self.called += 1 + if self.called == 1: + # raise the exception on first call + raise select.error(errno.EINTR, os.strerror(errno.EINTR)) + else: + # Return real select value for consecutive calls + return old_select(*args) + + select.select = MockSelect() + try: + yield select.select + finally: + select.select = old_select + + def test_InterruptServerSelectCall(self): + with self.mocked_select_module() as mock_select: + pid = self.run_server(SocketServer.TCPServer, + SocketServer.StreamRequestHandler, + self.stream_examine) + # Make sure select was called again: + self.assertGreater(mock_select.called, 1) + + @requires_unix_sockets + def test_UnixDatagramServer(self): + self.run_server(SocketServer.UnixDatagramServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + @requires_unix_sockets + def test_ThreadingUnixDatagramServer(self): + self.run_server(SocketServer.ThreadingUnixDatagramServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + @requires_unix_sockets + @requires_forking + def test_ForkingUnixDatagramServer(self): + self.run_server(ForkingUnixDatagramServer, + SocketServer.DatagramRequestHandler, + self.dgram_examine) + + @reap_threads + def test_shutdown(self): + # Issue #2302: shutdown() should always succeed in making an + # other thread leave serve_forever(). + class MyServer(SocketServer.TCPServer): + pass + + class MyHandler(SocketServer.StreamRequestHandler): + pass + + threads = [] + for i in range(20): + s = MyServer((HOST, 0), MyHandler) + t = threading.Thread( + name='MyServer serving', + target=s.serve_forever, + kwargs={'poll_interval':0.01}) + t.daemon = True # In case this function raises. + threads.append((t, s)) + for t, s in threads: + t.start() + s.shutdown() + for t, s in threads: + t.join() + + def test_tcpserver_bind_leak(self): + # Issue #22435: the server socket wouldn't be closed if bind()/listen() + # failed. + # Create many servers for which bind() will fail, to see if this result + # in FD exhaustion. + for i in range(1024): + with self.assertRaises(OverflowError): + SocketServer.TCPServer((HOST, -1), + SocketServer.StreamRequestHandler) + + +class MiscTestCase(unittest.TestCase): + + def test_shutdown_request_called_if_verify_request_false(self): + # Issue #26309: BaseServer should call shutdown_request even if + # verify_request is False + + class MyServer(SocketServer.TCPServer): + def verify_request(self, request, client_address): + return False + + shutdown_called = 0 + def shutdown_request(self, request): + self.shutdown_called += 1 + SocketServer.TCPServer.shutdown_request(self, request) + + server = MyServer((HOST, 0), SocketServer.StreamRequestHandler) + s = socket.socket(server.address_family, socket.SOCK_STREAM) + s.connect(server.server_address) + s.close() + server.handle_request() + self.assertEqual(server.shutdown_called, 1) + server.server_close() + + +def test_main(): + if imp.lock_held(): + # If the import lock is held, the threads will hang + raise unittest.SkipTest("can't run when import lock is held") + + test.test_support.run_unittest(SocketServerTest) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_ssl.py b/src/greentest/2.7pypy/test_ssl.py new file mode 100644 index 0000000..a17d613 --- /dev/null +++ b/src/greentest/2.7pypy/test_ssl.py @@ -0,0 +1,3187 @@ +# -*- coding: utf-8 -*- +# Test the support for SSL and sockets + +import sys +import unittest +from test import test_support as support +from test.script_helper import assert_python_ok +import asyncore +import socket +import select +import time +import datetime +import gc +import os +import errno +import pprint +import tempfile +import urllib2 +import traceback +import weakref +import platform +import functools +from contextlib import closing + +ssl = support.import_module("ssl") + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = support.HOST +IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL') +IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0) + + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/make_ssl_certs.py. +# Other certificates are simply fetched from the Internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = CERTFILE.encode(sys.getfilesystemencoding()) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = ONLYCERT.encode(sys.getfilesystemencoding()) +BYTES_ONLYKEY = ONLYKEY.encode(sys.getfilesystemencoding()) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = CAPATH.encode(sys.getfilesystemencoding()) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") + + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNING_CA = data_file("pycacert.pem") +# cert with all kinds of subject alt names +ALLSANFILE = data_file("allsans.pem") + +REMOTE_HOST = "self-signed.pythontest.net" +REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +NONEXISTINGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") + +DHFILE = data_file("dh1024.pem") +BYTES_DHFILE = DHFILE.encode(sys.getfilesystemencoding()) + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + + +class BasicTests(unittest.TestCase): + + def test_sslwrap_simple(self): + # A crude test for the legacy API + try: + ssl.sslwrap_simple(socket.socket(socket.AF_INET)) + except IOError, e: + if e.errno == 32: # broken pipe when ssl_sock.do_handshake(), this test doesn't care about that + pass + else: + raise + try: + ssl.sslwrap_simple(socket.socket(socket.AF_INET)._sock) + except IOError, e: + if e.errno == 32: # broken pipe when ssl_sock.do_handshake(), this test doesn't care about that + pass + else: + raise + + +def can_clear_options(): + # 0.9.8m or higher + return ssl._OPENSSL_API_VERSION >= (0, 9, 8, 13, 15) + +def no_sslv2_implies_sslv3_hello(): + # 0.9.7h or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) + +def have_verify_flags(): + # 0.9.8 or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 8, 0, 15) + +def utc_offset(): #NOTE: ignore issues like #1647654 + # local time = utc time + utc offset + if time.daylight and time.localtime().tm_isdst > 0: + return -time.altzone # seconds + return -time.timezone + +def asn1time(cert_time): + # Some versions of OpenSSL ignore seconds, see #18207 + # 0.9.8.i + if ssl._OPENSSL_API_VERSION == (0, 9, 8, 9, 15): + fmt = "%b %d %H:%M:%S %Y GMT" + dt = datetime.datetime.strptime(cert_time, fmt) + dt = dt.replace(second=0) + cert_time = dt.strftime(fmt) + # %d adds leading zero but ASN1_TIME_print() uses leading space + if cert_time[4] == "0": + cert_time = cert_time[:4] + " " + cert_time[5:] + + return cert_time + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): + if hasattr(ssl, 'PROTOCOL_SSLv2'): + @functools.wraps(func) + def f(*args, **kwargs): + try: + ssl.SSLContext(ssl.PROTOCOL_SSLv2) + except ssl.SSLError: + if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and + platform.linux_distribution() == ('debian', 'squeeze/sid', '')): + raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") + return func(*args, **kwargs) + return f + else: + return func + +needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + if ssl.HAS_ECDH: + ssl.OP_SINGLE_ECDH_USE + if ssl.OPENSSL_VERSION_INFO >= (1, 0): + ssl.OP_NO_COMPRESSION + self.assertIn(ssl.HAS_SNI, {True, False}) + self.assertIn(ssl.HAS_ECDH, {True, False}) + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + if hasattr(ssl, 'RAND_egd'): + self.assertRaises(TypeError, ssl.RAND_egd, 1) + self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1) + ssl.RAND_add("this is a random string", 75.0) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + p = ssl._ssl._test_decode_cert(CERTFILE) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['issuer'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + # Note the next three asserts will fail if the keys are regenerated + self.assertEqual(p['notAfter'], asn1time('Oct 5 23:01:56 2020 GMT')) + self.assertEqual(p['notBefore'], asn1time('Oct 8 23:01:56 2010 GMT')) + self.assertEqual(p['serialNumber'], 'D7C7381919AFC24E') + self.assertEqual(p['subject'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + + def test_parse_all_sans(self): + p = ssl._ssl._test_decode_cert(ALLSANFILE) + self.assertEqual(p['subjectAltName'], + ( + ('DNS', 'allsans'), + ('othername', ''), + ('othername', ''), + ('email', 'user@example.org'), + ('DNS', 'www.example.org'), + ('DirName', + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'dirname example'),))), + ('URI', 'https://www.python.org/'), + ('IP Address', '127.0.0.1'), + ('IP Address', '0:0:0:0:0:0:0:1\n'), + ('Registered ID', '1.2.3.4.5') + ) + ) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, (int, long)) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 0.9 + self.assertGreaterEqual(n, 0x900000) + # < 3.0 + self.assertLess(n, 0x30000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 0) + self.assertLess(major, 3) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + # Version string as returned by {Open,Libre}SSL, the format might change + if IS_LIBRESSL: + self.assertTrue(s.startswith("LibreSSL {:d}".format(major)), + (s, t, hex(n))) + else: + self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)), + (s, t)) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + wr = weakref.ref(ss) + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # socket.error raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with closing(ssl.wrap_socket(s)) as ss: + self.assertRaises(socket.error, ss.recv, 1) + self.assertRaises(socket.error, ss.recv_into, bytearray(b'x')) + self.assertRaises(socket.error, ss.recvfrom, 1) + self.assertRaises(socket.error, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(socket.error, ss.send, b'x') + self.assertRaises(socket.error, ss.sendto, b'x', ('0.0.0.0', 0)) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with closing(ssl.wrap_socket(s)) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + def test_errors(self): + sock = socket.socket() + self.assertRaisesRegexp(ValueError, + "certfile must be specified", + ssl.wrap_socket, sock, keyfile=CERTFILE) + self.assertRaisesRegexp(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True) + self.assertRaisesRegexp(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True, certfile="") + with closing(ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE)) as s: + self.assertRaisesRegexp(ValueError, "can't connect in server-side mode", + s.connect, (HOST, 8080)) + with self.assertRaises(IOError) as cm: + with closing(socket.socket()) as sock: + ssl.wrap_socket(sock, certfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(IOError) as cm: + with closing(socket.socket()) as sock: + ssl.wrap_socket(sock, + certfile=CERTFILE, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(IOError) as cm: + with closing(socket.socket()) as sock: + ssl.wrap_socket(sock, + certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def bad_cert_test(self, certfile): + """Check that trying to use the given client certificate fails""" + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + certfile) + sock = socket.socket() + self.addCleanup(sock.close) + with self.assertRaises(ssl.SSLError): + ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) + + def test_empty_cert(self): + """Wrapping with an empty cert file""" + self.bad_cert_test("nullcert.pem") + + def test_malformed_cert(self): + """Wrapping with a badly formatted certificate (syntax error)""" + self.bad_cert_test("badcert.pem") + + def test_malformed_key(self): + """Wrapping with a badly formatted key (syntax error)""" + self.bad_cert_test("badkey.pem") + + def test_match_hostname(self): + def ok(cert, hostname): + ssl.match_hostname(cert, hostname) + def fail(cert, hostname): + self.assertRaises(ssl.CertificateError, + ssl.match_hostname, cert, hostname) + + cert = {'subject': ((('commonName', 'example.com'),),)} + ok(cert, 'example.com') + ok(cert, 'ExAmple.cOm') + fail(cert, 'www.example.com') + fail(cert, '.example.com') + fail(cert, 'example.org') + fail(cert, 'exampleXcom') + + cert = {'subject': ((('commonName', '*.a.com'),),)} + ok(cert, 'foo.a.com') + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + # only match one left-most wildcard + cert = {'subject': ((('commonName', 'f*.com'),),)} + ok(cert, 'foo.com') + ok(cert, 'f.com') + fail(cert, 'bar.com') + fail(cert, 'foo.a.com') + fail(cert, 'bar.foo.com') + + # NULL bytes are bad, CVE-2013-4073 + cert = {'subject': ((('commonName', + 'null.python.org\x00example.org'),),)} + ok(cert, 'null.python.org\x00example.org') # or raise an error? + fail(cert, 'example.org') + fail(cert, 'null.python.org') + + # error cases with wildcards + cert = {'subject': ((('commonName', '*.*.a.com'),),)} + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + cert = {'subject': ((('commonName', 'a.*.com'),),)} + fail(cert, 'a.foo.com') + fail(cert, 'a..com') + fail(cert, 'a.com') + + # wildcard doesn't match IDNA prefix 'xn--' + idna = u'püthon.python.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, idna) + cert = {'subject': ((('commonName', 'x*.python.org'),),)} + fail(cert, idna) + cert = {'subject': ((('commonName', 'xn--p*.python.org'),),)} + fail(cert, idna) + + # wildcard in first fragment and IDNA A-labels in sequent fragments + # are supported. + idna = u'www*.pythön.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, u'www.pythön.org'.encode("idna").decode("ascii")) + ok(cert, u'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, u'ftp.pythön.org'.encode("idna").decode("ascii")) + fail(cert, u'pythön.org'.encode("idna").decode("ascii")) + + # Slightly fake real-world example + cert = {'notAfter': 'Jun 26 21:41:46 2011 GMT', + 'subject': ((('commonName', 'linuxfrz.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('DNS', 'linuxfr.com'), + ('othername', ''))} + ok(cert, 'linuxfr.org') + ok(cert, 'linuxfr.com') + # Not a "DNS" entry + fail(cert, '') + # When there is a subjectAltName, commonName isn't used + fail(cert, 'linuxfrz.org') + + # A pristine real-world example + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),), + (('commonName', 'mail.google.com'),))} + ok(cert, 'mail.google.com') + fail(cert, 'gmail.com') + # Only commonName is considered + fail(cert, 'California') + + # Neither commonName nor subjectAltName + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),))} + fail(cert, 'mail.google.com') + + # No DNS entry in subjectAltName but a commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('commonName', 'mail.google.com'),)), + 'subjectAltName': (('othername', 'blabla'), )} + ok(cert, 'mail.google.com') + + # No DNS entry subjectAltName and no commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),)), + 'subjectAltName': (('othername', 'blabla'),)} + fail(cert, 'google.com') + + # Empty cert / no cert + self.assertRaises(ValueError, ssl.match_hostname, None, 'example.com') + self.assertRaises(ValueError, ssl.match_hostname, {}, 'example.com') + + # Issue #17980: avoid denials of service by refusing more than one + # wildcard per fragment. + cert = {'subject': ((('commonName', 'a*b.com'),),)} + ok(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b.co*'),),)} + fail(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b*.com'),),)} + with self.assertRaises(ssl.CertificateError) as cm: + ssl.match_hostname(cert, 'axxbxxc.com') + self.assertIn("too many wildcards", str(cm.exception)) + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with closing(socket.socket()) as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + with closing(ssl.wrap_socket(s)) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with closing(ssl.wrap_socket(s)) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with closing(ssl.wrap_socket(s, server_side=True, certfile=CERTFILE)) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (set, bool)) + if isinstance(trust, set): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegexp(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegexp(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + def cert_time_ok(self, timestring, timestamp): + self.assertEqual(ssl.cert_time_to_seconds(timestring), timestamp) + + def cert_time_fail(self, timestring): + with self.assertRaises(ValueError): + ssl.cert_time_to_seconds(timestring) + + @unittest.skipUnless(utc_offset(), + 'local time needs to be different from UTC') + def test_cert_time_to_seconds_timezone(self): + # Issue #19940: ssl.cert_time_to_seconds() returns wrong + # results if local timezone is not UTC + self.cert_time_ok("May 9 00:00:00 2007 GMT", 1178668800.0) + self.cert_time_ok("Jan 5 09:34:43 2018 GMT", 1515144883.0) + + def test_cert_time_to_seconds(self): + timestring = "Jan 5 09:34:43 2018 GMT" + ts = 1515144883.0 + self.cert_time_ok(timestring, ts) + # accept keyword parameter, assert its name + self.assertEqual(ssl.cert_time_to_seconds(cert_time=timestring), ts) + # accept both %e and %d (space or zero generated by strftime) + self.cert_time_ok("Jan 05 09:34:43 2018 GMT", ts) + # case-insensitive + self.cert_time_ok("JaN 5 09:34:43 2018 GmT", ts) + self.cert_time_fail("Jan 5 09:34 2018 GMT") # no seconds + self.cert_time_fail("Jan 5 09:34:43 2018") # no GMT + self.cert_time_fail("Jan 5 09:34:43 2018 UTC") # not GMT timezone + self.cert_time_fail("Jan 35 09:34:43 2018 GMT") # invalid day + self.cert_time_fail("Jon 5 09:34:43 2018 GMT") # invalid month + self.cert_time_fail("Jan 5 24:00:00 2018 GMT") # invalid hour + self.cert_time_fail("Jan 5 09:60:43 2018 GMT") # invalid minute + + newyear_ts = 1230768000.0 + # leap seconds + self.cert_time_ok("Dec 31 23:59:60 2008 GMT", newyear_ts) + # same timestamp + self.cert_time_ok("Jan 1 00:00:00 2009 GMT", newyear_ts) + + self.cert_time_ok("Jan 5 09:34:59 2018 GMT", 1515144899) + # allow 60th second (even if it is not a leap second) + self.cert_time_ok("Jan 5 09:34:60 2018 GMT", 1515144900) + # allow 2nd leap second for compatibility with time.strptime() + self.cert_time_ok("Jan 5 09:34:61 2018 GMT", 1515144901) + self.cert_time_fail("Jan 5 09:34:62 2018 GMT") # invalid seconds + + # no special treatement for the special value: + # 99991231235959Z (rfc 5280) + self.cert_time_ok("Dec 31 23:59:59 9999 GMT", 253402300799.0) + + @support.run_with_locale('LC_ALL', '') + def test_cert_time_to_seconds_locale(self): + # `cert_time_to_seconds()` should be locale independent + + def local_february_name(): + return time.strftime('%b', (1, 2, 3, 4, 5, 6, 0, 0, 0)) + + if local_february_name().lower() == 'feb': + self.skipTest("locale-specific month name needs to be " + "different from C locale") + + # locale-independent + self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) + self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + + +class ContextTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_constructor(self): + for protocol in PROTOCOLS: + ssl.SSLContext(protocol) + self.assertRaises(TypeError, ssl.SSLContext) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + @skip_if_broken_ubuntu_ssl + def test_protocol(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.protocol, proto) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegexp(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @skip_if_broken_ubuntu_ssl + def test_options(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0): + default |= ssl.OP_NO_COMPRESSION + self.assertEqual(default, ctx.options) + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + if can_clear_options(): + ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) + self.assertEqual(default, ctx.options) + ctx.options = 0 + self.assertEqual(0, ctx.options) + else: + with self.assertRaises(ValueError): + ctx.options = 0 + + def test_verify_mode(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(IOError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaisesRegexp(ssl.SSLError, "key values mismatch"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegexp(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegexp(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegexp(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegexp(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegexp(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + ctx.load_verify_locations(cafile=BYTES_CERTFILE.decode('utf-8')) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(IOError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(IOError): + ctx.load_verify_locations(u'') + with self.assertRaisesRegexp(ssl.SSLError, "PEM lib"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read().decode("ascii") + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read().decode("ascii") + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegexp(ssl.SSLError, "no start line"): + ctx.load_verify_locations(cadata=u"broken") + with self.assertRaisesRegexp(ssl.SSLError, "not enough data"): + ctx.load_verify_locations(cadata=b"broken") + + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_dh_params(DHFILE) + if os.name != 'nt': + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(IOError) as cm: + ctx.load_dh_params(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + @skip_if_broken_ubuntu_ssl + def test_session_stats(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + @needs_sni + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @needs_sni + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': asn1time('Mar 29 12:29:49 2033 GMT'), + 'notBefore': asn1time('Mar 30 12:29:49 2003 GMT'), + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + with open(SIGNING_CA) as f: + cadata = f.read().decode("ascii") + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_DH_USE", 0), + getattr(ssl, "OP_SINGLE_DH_USE", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + ) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + def test__https_verify_certificates(self): + # Unit test to check the contect factory mapping + # The factories themselves are tested above + # This test will fail by design if run under PYTHONHTTPSVERIFY=0 + # (as will various test_httplib tests) + + # Uses a fresh SSL module to avoid affecting the real one + local_ssl = support.import_fresh_module("ssl") + # Certificate verification is enabled by default + self.assertIs(local_ssl._create_default_https_context, + local_ssl.create_default_context) + # Turn default verification off + local_ssl._https_verify_certificates(enable=False) + self.assertIs(local_ssl._create_default_https_context, + local_ssl._create_unverified_context) + # And back on + local_ssl._https_verify_certificates(enable=True) + self.assertIs(local_ssl._create_default_https_context, + local_ssl.create_default_context) + # The default behaviour is to enable + local_ssl._https_verify_certificates(enable=False) + local_ssl._https_verify_certificates() + self.assertIs(local_ssl._create_default_https_context, + local_ssl.create_default_context) + + def test__https_verify_envvar(self): + # Unit test to check the PYTHONHTTPSVERIFY handling + # Need to use a subprocess so it can still be run under -E + https_is_verified = """import ssl, sys; \ + status = "Error: _create_default_https_context does not verify certs" \ + if ssl._create_default_https_context is \ + ssl._create_unverified_context \ + else None; \ + sys.exit(status)""" + https_is_not_verified = """import ssl, sys; \ + status = "Error: _create_default_https_context verifies certs" \ + if ssl._create_default_https_context is \ + ssl.create_default_context \ + else None; \ + sys.exit(status)""" + extra_env = {} + # Omitting it leaves verification on + assert_python_ok("-c", https_is_verified, **extra_env) + # Setting it to zero turns verification off + extra_env[ssl._https_verify_envvar] = "0" + assert_python_ok("-c", https_is_not_verified, **extra_env) + # Any other value should also leave it on + for setting in ("", "1", "enabled", "foo"): + extra_env[ssl._https_verify_envvar] = setting + assert_python_ok("-c", https_is_verified, **extra_env) + + def test_check_hostname(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertFalse(ctx.check_hostname) + + # Requires CERT_REQUIRED or CERT_OPTIONAL + with self.assertRaises(ValueError): + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + self.assertEqual(cm.exception.library, 'PEM') + self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) + self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with closing(socket.socket()) as s: + s.bind(("127.0.0.1", 0)) + s.listen(5) + c = socket.socket() + c.connect(s.getsockname()) + c.setblocking(False) + with closing(ctx.wrap_socket(c, False, do_handshake_on_connect=False)) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertTrue(s.startswith("The operation did not complete (read)"), s) + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + +class NetworkedTests(unittest.TestCase): + + def test_connect(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) + try: + s.connect((REMOTE_HOST, 443)) + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + + # this should fail because we have no verification certs + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.assertRaisesRegexp(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + + # this should succeed because we specify the root cert + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + s.connect((REMOTE_HOST, 443)) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.setblocking(False) + rc = s.connect_ex((REMOTE_HOST, 443)) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + s.close() + + def test_connect_ex_error(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + rc = s.connect_ex((REMOTE_HOST, 444)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + finally: + s.close() + + def test_connect_with_context(self): + with support.transient_internet(REMOTE_HOST): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + # Same with a server hostname + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=REMOTE_HOST) + s.connect((REMOTE_HOST, 443)) + s.close() + # This should fail because we have no verification certs + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.assertRaisesRegexp(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + # This should succeed because we specify the root cert + ctx.load_verify_locations(REMOTE_ROOT_CERT) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_cadata(self): + with open(REMOTE_ROOT_CERT) as f: + pem = f.read().decode('ascii') + der = ssl.PEM_cert_to_DER_cert(pem) + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with closing(ctx.wrap_socket(socket.socket(socket.AF_INET))) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with closing(ctx.wrap_socket(socket.socket(socket.AF_INET))) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + with support.transient_internet(REMOTE_HOST): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss.connect((REMOTE_HOST, 443)) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + with support.transient_internet(REMOTE_HOST): + s = socket.socket(socket.AF_INET) + s.connect((REMOTE_HOST, 443)) + s.setblocking(False) + s = ssl.wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + s.close() + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + def _test_get_server_certificate(host, port, cert=None): + with support.transient_internet(host): + pem = ssl.get_server_certificate((host, port)) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + + try: + pem = ssl.get_server_certificate((host, port), + ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + pem = ssl.get_server_certificate((host, port), + ca_certs=cert) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + + _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) + if support.IPV6_ENABLED: + _test_get_server_certificate('ipv6.google.com', 443) + + def test_ciphers(self): + remote = (REMOTE_HOST, 443) + with support.transient_internet(remote[0]): + with closing(ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL")) as s: + s.connect(remote) + with closing(ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT")) as s: + s.connect(remote) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegexp(ssl.SSLError, "No cipher can be selected"): + with closing(socket.socket(socket.AF_INET)) as sock: + s = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(remote) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + # sha256.tbs-internet.com needs SNI to use the correct certificate + if not ssl.HAS_SNI: + self.skipTest("SNI needed for this test") + # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with support.transient_internet("sha256.tbs-internet.com"): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(sha256_cert) + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="sha256.tbs-internet.com") + try: + s.connect(remote) + if support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + self.assertEqual(len(ctx.get_ca_certs()), 1) + + @needs_sni + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + with support.transient_internet(REMOTE_HOST): + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with closing(ctx1.wrap_socket(s)) as ss: + ss.connect((REMOTE_HOST, 443)) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + + from test.ssl_servers import make_https_server + + class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(1) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) + self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) + except socket.error as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + if not isinstance(e, ssl.SSLError) and e.errno != errno.ECONNRESET: + raise + self.server.conn_errors.append(e) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.server.stop() + self.close() + return False + else: + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + sys.stdout.write(" cert binary is " + str(len(cert_binary)) + " bytes\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + sys.stdout.write(" server: selected protocol is now " + + str(self.sslconn.selected_npn_protocol()) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except ssl.SSLError: + if self.server.chatty: + handle_error("Test server failure:\n") + self.close() + self.running = False + # normally, we'd just stop here, but for the test + # harness, we want to stop the server + self.server.stop() + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + npn_protocols=None, alpn_protocols=None, + ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLSv1) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if npn_protocols: + self.context.set_npn_protocols(npn_protocols) + if alpn_protocols: + self.context.set_alpn_protocols(alpn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = support.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_npn_protocols = [] + self.selected_alpn_protocols = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.sock.settimeout(0.05) + self.sock.listen(5) + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except socket.timeout: + pass + except KeyboardInterrupt: + self.stop() + self.sock.close() + + def stop(self): + self.active = False + + class AsyncoreEchoServer(threading.Thread): + + class EchoServer(asyncore.dispatcher): + + class ConnectionHandler(asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = ssl.wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except socket.error, err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accept(self): + sock_obj, addr = self.accept() + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + + def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with closing(client_context.wrap_socket(socket.socket(), + server_hostname=sni_name)) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_alpn_protocol': s.selected_alpn_protocol(), + 'client_npn_protocol': s.selected_npn_protocol(), + 'version': s.version(), + }) + s.close() + stats['server_alpn_protocols'] = server.selected_alpn_protocols + stats['server_npn_protocols'] = server.selected_npn_protocols + return stats + + def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_SSLv23: + client_context.set_ciphers("ALL") + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(CERTFILE) + ctx.load_verify_locations(CERTFILE) + try: + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except socket.error as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) + + + class ThreadedTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + for protocol in PROTOCOLS: + context = ssl.SSLContext(protocol) + context.load_cert_chain(CERTFILE) + server_params_test(context, context, + chatty=True, connectionchatty=True) + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + s = context.wrap_socket(socket.socket(), + do_handshake_on_connect=False) + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + s.close() + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(SIGNING_CA) + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket())) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket())) as s: + with self.assertRaisesRegexp(ssl.SSLError, + "certificate verify failed"): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket())) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket(), + server_hostname="localhost")) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(context.wrap_socket(socket.socket(), + server_hostname="invalid")) as s: + with self.assertRaisesRegexp(ssl.CertificateError, + "hostname 'invalid' doesn't match u?'localhost'"): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with closing(socket.socket()) as s: + with self.assertRaisesRegexp(ValueError, + "check_hostname requires server_hostname"): + context.wrap_socket(s) + + def test_wrong_cert(self): + """Connecting when the server rejects the client's certificate + + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with a wrong client certificate fails. + """ + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + "wrongcert.pem") + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_REQUIRED, + cacerts=CERTFILE, chatty=False, + connectionchatty=False) + with server, \ + closing(socket.socket()) as sock, \ + closing(ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1)) as s: + try: + # Expect either an SSL error about the server rejecting + # the connection, or a low-level connection reset (which + # sometimes happens on Windows) + s.connect((HOST, server.port)) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except socket.error as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = support.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen(5) + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with closing(socket.socket()) as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = ssl.wrap_socket(c) + except socket.error: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") + def test_protocol_sslv2(self): + """Connecting to an SSLv2 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) + # SSLv23 client with specific SSL options + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv2) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv23(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) + except socket.error as x: + # this fails on some older versions of OpenSSL (0.9.7l, for instance) + if support.verbose: + sys.stdout.write( + " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" + % str(x)) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + + # Server with specific SSL options + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), + "OpenSSL is compiled without SSLv3 support") + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, + False, client_options=ssl.OP_NO_SSLv2) + + @skip_if_broken_ubuntu_ssl + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), + "TLS version 1.1 not supported.") + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), + "TLS version 1.2 not supported.") + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(1) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using a SocketServer to create and manage SSL connections.""" + server = make_https_server(self, certfile=CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + with open(CERTFILE, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = 'https://localhost:%d/%s' % ( + server.port, os.path.split(CERTFILE)[1]) + context = ssl.create_default_context(cafile=CERTFILE) + f = urllib2.urlopen(url, context=context) + try: + dlen = f.info().getheader("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = ssl.wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, whether to expect success, *args) + send_methods = [ + ('send', s.send, True, []), + ('sendto', s.sendto, False, ["some.address"]), + ('sendall', s.sendall, True, []), + ] + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = u"PREFIX_" + + for meth_name, send_meth, expect_success, args in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + send_meth(indata, *args) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # read(-1, buffer) is supported, even though read(-1) is not + data = b"data" + s.send(data) + buffer = bytearray(len(data)) + self.assertEqual(s.read(-1, buffer), len(data)) + self.assertEqual(buffer, data) + + s.write(b"over\n") + + self.assertRaises(ValueError, s.recv, -1) + self.assertRaises(ValueError, s.read, -1) + + s.close() + + def test_recv_zero(self): + server = ThreadedEchoServer(CERTFILE) + server.__enter__() + self.addCleanup(server.__exit__, None, None) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = ssl.wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + # recv/read(0) should return no data + s.send(b"data") + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.read(0), b"") + self.assertEqual(s.read(), b"data") + + # Should not block if the other end sends no data + s.setblocking(False) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.recv_into(bytearray()), 0) + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen(5) + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegexp(ssl.SSLError, "timed out", + ssl.wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = ssl.wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegexp(ssl.SSLError, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + server = context.wrap_socket(server, server_side=True) + + evt = threading.Event() + remote = [None] + peer = [None] + def serve(): + server.listen(5) + # Block on the accept and wait on the connection to close. + evt.set() + remote[0], peer[0] = server.accept() + remote[0].recv(1) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = context.wrap_socket(socket.socket()) + client.connect((host, port)) + client_addr = client.getsockname() + client.close() + t.join() + remote[0].close() + server.close() + # Sanity checks. + self.assertIsInstance(remote[0], ssl.SSLSocket) + self.assertEqual(peer[0], client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with closing(context.wrap_socket(socket.socket())) as sock: + with self.assertRaises(socket.error) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with closing(context.wrap_socket(socket.socket())) as sock: + with self.assertRaises(socket.error) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_default_ciphers(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + try: + # Force a set of weak ciphers on our client context + context.set_ciphers("DES") + except ssl.SSLError: + self.skipTest("no DES cipher available") + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_SSLv23, + chatty=False) as server: + with closing(context.wrap_socket(socket.socket())) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + self.assertIn("no shared cipher", str(server.conn_errors[0])) + + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + chatty=False) as server: + with closing(context.wrap_socket(socket.socket())) as s: + self.assertIs(s.version(), None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1') + self.assertIs(s.version(), None) + + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain(CERTFILE) + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + if ssl.OPENSSL_VERSION_INFO < (1, 0, 0): + context.set_ciphers("ECCdraft:ECDH") + with ThreadedEchoServer(context=context) as server: + with closing(context.wrap_socket(socket.socket())) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got channel binding data: {0!r}\n" + .format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + s.close() + + # now, again + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got another channel binding data: {0!r}\n" + .format(new_cb_data)) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + s.close() + + def test_compression(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['compression'], None) + + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.load_dh_params(DHFILE) + context.set_ciphers("kEDH") + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: + self.fail("Non-DH cipher: " + cipher[0]) + + def test_selected_alpn_protocol(self): + # selected_alpn_protocol() is None unless ALPN is used. + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support required") + def test_selected_alpn_protocol_if_server_uses_alpn(self): + # selected_alpn_protocol() is None unless ALPN is used by the client. + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_verify_locations(CERTFILE) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(['foo', 'bar']) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support needed for this test") + def test_alpn_protocols(self): + server_protocols = ['foo', 'bar', 'milkshake'] + protocol_tests = [ + (['foo', 'bar'], 'foo'), + (['bar', 'foo'], 'foo'), + (['milkshake'], 'milkshake'), + (['http/3.0', 'http/4.0'], None) + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + client_context.load_cert_chain(CERTFILE) + client_context.set_alpn_protocols(client_protocols) + + try: + stats = server_params_test(client_context, + server_context, + chatty=True, + connectionchatty=True) + except ssl.SSLError as e: + stats = e + + if expected is None and IS_OPENSSL_1_1: + # OpenSSL 1.1.0 raises handshake error + self.assertIsInstance(stats, ssl.SSLError) + else: + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_alpn_protocol'] + self.assertEqual(client_result, expected, + msg % (client_result, "client")) + server_result = stats['server_alpn_protocols'][-1] \ + if len(stats['server_alpn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, + msg % (server_result, "server")) + + def test_selected_npn_protocol(self): + # selected_npn_protocol() is None unless NPN is used + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_npn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test") + def test_npn_protocols(self): + server_protocols = ['http/1.1', 'spdy/2'] + protocol_tests = [ + (['http/1.1', 'spdy/2'], 'http/1.1'), + (['spdy/2', 'http/1.1'], 'http/1.1'), + (['spdy/2', 'test'], 'spdy/2'), + (['abc', 'def'], 'abc') + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_npn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_cert_chain(CERTFILE) + client_context.set_npn_protocols(client_protocols) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_npn_protocol'] + self.assertEqual(client_result, expected, msg % (client_result, "client")) + server_result = stats['server_npn_protocols'][-1] \ + if len(stats['server_npn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, msg % (server_result, "server")) + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + @needs_sni + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, 'localhost') + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, 'localhost') + self.assertEqual(calls, []) + + @needs_sni + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @needs_sni + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1.0/0.0 + server_context.set_servername_callback(cb_raising) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertIn("ZeroDivisionError", stderr.getvalue()) + + @needs_sni + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertIn("TypeError", stderr.getvalue()) + + def test_read_write_after_close_raises_valuerror(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + + with server: + s = context.wrap_socket(socket.socket()) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + +def test_main(verbose=False): + if support.verbose: + plats = { + 'Linux': platform.linux_distribution, + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + tests = [ContextTests, BasicTests, BasicSocketTests, SSLErrorTests] + + if support.is_resource_enabled('network'): + tests.append(NetworkedTests) + + if _have_threads: + thread_info = support.threading_setup() + if thread_info: + tests.append(ThreadedTests) + + try: + support.run_unittest(*tests) + finally: + if _have_threads: + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_subprocess.py b/src/greentest/2.7pypy/test_subprocess.py new file mode 100644 index 0000000..65212df --- /dev/null +++ b/src/greentest/2.7pypy/test_subprocess.py @@ -0,0 +1,1447 @@ +import unittest +from test import test_support +import subprocess +import sys +import signal +import os +import errno +import tempfile +import time +import re +import sysconfig + +try: + import resource +except ImportError: + resource = None +try: + import threading +except ImportError: + threading = None + +mswindows = (sys.platform == "win32") + +# +# Depends on the following external programs: Python +# + +#if mswindows: +# SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), ' +# 'os.O_BINARY);') +#else: +# SETBINARY = '' + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + # Try to minimize the number of children we have so this test + # doesn't crash on some buildbots (Alphas in particular). + test_support.reap_children() + + def tearDown(self): + for inst in subprocess._active: + inst.wait() + subprocess._cleanup() + self.assertFalse(subprocess._active, "subprocess._active not empty") + + def assertStderrEqual(self, stderr, expected, msg=None): + # In a debug build, stuff like "[6580 refs]" is printed to stderr at + # shutdown time. That frustrates tests trying to check stderr produced + # from a spawned Python process. + actual = re.sub(r"\[\d+ refs\]\r?\n?$", "", stderr) + self.assertEqual(actual, expected, msg) + + +class PopenTestException(Exception): + pass + + +class PopenExecuteChildRaises(subprocess.Popen): + """Popen subclass for testing cleanup of subprocess.PIPE filehandles when + _execute_child fails. + """ + def _execute_child(self, *args, **kwargs): + raise PopenTestException("Forced Exception for Test") + + +class ProcessTestCase(BaseTestCase): + + def test_call_seq(self): + # call() function with sequence argument + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(rc, 47) + + def test_check_call_zero(self): + # check_call() function with zero return code + rc = subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(0)"]) + self.assertEqual(rc, 0) + + def test_check_call_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(c.exception.returncode, 47) + + def test_check_output(self): + # check_output() function with zero return code + output = subprocess.check_output( + [sys.executable, "-c", "print 'BDFL'"]) + self.assertIn('BDFL', output) + + def test_check_output_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_output( + [sys.executable, "-c", "import sys; sys.exit(5)"]) + self.assertEqual(c.exception.returncode, 5) + + def test_check_output_stderr(self): + # check_output() function stderr redirected to stdout + output = subprocess.check_output( + [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], + stderr=subprocess.STDOUT) + self.assertIn('BDFL', output) + + def test_check_output_stdout_arg(self): + # check_output() function stderr redirected to stdout + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print 'will not be run'"], + stdout=sys.stdout) + self.fail("Expected ValueError when stdout arg supplied.") + self.assertIn('stdout', c.exception.args[0]) + + def test_call_kwargs(self): + # call() function with keyword args + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + rc = subprocess.call([sys.executable, "-c", + 'import sys, os;' + 'sys.exit(os.getenv("FRUIT")=="banana")'], + env=newenv) + self.assertEqual(rc, 1) + + def test_invalid_args(self): + # Popen() called with invalid arguments should raise TypeError + # but Popen.__del__ should not complain (issue #12085) + with test_support.captured_stderr() as s: + self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) + argcount = subprocess.Popen.__init__.__code__.co_argcount + too_many_args = [0] * (argcount + 1) + self.assertRaises(TypeError, subprocess.Popen, *too_many_args) + self.assertEqual(s.getvalue(), '') + + def test_stdin_none(self): + # .stdin is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print "banana"'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + p.wait() + self.assertEqual(p.stdin, None) + + def test_stdout_none(self): + # .stdout is None when not redirected, and the child's stdout will + # be inherited from the parent. In order to test this we run a + # subprocess in a subprocess: + # this_test + # \-- subprocess created by this test (parent) + # \-- subprocess created by the parent subprocess (child) + # The parent doesn't specify stdout, so the child will use the + # parent's stdout. This test checks that the message printed by the + # child goes to the parent stdout. The parent also checks that the + # child's stdout is None. See #11963. + code = ('import sys; from subprocess import Popen, PIPE;' + 'p = Popen([sys.executable, "-c", "print \'test_stdout_none\'"],' + ' stdin=PIPE, stderr=PIPE);' + 'p.wait(); assert p.stdout is None;') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), 'test_stdout_none') + + def test_stderr_none(self): + # .stderr is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print "banana"'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + p.wait() + self.assertEqual(p.stderr, None) + + def test_executable_with_cwd(self): + python_dir = os.path.dirname(os.path.realpath(sys.executable)) + p = subprocess.Popen(["somethingyoudonthave", "-c", + "import sys; sys.exit(47)"], + executable=sys.executable, cwd=python_dir) + p.wait() + self.assertEqual(p.returncode, 47) + + @unittest.skipIf(sysconfig.is_python_build(), + "need an installed Python. See #7774") + def test_executable_without_cwd(self): + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + p = subprocess.Popen(["somethingyoudonthave", "-c", + "import sys; sys.exit(47)"], + executable=sys.executable) + p.wait() + self.assertEqual(p.returncode, 47) + + def test_stdin_pipe(self): + # stdin redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.stdin.write("pear") + p.stdin.close() + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_filedes(self): + # stdin is set to open file descriptor + tf = tempfile.TemporaryFile() + d = tf.fileno() + os.write(d, "pear") + os.lseek(d, 0, 0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=d) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_fileobj(self): + # stdin is set to open file object + tf = tempfile.TemporaryFile() + tf.write("pear") + tf.seek(0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=tf) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdout_pipe(self): + # stdout redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), "orange") + + def test_stdout_filedes(self): + # stdout is set to open file descriptor + tf = tempfile.TemporaryFile() + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=d) + p.wait() + os.lseek(d, 0, 0) + self.assertEqual(os.read(d, 1024), "orange") + + def test_stdout_fileobj(self): + # stdout is set to open file object + tf = tempfile.TemporaryFile() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=tf) + p.wait() + tf.seek(0) + self.assertEqual(tf.read(), "orange") + + def test_stderr_pipe(self): + # stderr redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=subprocess.PIPE) + self.addCleanup(p.stderr.close) + self.assertStderrEqual(p.stderr.read(), "strawberry") + + def test_stderr_filedes(self): + # stderr is set to open file descriptor + tf = tempfile.TemporaryFile() + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=d) + p.wait() + os.lseek(d, 0, 0) + self.assertStderrEqual(os.read(d, 1024), "strawberry") + + def test_stderr_fileobj(self): + # stderr is set to open file object + tf = tempfile.TemporaryFile() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), "strawberry") + + def test_stderr_redirect_with_no_stdout_redirect(self): + # test stderr=STDOUT while stdout=None (not set) + + # - grandchild prints to stderr + # - child redirects grandchild's stderr to its stdout + # - the parent should get grandchild's stderr in child's stdout + p = subprocess.Popen([sys.executable, "-c", + 'import sys, subprocess;' + 'rc = subprocess.call([sys.executable, "-c",' + ' "import sys;"' + ' "sys.stderr.write(\'42\')"],' + ' stderr=subprocess.STDOUT);' + 'sys.exit(rc)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + #NOTE: stdout should get stderr from grandchild + self.assertStderrEqual(stdout, b'42') + self.assertStderrEqual(stderr, b'') # should be empty + self.assertEqual(p.returncode, 0) + + def test_stdout_stderr_pipe(self): + # capture stdout and stderr to the same pipe + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + self.addCleanup(p.stdout.close) + self.assertStderrEqual(p.stdout.read(), "appleorange") + + def test_stdout_stderr_file(self): + # capture stdout and stderr to the same open file + tf = tempfile.TemporaryFile() + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=tf, + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), "appleorange") + + def test_stdout_filedes_of_stdout(self): + # stdout is set to 1 (#1531862). + # To avoid printing the text on stdout, we do something similar to + # test_stdout_none (see above). The parent subprocess calls the child + # subprocess passing stdout=1, and this test uses stdout=PIPE in + # order to capture and check the output of the parent. See #11963. + code = ('import sys, subprocess; ' + 'rc = subprocess.call([sys.executable, "-c", ' + ' "import os, sys; sys.exit(os.write(sys.stdout.fileno(), ' + '\'test with stdout=1\'))"], stdout=1); ' + 'assert rc == 18') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), 'test with stdout=1') + + def test_cwd(self): + tmpdir = tempfile.gettempdir() + # We cannot use os.path.realpath to canonicalize the path, + # since it doesn't expand Tru64 {memb} strings. See bug 1063571. + cwd = os.getcwd() + os.chdir(tmpdir) + tmpdir = os.getcwd() + os.chdir(cwd) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getcwd())'], + stdout=subprocess.PIPE, + cwd=tmpdir) + self.addCleanup(p.stdout.close) + normcase = os.path.normcase + self.assertEqual(normcase(p.stdout.read()), normcase(tmpdir)) + + def test_env(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "orange" + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), "orange") + + def test_communicate_stdin(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.communicate("pear") + self.assertEqual(p.returncode, 1) + + def test_communicate_stdout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("pineapple")'], + stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "pineapple") + self.assertEqual(stderr, None) + + def test_communicate_stderr(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("pineapple")'], + stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertStderrEqual(stderr, "pineapple") + + def test_communicate(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple");' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate("banana") + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr, "pineapple") + + # This test is Linux specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + # Test for the fd leak reported in http://bugs.python.org/issue2791. + def test_communicate_pipe_fd_leak(self): + fd_directory = '/proc/%d/fd' % os.getpid() + num_fds_before_popen = len(os.listdir(fd_directory)) + p = subprocess.Popen([sys.executable, "-c", "print('')"], + stdout=subprocess.PIPE) + p.communicate() + num_fds_after_communicate = len(os.listdir(fd_directory)) + del p + num_fds_after_destruction = len(os.listdir(fd_directory)) + self.assertEqual(num_fds_before_popen, num_fds_after_destruction) + self.assertEqual(num_fds_before_popen, num_fds_after_communicate) + + def test_communicate_returns(self): + # communicate() should return None if no redirection is active + p = subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(47)"]) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertEqual(stderr, None) + + def test_communicate_pipe_buf(self): + # communicate() with writes larger than pipe_buf + # This test will probably deadlock rather than fail, if + # communicate() does not work properly. + x, y = os.pipe() + if mswindows: + pipe_buf = 512 + else: + pipe_buf = os.fpathconf(x, "PC_PIPE_BUF") + os.close(x) + os.close(y) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read(47));' + 'sys.stderr.write("xyz"*%d);' + 'sys.stdout.write(sys.stdin.read())' % pipe_buf], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + string_to_write = "abc"*pipe_buf + (stdout, stderr) = p.communicate(string_to_write) + self.assertEqual(stdout, string_to_write) + + def test_writes_before_communicate(self): + # stdin.write before communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.stdin.write("banana") + (stdout, stderr) = p.communicate("split") + self.assertEqual(stdout, "bananasplit") + self.assertStderrEqual(stderr, "") + + def test_universal_newlines(self): + # NB. replaced SETBINARY with the -u flag + p = subprocess.Popen([sys.executable, "-u", "-c", + 'import sys,os;' + #SETBINARY + + 'sys.stdout.write("line1\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line2\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("line3\\r\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line4\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline5");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline6");'], + stdout=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + stdout = p.stdout.read() + if hasattr(file, 'newlines'): + # Interpreter with universal newline support + self.assertEqual(stdout, + "line1\nline2\nline3\nline4\nline5\nline6") + else: + # Interpreter without universal newline support + self.assertEqual(stdout, + "line1\nline2\rline3\r\nline4\r\nline5\nline6") + + def test_universal_newlines_communicate(self): + # universal newlines through communicate() + # NB. replaced SETBINARY with the -u flag + p = subprocess.Popen([sys.executable, "-u", "-c", + 'import sys,os;' + #SETBINARY + + 'sys.stdout.write("line1\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line2\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("line3\\r\\n");' + 'sys.stdout.flush();' + 'sys.stdout.write("line4\\r");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline5");' + 'sys.stdout.flush();' + 'sys.stdout.write("\\nline6");'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate() + if hasattr(file, 'newlines'): + # Interpreter with universal newline support + self.assertEqual(stdout, + "line1\nline2\nline3\nline4\nline5\nline6") + else: + # Interpreter without universal newline support + self.assertEqual(stdout, + "line1\nline2\rline3\r\nline4\r\nline5\nline6") + + def test_no_leaking(self): + # Make sure we leak no resources + if not mswindows: + max_handles = 1026 # too much for most UNIX systems + else: + max_handles = 2050 # too much for (at least some) Windows setups + handles = [] + try: + for i in range(max_handles): + try: + handles.append(os.open(test_support.TESTFN, + os.O_WRONLY | os.O_CREAT)) + except OSError as e: + if e.errno != errno.EMFILE: + raise + break + else: + self.skipTest("failed to reach the file descriptor limit " + "(tried %d)" % max_handles) + # Close a couple of them (should be enough for a subprocess) + for i in range(10): + os.close(handles.pop()) + # Loop creating some subprocesses. If one of them leaks some fds, + # the next loop iteration will fail by reaching the max fd limit. + for i in range(15): + p = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write(sys.stdin.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + data = p.communicate(b"lime")[0] + self.assertEqual(data, b"lime") + finally: + for h in handles: + os.close(h) + test_support.unlink(test_support.TESTFN) + + def test_list2cmdline(self): + self.assertEqual(subprocess.list2cmdline(['a b c', 'd', 'e']), + '"a b c" d e') + self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']), + 'ab\\"c \\ d') + self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']), + 'ab\\"c " \\\\" d') + self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']), + 'a\\\\\\b "de fg" h') + self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']), + 'a\\\\\\"b c d') + self.assertEqual(subprocess.list2cmdline(['a\\\\b c', 'd', 'e']), + '"a\\\\b c" d e') + self.assertEqual(subprocess.list2cmdline(['a\\\\b\\ c', 'd', 'e']), + '"a\\\\b\\ c" d e') + self.assertEqual(subprocess.list2cmdline(['ab', '']), + 'ab ""') + + + def test_poll(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(1)"]) + count = 0 + while p.poll() is None: + time.sleep(0.1) + count += 1 + # We expect that the poll loop probably went around about 10 times, + # but, based on system scheduling we can't control, it's possible + # poll() never returned None. It "should be" very rare that it + # didn't go around at least twice. + self.assertGreaterEqual(count, 2) + # Subsequent invocations should just return the returncode + self.assertEqual(p.poll(), 0) + + + def test_wait(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(2)"]) + self.assertEqual(p.wait(), 0) + # Subsequent invocations should just return the returncode + self.assertEqual(p.wait(), 0) + + + def test_invalid_bufsize(self): + # an invalid type of the bufsize argument should raise + # TypeError. + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], "orange") + + def test_leaking_fds_on_error(self): + # see bug #5179: Popen leaks file descriptors to PIPEs if + # the child fails to execute; this will eventually exhaust + # the maximum number of open fds. 1024 seems a very common + # value for that limit, but Windows has 2048, so we loop + # 1024 times (each call leaked two fds). + for i in range(1024): + # Windows raises IOError. Others raise OSError. + with self.assertRaises(EnvironmentError) as c: + subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # ignore errors that indicate the command was not found + if c.exception.errno not in (errno.ENOENT, errno.EACCES): + raise c.exception + + @unittest.skipIf(threading is None, "threading required") + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(['nonexisting_i_hope'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + + def test_handles_closed_on_exception(self): + # If CreateProcess exits with an error, ensure the + # duplicate output handles are released + ifhandle, ifname = tempfile.mkstemp() + ofhandle, ofname = tempfile.mkstemp() + efhandle, efname = tempfile.mkstemp() + try: + subprocess.Popen (["*"], stdin=ifhandle, stdout=ofhandle, + stderr=efhandle) + except OSError: + os.close(ifhandle) + os.remove(ifname) + os.close(ofhandle) + os.remove(ofname) + os.close(efhandle) + os.remove(efname) + self.assertFalse(os.path.exists(ifname)) + self.assertFalse(os.path.exists(ofname)) + self.assertFalse(os.path.exists(efname)) + + def test_communicate_epipe(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.communicate("x" * 2**20) + + def test_communicate_epipe_only_stdin(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + time.sleep(2) + p.communicate("x" * 2**20) + + # This test is Linux-ish specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + def test_failed_child_execute_fd_leak(self): + """Test for the fork() failure fd leak reported in issue16327.""" + fd_directory = '/proc/%d/fd' % os.getpid() + fds_before_popen = os.listdir(fd_directory) + with self.assertRaises(PopenTestException): + PopenExecuteChildRaises( + [sys.executable, '-c', 'pass'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # NOTE: This test doesn't verify that the real _execute_child + # does not close the file descriptors itself on the way out + # during an exception. Code inspection has confirmed that. + + fds_after_exception = os.listdir(fd_directory) + self.assertEqual(fds_before_popen, fds_after_exception) + + +# context manager +class _SuppressCoreFiles(object): + """Try to prevent core files from being created.""" + old_limit = None + + def __enter__(self): + """Try to save previous ulimit, then set it to (0, 0).""" + if resource is not None: + try: + self.old_limit = resource.getrlimit(resource.RLIMIT_CORE) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + except (ValueError, resource.error): + pass + + if sys.platform == 'darwin': + # Check if the 'Crash Reporter' on OSX was configured + # in 'Developer' mode and warn that it will get triggered + # when it is. + # + # This assumes that this context manager is used in tests + # that might trigger the next manager. + value = subprocess.Popen(['/usr/bin/defaults', 'read', + 'com.apple.CrashReporter', 'DialogType'], + stdout=subprocess.PIPE).communicate()[0] + if value.strip() == b'developer': + print "this tests triggers the Crash Reporter, that is intentional" + sys.stdout.flush() + + def __exit__(self, *args): + """Return core file behavior to default.""" + if self.old_limit is None: + return + if resource is not None: + try: + resource.setrlimit(resource.RLIMIT_CORE, self.old_limit) + except (ValueError, resource.error): + pass + + @unittest.skipUnless(hasattr(signal, 'SIGALRM'), + "Requires signal.SIGALRM") + def test_communicate_eintr(self): + # Issue #12493: communicate() should handle EINTR + def handler(signum, frame): + pass + old_handler = signal.signal(signal.SIGALRM, handler) + self.addCleanup(signal.signal, signal.SIGALRM, old_handler) + + # the process is running for 2 seconds + args = [sys.executable, "-c", 'import time; time.sleep(2)'] + for stream in ('stdout', 'stderr'): + kw = {stream: subprocess.PIPE} + with subprocess.Popen(args, **kw) as process: + signal.alarm(1) + # communicate() will be interrupted by SIGALRM + process.communicate() + + +@unittest.skipIf(mswindows, "POSIX specific tests") +class POSIXProcessTestCase(BaseTestCase): + + def test_exceptions(self): + # caught & re-raised exceptions + with self.assertRaises(OSError) as c: + p = subprocess.Popen([sys.executable, "-c", ""], + cwd="/this/path/does/not/exist") + # The attribute child_traceback should contain "os.chdir" somewhere. + self.assertIn("os.chdir", c.exception.child_traceback) + + def test_run_abort(self): + # returncode handles signal termination + with _SuppressCoreFiles(): + p = subprocess.Popen([sys.executable, "-c", + "import os; os.abort()"]) + p.wait() + self.assertEqual(-p.returncode, signal.SIGABRT) + + def test_preexec(self): + # preexec function + p = subprocess.Popen([sys.executable, "-c", + "import sys, os;" + "sys.stdout.write(os.getenv('FRUIT'))"], + stdout=subprocess.PIPE, + preexec_fn=lambda: os.putenv("FRUIT", "apple")) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), "apple") + + class _TestExecuteChildPopen(subprocess.Popen): + """Used to test behavior at the end of _execute_child.""" + def __init__(self, testcase, *args, **kwargs): + self._testcase = testcase + subprocess.Popen.__init__(self, *args, **kwargs) + + def _execute_child( + self, args, executable, preexec_fn, close_fds, cwd, env, + universal_newlines, startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): + try: + subprocess.Popen._execute_child( + self, args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, + startupinfo, creationflags, shell, to_close, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + finally: + # Open a bunch of file descriptors and verify that + # none of them are the same as the ones the Popen + # instance is using for stdin/stdout/stderr. + devzero_fds = [os.open("/dev/zero", os.O_RDONLY) + for _ in range(8)] + try: + for fd in devzero_fds: + self._testcase.assertNotIn( + fd, (p2cwrite, c2pread, errread)) + finally: + for fd in devzero_fds: + os.close(fd) + + @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") + def test_preexec_errpipe_does_not_double_close_pipes(self): + """Issue16140: Don't double close pipes on preexec error.""" + + def raise_it(): + raise RuntimeError("force the _execute_child() errpipe_data path.") + + with self.assertRaises(RuntimeError): + self._TestExecuteChildPopen( + self, [sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=raise_it) + + def test_args_string(self): + # args is a string + f, fname = tempfile.mkstemp() + os.write(f, "#!/bin/sh\n") + os.write(f, "exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.close(f) + os.chmod(fname, 0o700) + p = subprocess.Popen(fname) + p.wait() + os.remove(fname) + self.assertEqual(p.returncode, 47) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + startupinfo=47) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + creationflags=47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen(["echo $FRUIT"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), "apple") + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen("echo $FRUIT", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), "apple") + + def test_call_string(self): + # call() function with string argument on UNIX + f, fname = tempfile.mkstemp() + os.write(f, "#!/bin/sh\n") + os.write(f, "exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.close(f) + os.chmod(fname, 0700) + rc = subprocess.call(fname) + os.remove(fname) + self.assertEqual(rc, 47) + + def test_specific_shell(self): + # Issue #9265: Incorrect name passed as arg[0]. + shells = [] + for prefix in ['/bin', '/usr/bin/', '/usr/local/bin']: + for name in ['bash', 'ksh']: + sh = os.path.join(prefix, name) + if os.path.isfile(sh): + shells.append(sh) + if not shells: # Will probably work for any shell but csh. + self.skipTest("bash or ksh required for this test") + sh = '/bin/sh' + if os.path.isfile(sh) and not os.path.islink(sh): + # Test will fail if /bin/sh is a symlink to csh. + shells.append(sh) + for sh in shells: + p = subprocess.Popen("echo $0", executable=sh, shell=True, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), sh) + + def _kill_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + return p + + @unittest.skipIf(sys.platform.startswith(('netbsd', 'openbsd')), + "Due to known OS bug (issue #16762)") + def _kill_dead_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + p.communicate() + + def test_send_signal(self): + p = self._kill_process('send_signal', signal.SIGINT) + _, stderr = p.communicate() + self.assertIn('KeyboardInterrupt', stderr) + self.assertNotEqual(p.wait(), 0) + + def test_kill(self): + p = self._kill_process('kill') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, '') + self.assertEqual(p.wait(), -signal.SIGKILL) + + def test_terminate(self): + p = self._kill_process('terminate') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, '') + self.assertEqual(p.wait(), -signal.SIGTERM) + + def test_send_signal_dead(self): + # Sending a signal to a dead process + self._kill_dead_process('send_signal', signal.SIGINT) + + def test_kill_dead(self): + # Killing a dead process + self._kill_dead_process('kill') + + def test_terminate_dead(self): + # Terminating a dead process + self._kill_dead_process('terminate') + + def check_close_std_fds(self, fds): + # Issue #9905: test that subprocess pipes still work properly with + # some standard fds closed + stdin = 0 + newfds = [] + for a in fds: + b = os.dup(a) + newfds.append(b) + if a == 0: + stdin = b + try: + for fd in fds: + os.close(fd) + out, err = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + err = test_support.strip_python_stderr(err) + self.assertEqual((out, err), (b'apple', b'orange')) + finally: + for b, a in zip(newfds, fds): + os.dup2(b, a) + for b in newfds: + os.close(b) + + def test_close_fd_0(self): + self.check_close_std_fds([0]) + + def test_close_fd_1(self): + self.check_close_std_fds([1]) + + def test_close_fd_2(self): + self.check_close_std_fds([2]) + + def test_close_fds_0_1(self): + self.check_close_std_fds([0, 1]) + + def test_close_fds_0_2(self): + self.check_close_std_fds([0, 2]) + + def test_close_fds_1_2(self): + self.check_close_std_fds([1, 2]) + + def test_close_fds_0_1_2(self): + # Issue #10806: test that subprocess pipes still work properly with + # all standard fds closed. + self.check_close_std_fds([0, 1, 2]) + + def check_swap_fds(self, stdin_no, stdout_no, stderr_no): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + temp_fds = [fd for fd, fname in temps] + try: + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # save a copy of the standard file descriptors + saved_fds = [os.dup(fd) for fd in range(3)] + try: + # duplicate the temp files over the standard fd's 0, 1, 2 + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # write some data to what will become stdin, and rewind + os.write(stdin_no, b"STDIN") + os.lseek(stdin_no, 0, 0) + + # now use those files in the given order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=stdin_no, + stdout=stdout_no, + stderr=stderr_no) + p.wait() + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(stdout_no, 1024) + err = test_support.strip_python_stderr(os.read(stderr_no, 1024)) + finally: + for std, saved in enumerate(saved_fds): + os.dup2(saved, std) + os.close(saved) + + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + # When duping fds, if there arises a situation where one of the fds is + # either 0, 1 or 2, it is possible that it is overwritten (#12607). + # This tests all combinations of this. + def test_swap_fds(self): + self.check_swap_fds(0, 1, 2) + self.check_swap_fds(0, 2, 1) + self.check_swap_fds(1, 0, 2) + self.check_swap_fds(1, 2, 0) + self.check_swap_fds(2, 0, 1) + self.check_swap_fds(2, 1, 0) + + def test_wait_when_sigchild_ignored(self): + # NOTE: sigchild_ignore.py may not be an effective test on all OSes. + sigchild_ignore = test_support.findfile("sigchild_ignore.py", + subdir="subprocessdata") + p = subprocess.Popen([sys.executable, sigchild_ignore], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" + " non-zero with this error:\n%s" % stderr) + + def test_zombie_fast_process_del(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, it wouldn't be added to subprocess._active, and would + # remain a zombie. + # spawn a Popen, and delete its reference before it exits + p = subprocess.Popen([sys.executable, "-c", + 'import sys, time;' + 'time.sleep(0.2)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + test_support.gc_collect() + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + def test_leak_fast_process_del_killed(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, and the process got killed by a signal, it would never + # be removed from subprocess._active, which triggered a FD and memory + # leak. + # spawn a Popen, delete its reference and kill it + p = subprocess.Popen([sys.executable, "-c", + 'import time;' + 'time.sleep(3)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + test_support.gc_collect() + os.kill(pid, signal.SIGKILL) + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + # let some time for the process to exit, and create a new Popen: this + # should trigger the wait() of p + time.sleep(0.2) + with self.assertRaises(EnvironmentError) as c: + with subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + # p should have been wait()ed on, and removed from the _active list + self.assertRaises(OSError, os.waitpid, pid, 0) + self.assertNotIn(ident, [id(o) for o in subprocess._active]) + + def test_pipe_cloexec(self): + # Issue 12786: check that the communication pipes' FDs are set CLOEXEC, + # and are not inherited by another child process. + p1 = subprocess.Popen([sys.executable, "-c", + 'import os;' + 'os.read(0, 1)' + ], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + p2 = subprocess.Popen([sys.executable, "-c", """if True: + import os, errno, sys + for fd in %r: + try: + os.close(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + sys.exit(1) + sys.exit(0) + """ % [f.fileno() for f in (p1.stdin, p1.stdout, + p1.stderr)] + ], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=False) + p1.communicate('foo') + _, stderr = p2.communicate() + + self.assertEqual(p2.returncode, 0, "Unexpected error: " + repr(stderr)) + + +@unittest.skipUnless(mswindows, "Windows specific tests") +class Win32ProcessTestCase(BaseTestCase): + + def test_startupinfo(self): + # startupinfo argument + # We uses hardcoded constants, because we do not want to + # depend on win32all. + STARTF_USESHOWWINDOW = 1 + SW_MAXIMIZE = 3 + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = STARTF_USESHOWWINDOW + startupinfo.wShowWindow = SW_MAXIMIZE + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_creationflags(self): + # creationflags argument + CREATE_NEW_CONSOLE = 16 + sys.stderr.write(" a DOS box should flash briefly ...\n") + subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + preexec_fn=lambda: 1) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + stdout=subprocess.PIPE, + close_fds=True) + + def test_close_fds(self): + # close file descriptors + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"], + close_fds=True) + self.assertEqual(rc, 47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen(["set"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn("physalis", p.stdout.read()) + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn("physalis", p.stdout.read()) + + def test_call_string(self): + # call() function with string argument on Windows + rc = subprocess.call(sys.executable + + ' -c "import sys; sys.exit(47)"') + self.assertEqual(rc, 47) + + def _kill_process(self, method, *args): + # Some win32 buildbot raises EOFError if stdin is inherited + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, '') + returncode = p.wait() + self.assertNotEqual(returncode, 0) + + def _kill_dead_process(self, method, *args): + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + sys.exit(42) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() + self.assertEqual(rc, 42) + + def test_send_signal(self): + self._kill_process('send_signal', signal.SIGTERM) + + def test_kill(self): + self._kill_process('kill') + + def test_terminate(self): + self._kill_process('terminate') + + def test_send_signal_dead(self): + self._kill_dead_process('send_signal', signal.SIGTERM) + + def test_kill_dead(self): + self._kill_dead_process('kill') + + def test_terminate_dead(self): + self._kill_dead_process('terminate') + + +@unittest.skipUnless(getattr(subprocess, '_has_poll', False), + "poll system call not supported") +class ProcessTestCaseNoPoll(ProcessTestCase): + def setUp(self): + subprocess._has_poll = False + ProcessTestCase.setUp(self) + + def tearDown(self): + subprocess._has_poll = True + ProcessTestCase.tearDown(self) + + +class HelperFunctionTests(unittest.TestCase): + @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows") + def test_eintr_retry_call(self): + record_calls = [] + def fake_os_func(*args): + record_calls.append(args) + if len(record_calls) == 2: + raise OSError(errno.EINTR, "fake interrupted system call") + return tuple(reversed(args)) + + self.assertEqual((999, 256), + subprocess._eintr_retry_call(fake_os_func, 256, 999)) + self.assertEqual([(256, 999)], record_calls) + # This time there will be an EINTR so it will loop once. + self.assertEqual((666,), + subprocess._eintr_retry_call(fake_os_func, 666)) + self.assertEqual([(256, 999), (666,), (666,)], record_calls) + +@unittest.skipUnless(mswindows, "mswindows only") +class CommandsWithSpaces (BaseTestCase): + + def setUp(self): + super(CommandsWithSpaces, self).setUp() + f, fname = tempfile.mkstemp(".py", "te st") + self.fname = fname.lower () + os.write(f, b"import sys;" + b"sys.stdout.write('%d %s' % (len(sys.argv), [a.lower () for a in sys.argv]))" + ) + os.close(f) + + def tearDown(self): + os.remove(self.fname) + super(CommandsWithSpaces, self).tearDown() + + def with_spaces(self, *args, **kwargs): + kwargs['stdout'] = subprocess.PIPE + p = subprocess.Popen(*args, **kwargs) + self.addCleanup(p.stdout.close) + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) + + def test_shell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd"), shell=1) + + def test_shell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) + + def test_noshell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd")) + + def test_noshell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"]) + +def test_main(): + unit_tests = (ProcessTestCase, + POSIXProcessTestCase, + Win32ProcessTestCase, + ProcessTestCaseNoPoll, + HelperFunctionTests, + CommandsWithSpaces) + + test_support.run_unittest(*unit_tests) + test_support.reap_children() + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_telnetlib.py b/src/greentest/2.7pypy/test_telnetlib.py new file mode 100644 index 0000000..1534e14 --- /dev/null +++ b/src/greentest/2.7pypy/test_telnetlib.py @@ -0,0 +1,461 @@ +import socket +import telnetlib +import time +import Queue + +import unittest +from unittest import TestCase +from test import test_support +threading = test_support.import_module('threading') + +HOST = test_support.HOST +EOF_sigil = object() + +def server(evt, serv, dataq=None): + """ Open a tcp server in three steps + 1) set evt to true to let the parent know we are ready + 2) [optional] if is not False, write the list of data from dataq.get() + to the socket. + """ + serv.listen(5) + evt.set() + try: + conn, addr = serv.accept() + if dataq: + data = '' + new_data = dataq.get(True, 0.5) + dataq.task_done() + for item in new_data: + if item == EOF_sigil: + break + if type(item) in [int, float]: + time.sleep(item) + else: + data += item + written = conn.send(data) + data = data[written:] + conn.close() + except socket.timeout: + pass + finally: + serv.close() + +class GeneralTests(TestCase): + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(60) # Safety net. Look issue 11812 + self.port = test_support.bind_port(self.sock) + self.thread = threading.Thread(target=server, args=(self.evt,self.sock)) + self.thread.setDaemon(True) + self.thread.start() + self.evt.wait() + + def tearDown(self): + self.thread.join() + + def testBasic(self): + # connects + telnet = telnetlib.Telnet(HOST, self.port) + telnet.sock.close() + + def testTimeoutDefault(self): + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + telnet = telnetlib.Telnet(HOST, self.port) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testTimeoutNone(self): + # None, having other default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + telnet = telnetlib.Telnet(HOST, self.port, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertTrue(telnet.sock.gettimeout() is None) + telnet.sock.close() + + def testTimeoutValue(self): + telnet = telnetlib.Telnet(HOST, self.port, timeout=30) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testTimeoutOpen(self): + telnet = telnetlib.Telnet() + telnet.open(HOST, self.port, timeout=30) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testGetters(self): + # Test telnet getter methods + telnet = telnetlib.Telnet(HOST, self.port, timeout=30) + t_sock = telnet.sock + self.assertEqual(telnet.get_socket(), t_sock) + self.assertEqual(telnet.fileno(), t_sock.fileno()) + telnet.sock.close() + +def _read_setUp(self): + self.evt = threading.Event() + self.dataq = Queue.Queue() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(10) + self.port = test_support.bind_port(self.sock) + self.thread = threading.Thread(target=server, args=(self.evt,self.sock, self.dataq)) + self.thread.start() + self.evt.wait() + +def _read_tearDown(self): + self.thread.join() + +class ReadTests(TestCase): + setUp = _read_setUp + tearDown = _read_tearDown + + # use a similar approach to testing timeouts as test_timeout.py + # these will never pass 100% but make the fuzz big enough that it is rare + block_long = 0.6 + block_short = 0.3 + def test_read_until_A(self): + """ + read_until(expected, [timeout]) + Read until the expected string has been seen, or a timeout is + hit (default is no timeout); may block. + """ + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + data = telnet.read_until('match') + self.assertEqual(data, ''.join(want[:-2])) + + def test_read_until_B(self): + # test the timeout - it does NOT raise socket.timeout + want = ['hello', self.block_long, 'not seen', EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + data = telnet.read_until('not seen', self.block_short) + self.assertEqual(data, want[0]) + self.assertEqual(telnet.read_all(), 'not seen') + + def test_read_until_with_poll(self): + """Use select.poll() to implement telnet.read_until().""" + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + if not telnet._has_poll: + raise unittest.SkipTest('select.poll() is required') + telnet._has_poll = True + self.dataq.join() + data = telnet.read_until('match') + self.assertEqual(data, ''.join(want[:-2])) + + def test_read_until_with_select(self): + """Use select.select() to implement telnet.read_until().""" + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + telnet._has_poll = False + self.dataq.join() + data = telnet.read_until('match') + self.assertEqual(data, ''.join(want[:-2])) + + def test_read_all_A(self): + """ + read_all() + Read all data until EOF; may block. + """ + want = ['x' * 500, 'y' * 500, 'z' * 500, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + data = telnet.read_all() + self.assertEqual(data, ''.join(want[:-1])) + + def _test_blocking(self, func): + self.dataq.put([self.block_long, EOF_sigil]) + self.dataq.join() + start = time.time() + data = func() + self.assertTrue(self.block_short <= time.time() - start) + + def test_read_all_B(self): + self._test_blocking(telnetlib.Telnet(HOST, self.port).read_all) + + def test_read_all_C(self): + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + telnet.read_all() + telnet.read_all() # shouldn't raise + + def test_read_some_A(self): + """ + read_some() + Read at least one byte or EOF; may block. + """ + # test 'at least one byte' + want = ['x' * 500, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + data = telnet.read_all() + self.assertTrue(len(data) >= 1) + + def test_read_some_B(self): + # test EOF + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + self.assertEqual('', telnet.read_some()) + + def test_read_some_C(self): + self._test_blocking(telnetlib.Telnet(HOST, self.port).read_some) + + def _test_read_any_eager_A(self, func_name): + """ + read_very_eager() + Read all data available already queued or on the socket, + without blocking. + """ + want = [self.block_long, 'x' * 100, 'y' * 100, EOF_sigil] + expects = want[1] + want[2] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + func = getattr(telnet, func_name) + data = '' + while True: + try: + data += func() + self.assertTrue(expects.startswith(data)) + except EOFError: + break + self.assertEqual(expects, data) + + def _test_read_any_eager_B(self, func_name): + # test EOF + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + time.sleep(self.block_short) + func = getattr(telnet, func_name) + self.assertRaises(EOFError, func) + + # read_eager and read_very_eager make the same guarantees + # (they behave differently but we only test the guarantees) + def test_read_very_eager_A(self): + self._test_read_any_eager_A('read_very_eager') + def test_read_very_eager_B(self): + self._test_read_any_eager_B('read_very_eager') + def test_read_eager_A(self): + self._test_read_any_eager_A('read_eager') + def test_read_eager_B(self): + self._test_read_any_eager_B('read_eager') + # NB -- we need to test the IAC block which is mentioned in the docstring + # but not in the module docs + + def _test_read_any_lazy_B(self, func_name): + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + func = getattr(telnet, func_name) + telnet.fill_rawq() + self.assertRaises(EOFError, func) + + def test_read_lazy_A(self): + want = ['x' * 100, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + time.sleep(self.block_short) + self.assertEqual('', telnet.read_lazy()) + data = '' + while True: + try: + read_data = telnet.read_lazy() + data += read_data + if not read_data: + telnet.fill_rawq() + except EOFError: + break + self.assertTrue(want[0].startswith(data)) + self.assertEqual(data, want[0]) + + def test_read_lazy_B(self): + self._test_read_any_lazy_B('read_lazy') + + def test_read_very_lazy_A(self): + want = ['x' * 100, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + time.sleep(self.block_short) + self.assertEqual('', telnet.read_very_lazy()) + data = '' + while True: + try: + read_data = telnet.read_very_lazy() + except EOFError: + break + data += read_data + if not read_data: + telnet.fill_rawq() + self.assertEqual('', telnet.cookedq) + telnet.process_rawq() + self.assertTrue(want[0].startswith(data)) + self.assertEqual(data, want[0]) + + def test_read_very_lazy_B(self): + self._test_read_any_lazy_B('read_very_lazy') + +class nego_collector(object): + def __init__(self, sb_getter=None): + self.seen = '' + self.sb_getter = sb_getter + self.sb_seen = '' + + def do_nego(self, sock, cmd, opt): + self.seen += cmd + opt + if cmd == tl.SE and self.sb_getter: + sb_data = self.sb_getter() + self.sb_seen += sb_data + +tl = telnetlib +class OptionTests(TestCase): + setUp = _read_setUp + tearDown = _read_tearDown + # RFC 854 commands + cmds = [tl.AO, tl.AYT, tl.BRK, tl.EC, tl.EL, tl.GA, tl.IP, tl.NOP] + + def _test_command(self, data): + """ helper for testing IAC + cmd """ + self.setUp() + self.dataq.put(data) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + nego = nego_collector() + telnet.set_option_negotiation_callback(nego.do_nego) + txt = telnet.read_all() + cmd = nego.seen + self.assertTrue(len(cmd) > 0) # we expect at least one command + self.assertIn(cmd[0], self.cmds) + self.assertEqual(cmd[1], tl.NOOPT) + self.assertEqual(len(''.join(data[:-1])), len(txt + cmd)) + nego.sb_getter = None # break the nego => telnet cycle + self.tearDown() + + def test_IAC_commands(self): + # reset our setup + self.dataq.put([EOF_sigil]) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + self.tearDown() + + for cmd in self.cmds: + self._test_command(['x' * 100, tl.IAC + cmd, 'y'*100, EOF_sigil]) + self._test_command(['x' * 10, tl.IAC + cmd, 'y'*10, EOF_sigil]) + self._test_command([tl.IAC + cmd, EOF_sigil]) + # all at once + self._test_command([tl.IAC + cmd for (cmd) in self.cmds] + [EOF_sigil]) + self.assertEqual('', telnet.read_sb_data()) + + def test_SB_commands(self): + # RFC 855, subnegotiations portion + send = [tl.IAC + tl.SB + tl.IAC + tl.SE, + tl.IAC + tl.SB + tl.IAC + tl.IAC + tl.IAC + tl.SE, + tl.IAC + tl.SB + tl.IAC + tl.IAC + 'aa' + tl.IAC + tl.SE, + tl.IAC + tl.SB + 'bb' + tl.IAC + tl.IAC + tl.IAC + tl.SE, + tl.IAC + tl.SB + 'cc' + tl.IAC + tl.IAC + 'dd' + tl.IAC + tl.SE, + EOF_sigil, + ] + self.dataq.put(send) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + nego = nego_collector(telnet.read_sb_data) + telnet.set_option_negotiation_callback(nego.do_nego) + txt = telnet.read_all() + self.assertEqual(txt, '') + want_sb_data = tl.IAC + tl.IAC + 'aabb' + tl.IAC + 'cc' + tl.IAC + 'dd' + self.assertEqual(nego.sb_seen, want_sb_data) + self.assertEqual('', telnet.read_sb_data()) + nego.sb_getter = None # break the nego => telnet cycle + + +class ExpectTests(TestCase): + def setUp(self): + self.evt = threading.Event() + self.dataq = Queue.Queue() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(10) + self.port = test_support.bind_port(self.sock) + self.thread = threading.Thread(target=server, args=(self.evt,self.sock, + self.dataq)) + self.thread.start() + self.evt.wait() + + def tearDown(self): + self.thread.join() + + # use a similar approach to testing timeouts as test_timeout.py + # these will never pass 100% but make the fuzz big enough that it is rare + block_long = 0.6 + block_short = 0.3 + def test_expect_A(self): + """ + expect(expected, [timeout]) + Read until the expected string has been seen, or a timeout is + hit (default is no timeout); may block. + """ + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + (_,_,data) = telnet.expect(['match']) + self.assertEqual(data, ''.join(want[:-2])) + + def test_expect_B(self): + # test the timeout - it does NOT raise socket.timeout + want = ['hello', self.block_long, 'not seen', EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + self.dataq.join() + (_,_,data) = telnet.expect(['not seen'], self.block_short) + self.assertEqual(data, want[0]) + self.assertEqual(telnet.read_all(), 'not seen') + + def test_expect_with_poll(self): + """Use select.poll() to implement telnet.expect().""" + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + if not telnet._has_poll: + raise unittest.SkipTest('select.poll() is required') + telnet._has_poll = True + self.dataq.join() + (_,_,data) = telnet.expect(['match']) + self.assertEqual(data, ''.join(want[:-2])) + + def test_expect_with_select(self): + """Use select.select() to implement telnet.expect().""" + want = ['x' * 10, 'match', 'y' * 10, EOF_sigil] + self.dataq.put(want) + telnet = telnetlib.Telnet(HOST, self.port) + telnet._has_poll = False + self.dataq.join() + (_,_,data) = telnet.expect(['match']) + self.assertEqual(data, ''.join(want[:-2])) + + +def test_main(verbose=None): + test_support.run_unittest(GeneralTests, ReadTests, OptionTests, + ExpectTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7pypy/test_thread.py b/src/greentest/2.7pypy/test_thread.py new file mode 100644 index 0000000..40e6566 --- /dev/null +++ b/src/greentest/2.7pypy/test_thread.py @@ -0,0 +1,267 @@ +import os +import unittest +import random +from test import test_support +thread = test_support.import_module('thread') +import time +import sys +import weakref + +from test import lock_tests + +NUMTASKS = 10 +NUMTRIPS = 3 + + +_print_mutex = thread.allocate_lock() + +def verbose_print(arg): + """Helper function for printing out debugging output.""" + if test_support.verbose: + with _print_mutex: + print arg + + +class BasicThreadTest(unittest.TestCase): + + def setUp(self): + self.done_mutex = thread.allocate_lock() + self.done_mutex.acquire() + self.running_mutex = thread.allocate_lock() + self.random_mutex = thread.allocate_lock() + self.created = 0 + self.running = 0 + self.next_ident = 0 + + +class ThreadRunningTests(BasicThreadTest): + + def newtask(self): + with self.running_mutex: + self.next_ident += 1 + verbose_print("creating task %s" % self.next_ident) + thread.start_new_thread(self.task, (self.next_ident,)) + self.created += 1 + self.running += 1 + + def task(self, ident): + with self.random_mutex: + delay = random.random() / 10000.0 + verbose_print("task %s will run for %sus" % (ident, round(delay*1e6))) + time.sleep(delay) + verbose_print("task %s done" % ident) + with self.running_mutex: + self.running -= 1 + if self.created == NUMTASKS and self.running == 0: + self.done_mutex.release() + + def test_starting_threads(self): + # Basic test for thread creation. + for i in range(NUMTASKS): + self.newtask() + verbose_print("waiting for tasks to complete...") + self.done_mutex.acquire() + verbose_print("all tasks done") + + def test_stack_size(self): + # Various stack size tests. + self.assertEqual(thread.stack_size(), 0, "initial stack size is not 0") + + thread.stack_size(0) + self.assertEqual(thread.stack_size(), 0, "stack_size not reset to default") + + @unittest.skipIf(os.name not in ("nt", "os2", "posix"), 'test meant for nt, os2, and posix') + def test_nt_and_posix_stack_size(self): + try: + thread.stack_size(4096) + except ValueError: + verbose_print("caught expected ValueError setting " + "stack_size(4096)") + except thread.error: + self.skipTest("platform does not support changing thread stack " + "size") + + fail_msg = "stack_size(%d) failed - should succeed" + for tss in (262144, 0x100000, 0): + thread.stack_size(tss) + self.assertEqual(thread.stack_size(), tss, fail_msg % tss) + verbose_print("successfully set stack_size(%d)" % tss) + + for tss in (262144, 0x100000): + verbose_print("trying stack_size = (%d)" % tss) + self.next_ident = 0 + self.created = 0 + for i in range(NUMTASKS): + self.newtask() + + verbose_print("waiting for all tasks to complete") + self.done_mutex.acquire() + verbose_print("all tasks done") + + thread.stack_size(0) + + def test__count(self): + # Test the _count() function. + orig = thread._count() + mut = thread.allocate_lock() + mut.acquire() + started = [] + def task(): + started.append(None) + mut.acquire() + mut.release() + thread.start_new_thread(task, ()) + while not started: + time.sleep(0.01) + self.assertEqual(thread._count(), orig + 1) + # Allow the task to finish. + mut.release() + # The only reliable way to be sure that the thread ended from the + # interpreter's point of view is to wait for the function object to be + # destroyed. + done = [] + wr = weakref.ref(task, lambda _: done.append(None)) + del task + while not done: + time.sleep(0.01) + test_support.gc_collect() + self.assertEqual(thread._count(), orig) + + def test_save_exception_state_on_error(self): + # See issue #14474 + def task(): + started.release() + raise SyntaxError + def mywrite(self, *args): + try: + raise ValueError + except ValueError: + pass + real_write(self, *args) + c = thread._count() + started = thread.allocate_lock() + with test_support.captured_output("stderr") as stderr: + real_write = stderr.write + stderr.write = mywrite + started.acquire() + thread.start_new_thread(task, ()) + started.acquire() + while thread._count() > c: + time.sleep(0.01) + self.assertIn("Traceback", stderr.getvalue()) + + +class Barrier: + def __init__(self, num_threads): + self.num_threads = num_threads + self.waiting = 0 + self.checkin_mutex = thread.allocate_lock() + self.checkout_mutex = thread.allocate_lock() + self.checkout_mutex.acquire() + + def enter(self): + self.checkin_mutex.acquire() + self.waiting = self.waiting + 1 + if self.waiting == self.num_threads: + self.waiting = self.num_threads - 1 + self.checkout_mutex.release() + return + self.checkin_mutex.release() + + self.checkout_mutex.acquire() + self.waiting = self.waiting - 1 + if self.waiting == 0: + self.checkin_mutex.release() + return + self.checkout_mutex.release() + + +class BarrierTest(BasicThreadTest): + + def test_barrier(self): + self.bar = Barrier(NUMTASKS) + self.running = NUMTASKS + for i in range(NUMTASKS): + thread.start_new_thread(self.task2, (i,)) + verbose_print("waiting for tasks to end") + self.done_mutex.acquire() + verbose_print("tasks done") + + def task2(self, ident): + for i in range(NUMTRIPS): + if ident == 0: + # give it a good chance to enter the next + # barrier before the others are all out + # of the current one + delay = 0 + else: + with self.random_mutex: + delay = random.random() / 10000.0 + verbose_print("task %s will run for %sus" % + (ident, round(delay * 1e6))) + time.sleep(delay) + verbose_print("task %s entering %s" % (ident, i)) + self.bar.enter() + verbose_print("task %s leaving barrier" % ident) + with self.running_mutex: + self.running -= 1 + # Must release mutex before releasing done, else the main thread can + # exit and set mutex to None as part of global teardown; then + # mutex.release() raises AttributeError. + finished = self.running == 0 + if finished: + self.done_mutex.release() + + +class LockTests(lock_tests.LockTests): + locktype = thread.allocate_lock + + +class TestForkInThread(unittest.TestCase): + def setUp(self): + self.read_fd, self.write_fd = os.pipe() + + @unittest.skipIf(sys.platform.startswith('win'), + "This test is only appropriate for POSIX-like systems.") + @test_support.reap_threads + def test_forkinthread(self): + def thread1(): + try: + pid = os.fork() # fork in a thread + except RuntimeError: + sys.exit(0) # exit the child + + if pid == 0: # child + os.close(self.read_fd) + os.write(self.write_fd, "OK") + # Exiting the thread normally in the child process can leave + # any additional threads (such as the one started by + # importing _tkinter) still running, and this can prevent + # the half-zombie child process from being cleaned up. See + # Issue #26456. + os._exit(0) + else: # parent + os.close(self.write_fd) + + thread.start_new_thread(thread1, ()) + self.assertEqual(os.read(self.read_fd, 2), "OK", + "Unable to fork() in thread") + + def tearDown(self): + try: + os.close(self.read_fd) + except OSError: + pass + + try: + os.close(self.write_fd) + except OSError: + pass + + +def test_main(): + test_support.run_unittest(ThreadRunningTests, BarrierTest, LockTests, + TestForkInThread) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_threading.py b/src/greentest/2.7pypy/test_threading.py new file mode 100644 index 0000000..04ed4a6 --- /dev/null +++ b/src/greentest/2.7pypy/test_threading.py @@ -0,0 +1,945 @@ +# Very rudimentary test of threading module +print("GEVENT: Begin import") +import test.test_support +from test.test_support import verbose, cpython_only +from test.script_helper import assert_python_ok + +import random +import re +import sys +thread = test.test_support.import_module('thread') +threading = test.test_support.import_module('threading') +import time +import unittest +import weakref +import os +import subprocess +try: + import _testcapi +except ImportError: + _testcapi = None +except: + # gevent: a distutils.errors.LinkError is sometimes raised. + # It appears that it happens during concurrent test runs; + # some lib_pypy/_testcapimodule.o file is truncated + _testcapi = None + +from gevent.tests import lock_tests # gevent: use local copy + +# A trivial mutable counter. +class Counter(object): + def __init__(self): + self.value = 0 + def inc(self): + self.value += 1 + def dec(self): + self.value -= 1 + def get(self): + return self.value + +class TestThread(threading.Thread): + def __init__(self, name, testcase, sema, mutex, nrunning): + threading.Thread.__init__(self, name=name) + self.testcase = testcase + self.sema = sema + self.mutex = mutex + self.nrunning = nrunning + + def run(self): + delay = random.random() / 10000.0 + if verbose: + print 'task %s will run for %.1f usec' % ( + self.name, delay * 1e6) + + with self.sema: + with self.mutex: + self.nrunning.inc() + if verbose: + print self.nrunning.get(), 'tasks are running' + self.testcase.assertLessEqual(self.nrunning.get(), 3) + + time.sleep(delay) + if verbose: + print 'task', self.name, 'done' + + with self.mutex: + self.nrunning.dec() + self.testcase.assertGreaterEqual(self.nrunning.get(), 0) + if verbose: + print '%s is finished. %d tasks are running' % ( + self.name, self.nrunning.get()) + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test.test_support.threading_setup() + + def tearDown(self): + test.test_support.threading_cleanup(*self._threads) + test.test_support.reap_children() + + +class ThreadTests(BaseTestCase): + + # Create a bunch of threads, let each do some work, wait until all are + # done. + def test_various_ops(self): + # This takes about n/3 seconds to run (about n/3 clumps of tasks, + # times about 1 second per clump). + NUMTASKS = 10 + + # no more than 3 of the 10 can run at once + sema = threading.BoundedSemaphore(value=3) + mutex = threading.RLock() + numrunning = Counter() + + threads = [] + + for i in range(NUMTASKS): + t = TestThread(""%i, self, sema, mutex, numrunning) + threads.append(t) + self.assertIsNone(t.ident) + self.assertRegexpMatches(repr(t), r'^$') + t.start() + + if verbose: + print 'waiting for all tasks to complete' + for t in threads: + t.join(NUMTASKS) + self.assertFalse(t.is_alive()) + self.assertNotEqual(t.ident, 0) + self.assertIsNotNone(t.ident) + self.assertRegexpMatches(repr(t), r'^$') + if verbose: + print 'all tasks done' + self.assertEqual(numrunning.get(), 0) + + def test_ident_of_no_threading_threads(self): + # The ident still must work for the main thread and dummy threads. + self.assertIsNotNone(threading.currentThread().ident) + def f(): + ident.append(threading.currentThread().ident) + done.set() + done = threading.Event() + ident = [] + thread.start_new_thread(f, ()) + done.wait() + self.assertIsNotNone(ident[0]) + # Kill the "immortal" _DummyThread + del threading._active[ident[0]] + + # run with a small(ish) thread stack size (256kB) + def test_various_ops_small_stack(self): + if verbose: + print 'with 256kB thread stack size...' + try: + threading.stack_size(262144) + except thread.error: + self.skipTest('platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1MB) + def test_various_ops_large_stack(self): + if verbose: + print 'with 1MB thread stack size...' + try: + threading.stack_size(0x100000) + except thread.error: + self.skipTest('platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + def test_foreign_thread(self): + # Check that a "foreign" thread can use the threading module. + def f(mutex): + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + threading.current_thread() + mutex.release() + + mutex = threading.Lock() + mutex.acquire() + tid = thread.start_new_thread(f, (mutex,)) + # Wait for the thread to finish. + mutex.acquire() + self.assertIn(tid, threading._active) + self.assertIsInstance(threading._active[tid], threading._DummyThread) + del threading._active[tid] + + # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) + # exposed at the Python level. This test relies on ctypes to get at it. + @test.test_support.cpython_only + def test_PyThreadState_SetAsyncExc(self): + try: + import ctypes + except ImportError: + self.skipTest('requires ctypes') + + set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc + + class AsyncExc(Exception): + pass + + exception = ctypes.py_object(AsyncExc) + + # First check it works when setting the exception from the same thread. + tid = thread.get_ident() + + try: + result = set_async_exc(ctypes.c_long(tid), exception) + # The exception is async, so we might have to keep the VM busy until + # it notices. + while True: + pass + except AsyncExc: + pass + else: + # This code is unreachable but it reflects the intent. If we wanted + # to be smarter the above loop wouldn't be infinite. + self.fail("AsyncExc not raised") + try: + self.assertEqual(result, 1) # one thread state modified + except UnboundLocalError: + # The exception was raised too quickly for us to get the result. + pass + + # `worker_started` is set by the thread when it's inside a try/except + # block waiting to catch the asynchronously set AsyncExc exception. + # `worker_saw_exception` is set by the thread upon catching that + # exception. + worker_started = threading.Event() + worker_saw_exception = threading.Event() + + class Worker(threading.Thread): + def run(self): + self.id = thread.get_ident() + self.finished = False + + try: + while True: + worker_started.set() + time.sleep(0.1) + except AsyncExc: + self.finished = True + worker_saw_exception.set() + + t = Worker() + t.daemon = True # so if this fails, we don't hang Python at shutdown + t.start() + if verbose: + print " started worker thread" + + # Try a thread id that doesn't make sense. + if verbose: + print " trying nonsensical thread id" + result = set_async_exc(ctypes.c_long(-1), exception) + self.assertEqual(result, 0) # no thread states modified + + # Now raise an exception in the worker thread. + if verbose: + print " waiting for worker thread to get started" + ret = worker_started.wait() + self.assertTrue(ret) + if verbose: + print " verifying worker hasn't exited" + self.assertFalse(t.finished) + if verbose: + print " attempting to raise asynch exception in worker" + result = set_async_exc(ctypes.c_long(t.id), exception) + self.assertEqual(result, 1) # one thread state modified + if verbose: + print " waiting for worker to say it caught the exception" + worker_saw_exception.wait(timeout=10) + self.assertTrue(t.finished) + if verbose: + print " all OK -- joining worker" + if t.finished: + t.join() + # else the thread is still running, and we have no way to kill it + + def test_limbo_cleanup(self): + # Issue 7481: Failure to start thread should cleanup the limbo map. + def fail_new_thread(*args): + raise thread.error() + _start_new_thread = threading._start_new_thread + threading._start_new_thread = fail_new_thread + try: + t = threading.Thread(target=lambda: None) + self.assertRaises(thread.error, t.start) + self.assertFalse( + t in threading._limbo, + "Failed to cleanup _limbo map on failure of Thread.start().") + finally: + threading._start_new_thread = _start_new_thread + + @test.test_support.cpython_only + def test_finalize_runnning_thread(self): + # Issue 1402: the PyGILState_Ensure / _Release functions may be called + # very late on python exit: on deallocation of a running thread for + # example. + try: + import ctypes + except ImportError: + self.skipTest('requires ctypes') + + rc = subprocess.call([sys.executable, "-c", """if 1: + import ctypes, sys, time, thread + + # This lock is used as a simple event variable. + ready = thread.allocate_lock() + ready.acquire() + + # Module globals are cleared before __del__ is run + # So we save the functions in class dict + class C: + ensure = ctypes.pythonapi.PyGILState_Ensure + release = ctypes.pythonapi.PyGILState_Release + def __del__(self): + state = self.ensure() + self.release(state) + + def waitingThread(): + x = C() + ready.release() + time.sleep(100) + + thread.start_new_thread(waitingThread, ()) + ready.acquire() # Be sure the other thread is waiting. + sys.exit(42) + """]) + self.assertEqual(rc, 42) + + def test_finalize_with_trace(self): + # Issue1733757 + # Avoid a deadlock when sys.settrace steps into threading._shutdown + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, threading + + # A deadlock-killer, to prevent the + # testsuite to hang forever + def killer(): + import os, time + time.sleep(2) + print 'program blocked; aborting' + os._exit(2) + t = threading.Thread(target=killer) + t.daemon = True + t.start() + + # This is the trace function + def func(frame, event, arg): + threading.current_thread() + return func + + sys.settrace(func) + """], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + stdout, stderr = p.communicate() + rc = p.returncode + self.assertFalse(rc == 2, "interpreted was blocked") + self.assertTrue(rc == 0, + "Unexpected error: " + repr(stderr)) + + def test_join_nondaemon_on_shutdown(self): + # Issue 1722344 + # Raising SystemExit skipped threading._shutdown + p = subprocess.Popen([sys.executable, "-c", """if 1: + import threading + from time import sleep + + def child(): + sleep(1) + # As a non-daemon thread we SHOULD wake up and nothing + # should be torn down yet + print "Woke up, sleep function is:", sleep + + threading.Thread(target=child).start() + raise SystemExit + """], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + stdout, stderr = p.communicate() + self.assertEqual(stdout.strip(), + "Woke up, sleep function is: ") + stderr = re.sub(r"^\[\d+ refs\]", "", stderr, re.MULTILINE).strip() + self.assertEqual(stderr, "") + + def test_enumerate_after_join(self): + # Try hard to trigger #1703448: a thread is still returned in + # threading.enumerate() after it has been join()ed. + enum = threading.enumerate + old_interval = sys.getcheckinterval() + try: + for i in xrange(1, 100): + # Try a couple times at each thread-switching interval + # to get more interleavings. + sys.setcheckinterval(i // 5) + t = threading.Thread(target=lambda: None) + t.start() + t.join() + l = enum() + self.assertNotIn(t, l, + "#1703448 triggered after %d trials: %s" % (i, l)) + finally: + sys.setcheckinterval(old_interval) + + @test.test_support.cpython_only + def test_no_refcycle_through_target(self): + class RunSelfFunction(object): + def __init__(self, should_raise): + # The links in this refcycle from Thread back to self + # should be cleaned up when the thread completes. + self.should_raise = should_raise + self.thread = threading.Thread(target=self._run, + args=(self,), + kwargs={'yet_another':self}) + self.thread.start() + + def _run(self, other_ref, yet_another): + if self.should_raise: + raise SystemExit + + cyclic_object = RunSelfFunction(should_raise=False) + weak_cyclic_object = weakref.ref(cyclic_object) + cyclic_object.thread.join() + del cyclic_object + self.assertEqual(None, weak_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_cyclic_object()))) + + raising_cyclic_object = RunSelfFunction(should_raise=True) + weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) + raising_cyclic_object.thread.join() + del raising_cyclic_object + self.assertEqual(None, weak_raising_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_raising_cyclic_object()))) + + @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + def test_dummy_thread_after_fork(self): + # Issue #14308: a dummy thread in the active list doesn't mess up + # the after-fork mechanism. + code = """if 1: + import thread, threading, os, time + + def background_thread(evt): + # Creates and registers the _DummyThread instance + threading.current_thread() + evt.set() + time.sleep(10) + + evt = threading.Event() + thread.start_new_thread(background_thread, (evt,)) + evt.wait() + assert threading.active_count() == 2, threading.active_count() + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + os.wait() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out, '') + self.assertEqual(err, '') + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + old_interval = sys.getcheckinterval() + + # Make the bug more likely to manifest. + sys.setcheckinterval(10) + + try: + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + pid = os.fork() + if pid == 0: + os._exit(1 if t.is_alive() else 0) + else: + t.join() + pid, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + finally: + sys.setcheckinterval(old_interval) + + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) + +class ThreadJoinOnShutdown(BaseTestCase): + + # Between fork() and exec(), only async-safe functions are allowed (issues + # #12316 and #11870), and fork() from a worker thread is known to trigger + # problems with some operating systems (issue #3863): skip problematic tests + # on platforms known to behave badly. + platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', + 'os2emx') + + def _run_and_join(self, script): + script = """if 1: + import sys, os, time, threading + + # a thread, which waits for the main program to terminate + def joiningfunc(mainthread): + mainthread.join() + print 'end of thread' + # stdout is fully buffered because not a tty, we have to flush + # before exit. + sys.stdout.flush() + \n""" + script + + p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE) + rc = p.wait() + data = p.stdout.read().replace('\r', '') + p.stdout.close() + self.assertEqual(data, "end of main\nend of thread\n") + self.assertFalse(rc == 2, "interpreter was blocked") + self.assertTrue(rc == 0, "Unexpected error") + + def test_1_join_on_shutdown(self): + # The usual case: on exit, wait for a non-daemon thread + script = """if 1: + import os + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + time.sleep(0.1) + print 'end of main' + """ + self._run_and_join(script) + + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_2_join_in_forked_process(self): + # Like the test above, but from a forked interpreter + script = """if 1: + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + print 'end of main' + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_3_join_in_forked_from_thread(self): + # Like the test above, but fork() was called from a worker thread + # In the forked process, the main Thread object must be marked as stopped. + script = """if 1: + main_thread = threading.current_thread() + def worker(): + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(main_thread,)) + print 'end of main' + t.start() + t.join() # Should not block: main_thread is already stopped + + w = threading.Thread(target=worker) + w.start() + """ + self._run_and_join(script) + + def assertScriptHasOutput(self, script, expected_output): + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE) + rc = p.wait() + data = p.stdout.read().decode().replace('\r', '') + self.assertEqual(rc, 0, "Unexpected error") + self.assertEqual(data, expected_output) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_4_joining_across_fork_in_worker_thread(self): + # There used to be a possible deadlock when forking from a child + # thread. See http://bugs.python.org/issue6643. + + # The script takes the following steps: + # - The main thread in the parent process starts a new thread and then + # tries to join it. + # - The join operation acquires the Lock inside the thread's _block + # Condition. (See threading.py:Thread.join().) + # - We stub out the acquire method on the condition to force it to wait + # until the child thread forks. (See LOCK ACQUIRED HERE) + # - The child thread forks. (See LOCK HELD and WORKER THREAD FORKS + # HERE) + # - The main thread of the parent process enters Condition.wait(), + # which releases the lock on the child thread. + # - The child process returns. Without the necessary fix, when the + # main thread of the child process (which used to be the child thread + # in the parent process) attempts to exit, it will try to acquire the + # lock in the Thread._block Condition object and hang, because the + # lock was held across the fork. + + script = """if 1: + import os, time, threading + + finish_join = False + start_fork = False + + def worker(): + # Wait until this thread's lock is acquired before forking to + # create the deadlock. + global finish_join + while not start_fork: + time.sleep(0.01) + # LOCK HELD: Main thread holds lock across this call. + childpid = os.fork() + finish_join = True + if childpid != 0: + # Parent process just waits for child. + os.waitpid(childpid, 0) + # Child process should just return. + + w = threading.Thread(target=worker) + + # Stub out the private condition variable's lock acquire method. + # This acquires the lock and then waits until the child has forked + # before returning, which will release the lock soon after. If + # someone else tries to fix this test case by acquiring this lock + # before forking instead of resetting it, the test case will + # deadlock when it shouldn't. + condition = w._block + orig_acquire = condition.acquire + call_count_lock = threading.Lock() + call_count = 0 + def my_acquire(): + global call_count + global start_fork + orig_acquire() # LOCK ACQUIRED HERE + start_fork = True + if call_count == 0: + while not finish_join: + time.sleep(0.01) # WORKER THREAD FORKS HERE + with call_count_lock: + call_count += 1 + condition.acquire = my_acquire + + w.start() + w.join() + print('end of main') + """ + self.assertScriptHasOutput(script, "end of main\n") + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_5_clear_waiter_locks_to_avoid_crash(self): + # Check that a spawned thread that forks doesn't segfault on certain + # platforms, namely OS X. This used to happen if there was a waiter + # lock in the thread's condition variable's waiters list. Even though + # we know the lock will be held across the fork, it is not safe to + # release locks held across forks on all platforms, so releasing the + # waiter lock caused a segfault on OS X. Furthermore, since locks on + # OS X are (as of this writing) implemented with a mutex + condition + # variable instead of a semaphore, while we know that the Python-level + # lock will be acquired, we can't know if the internal mutex will be + # acquired at the time of the fork. + + script = """if True: + import os, time, threading + + start_fork = False + + def worker(): + # Wait until the main thread has attempted to join this thread + # before continuing. + while not start_fork: + time.sleep(0.01) + childpid = os.fork() + if childpid != 0: + # Parent process just waits for child. + (cpid, rc) = os.waitpid(childpid, 0) + assert cpid == childpid + assert rc == 0 + print('end of worker thread') + else: + # Child process should just return. + pass + + w = threading.Thread(target=worker) + + # Stub out the private condition variable's _release_save method. + # This releases the condition's lock and flips the global that + # causes the worker to fork. At this point, the problematic waiter + # lock has been acquired once by the waiter and has been put onto + # the waiters list. + condition = w._block + orig_release_save = condition._release_save + def my_release_save(): + global start_fork + orig_release_save() + # Waiter lock held here, condition lock released. + start_fork = True + condition._release_save = my_release_save + + w.start() + w.join() + print('end of main thread') + """ + output = "end of worker thread\nend of main thread\n" + self.assertScriptHasOutput(script, output) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_reinit_tls_after_fork(self): + # Issue #13817: fork() would deadlock in a multithreaded program with + # the ad-hoc TLS implementation. + + def do_fork_and_wait(): + # just fork a child process and wait it + pid = os.fork() + if pid > 0: + os.waitpid(pid, 0) + else: + os._exit(0) + + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() + + for t in threads: + t.join() + + @cpython_only + @unittest.skipIf(_testcapi is None, "need _testcapi module") + def test_frame_tstate_tracing(self): + # Issue #14432: Crash when a generator is created in a C thread that is + # destroyed while the generator is still used. The issue was that a + # generator contains a frame, and the frame kept a reference to the + # Python state of the destroyed C thread. The crash occurs when a trace + # function is setup. + + def noop_trace(frame, event, arg): + # no operation + return noop_trace + + def generator(): + while 1: + yield "generator" + + def callback(): + if callback.gen is None: + callback.gen = generator() + return next(callback.gen) + callback.gen = None + + old_trace = sys.gettrace() + sys.settrace(noop_trace) + try: + # Install a trace function + threading.settrace(noop_trace) + + # Create a generator in a C thread which exits after the call + _testcapi.call_in_temporary_c_thread(callback) + + # Call the generator in a different Python thread, check that the + # generator didn't keep a reference to the destroyed thread state + for test in range(3): + # The trace function is still called here + callback() + finally: + sys.settrace(old_trace) + + +class ThreadingExceptionTests(BaseTestCase): + # A RuntimeError should be raised if Thread.start() is called + # multiple times. + def test_start_thread_again(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, thread.start) + + def test_joining_current_thread(self): + current_thread = threading.current_thread() + self.assertRaises(RuntimeError, current_thread.join); + + def test_joining_inactive_thread(self): + thread = threading.Thread() + self.assertRaises(RuntimeError, thread.join) + + def test_daemonize_active_thread(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + + def test_print_exception(self): + script = r"""if 1: + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1.0/0.0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, '') + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_1(self): + script = r"""if 1: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1.0/0.0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + sys.stderr = None + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, '') + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_2(self): + script = r"""if 1: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1.0/0.0 + sys.stderr = None + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, '') + self.assertNotIn("Unhandled exception", err) + + +class LockTests(lock_tests.LockTests): + locktype = staticmethod(threading.Lock) + +class RLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading.RLock) + +class EventTests(lock_tests.EventTests): + eventtype = staticmethod(threading.Event) + +class ConditionAsRLockTests(lock_tests.RLockTests): + # Condition uses an RLock by default and exports its API. + locktype = staticmethod(threading.Condition) + +class ConditionTests(lock_tests.ConditionTests): + condtype = staticmethod(threading.Condition) + +class SemaphoreTests(lock_tests.SemaphoreTests): + semtype = staticmethod(threading.Semaphore) + +class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): + semtype = staticmethod(threading.BoundedSemaphore) + + @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem') + def test_recursion_limit(self): + # Issue 9670 + # test that excessive recursion within a non-main thread causes + # an exception rather than crashing the interpreter on platforms + # like Mac OS X or FreeBSD which have small default stack sizes + # for threads + script = """if True: + import threading + + def recurse(): + return recurse() + + def outer(): + try: + recurse() + except RuntimeError: + pass + + w = threading.Thread(target=outer) + w.start() + w.join() + print('end of main thread') + """ + expected_output = "end of main thread\n" + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + data = stdout.decode().replace('\r', '') + self.assertEqual(p.returncode, 0, "Unexpected error") + self.assertEqual(data, expected_output) + +def test_main(): + test.test_support.run_unittest(LockTests, RLockTests, EventTests, + ConditionAsRLockTests, ConditionTests, + SemaphoreTests, BoundedSemaphoreTests, + ThreadTests, + ThreadJoinOnShutdown, + ThreadingExceptionTests, + ) + +if __name__ == "__main__": + print("GEVENT: Begin main") + test_main() diff --git a/src/greentest/2.7pypy/test_threading_local.py b/src/greentest/2.7pypy/test_threading_local.py new file mode 100644 index 0000000..1f8ad81 --- /dev/null +++ b/src/greentest/2.7pypy/test_threading_local.py @@ -0,0 +1,230 @@ +import unittest +from doctest import DocTestSuite +from test import test_support as support +import weakref +import gc + +# Modules under test +_thread = support.import_module('thread') +threading = support.import_module('threading') +import _threading_local + + +class Weak(object): + pass + +def target(local, weaklist): + weak = Weak() + local.weak = weak + weaklist.append(weakref.ref(weak)) + +class BaseLocalTest: + + def test_local_refs(self): + self._local_refs(20) + self._local_refs(50) + self._local_refs(100) + + def _local_refs(self, n): + local = self._local() + weaklist = [] + for i in range(n): + t = threading.Thread(target=target, args=(local, weaklist)) + t.start() + t.join() + del t + + gc.collect() + self.assertEqual(len(weaklist), n) + + # XXX _threading_local keeps the local of the last stopped thread alive. + deadlist = [weak for weak in weaklist if weak() is None] + self.assertIn(len(deadlist), (n-1, n)) + + # Assignment to the same thread local frees it sometimes (!) + local.someothervar = None + gc.collect() + deadlist = [weak for weak in weaklist if weak() is None] + self.assertIn(len(deadlist), (n-1, n), (n, len(deadlist))) + + def test_derived(self): + # Issue 3088: if there is a threads switch inside the __init__ + # of a threading.local derived class, the per-thread dictionary + # is created but not correctly set on the object. + # The first member set may be bogus. + import time + class Local(self._local): + def __init__(self): + time.sleep(0.01) + local = Local() + + def f(i): + local.x = i + # Simply check that the variable is correctly set + self.assertEqual(local.x, i) + + with support.start_threads(threading.Thread(target=f, args=(i,)) + for i in range(10)): + pass + + def test_derived_cycle_dealloc(self): + # http://bugs.python.org/issue6990 + class Local(self._local): + pass + locals = None + passed = [False] + e1 = threading.Event() + e2 = threading.Event() + + def f(): + # 1) Involve Local in a cycle + cycle = [Local()] + cycle.append(cycle) + cycle[0].foo = 'bar' + + # 2) GC the cycle (triggers threadmodule.c::local_clear + # before local_dealloc) + del cycle + gc.collect() + e1.set() + e2.wait() + + # 4) New Locals should be empty + passed[0] = all(not hasattr(local, 'foo') for local in locals) + + t = threading.Thread(target=f) + t.start() + e1.wait() + + # 3) New Locals should recycle the original's address. Creating + # them in the thread overwrites the thread state and avoids the + # bug + locals = [Local() for i in range(10)] + e2.set() + t.join() + + self.assertTrue(passed[0]) + + def test_arguments(self): + # Issue 1522237 + from thread import _local as local + from _threading_local import local as py_local + + for cls in (local, py_local): + class MyLocal(cls): + def __init__(self, *args, **kwargs): + pass + + MyLocal(a=1) + MyLocal(1) + self.assertRaises(TypeError, cls, a=1) + self.assertRaises(TypeError, cls, 1) + + def _test_one_class(self, c): + self._failed = "No error message set or cleared." + obj = c() + e1 = threading.Event() + e2 = threading.Event() + + def f1(): + obj.x = 'foo' + obj.y = 'bar' + del obj.y + e1.set() + e2.wait() + + def f2(): + try: + foo = obj.x + except AttributeError: + # This is expected -- we haven't set obj.x in this thread yet! + self._failed = "" # passed + else: + self._failed = ('Incorrectly got value %r from class %r\n' % + (foo, c)) + sys.stderr.write(self._failed) + + t1 = threading.Thread(target=f1) + t1.start() + e1.wait() + t2 = threading.Thread(target=f2) + t2.start() + t2.join() + # The test is done; just let t1 know it can exit, and wait for it. + e2.set() + t1.join() + + self.assertFalse(self._failed, self._failed) + + def test_threading_local(self): + self._test_one_class(self._local) + + def test_threading_local_subclass(self): + class LocalSubclass(self._local): + """To test that subclasses behave properly.""" + self._test_one_class(LocalSubclass) + + def _test_dict_attribute(self, cls): + obj = cls() + obj.x = 5 + self.assertEqual(obj.__dict__, {'x': 5}) + if support.check_impl_detail(): + with self.assertRaises(AttributeError): + obj.__dict__ = {} + with self.assertRaises(AttributeError): + del obj.__dict__ + + def test_dict_attribute(self): + self._test_dict_attribute(self._local) + + def test_dict_attribute_subclass(self): + class LocalSubclass(self._local): + """To test that subclasses behave properly.""" + self._test_dict_attribute(LocalSubclass) + + +class ThreadLocalTest(unittest.TestCase, BaseLocalTest): + _local = _thread._local + + # Fails for the pure Python implementation + def test_cycle_collection(self): + class X: + pass + + x = X() + x.local = self._local() + x.local.x = x + wr = weakref.ref(x) + del x + gc.collect() + self.assertIsNone(wr()) + +class PyThreadingLocalTest(unittest.TestCase, BaseLocalTest): + _local = _threading_local.local + + +def test_main(): + suite = unittest.TestSuite() + suite.addTest(DocTestSuite('_threading_local')) + suite.addTest(unittest.makeSuite(ThreadLocalTest)) + suite.addTest(unittest.makeSuite(PyThreadingLocalTest)) + + try: + from thread import _local + except ImportError: + pass + else: + import _threading_local + local_orig = _threading_local.local + def setUp(test): + _threading_local.local = _local + def tearDown(test): + _threading_local.local = local_orig + suite.addTest(DocTestSuite('_threading_local', + setUp=setUp, tearDown=tearDown) + ) + + support.run_unittest(suite) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7pypy/test_timeout.py b/src/greentest/2.7pypy/test_timeout.py new file mode 100644 index 0000000..bb9252d --- /dev/null +++ b/src/greentest/2.7pypy/test_timeout.py @@ -0,0 +1,205 @@ +"""Unit tests for socket timeout feature.""" + +import unittest +from test import test_support + +# This requires the 'network' resource as given on the regrtest command line. +skip_expected = not test_support.is_resource_enabled('network') + +import time +import socket + + +class CreationTestCase(unittest.TestCase): + """Test case for socket.gettimeout() and socket.settimeout()""" + + def setUp(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def tearDown(self): + self.sock.close() + + def testObjectCreation(self): + # Test Socket creation + self.assertEqual(self.sock.gettimeout(), None, + "timeout not disabled by default") + + def testFloatReturnValue(self): + # Test return value of gettimeout() + self.sock.settimeout(7.345) + self.assertEqual(self.sock.gettimeout(), 7.345) + + self.sock.settimeout(3) + self.assertEqual(self.sock.gettimeout(), 3) + + self.sock.settimeout(None) + self.assertEqual(self.sock.gettimeout(), None) + + def testReturnType(self): + # Test return type of gettimeout() + self.sock.settimeout(1) + self.assertEqual(type(self.sock.gettimeout()), type(1.0)) + + self.sock.settimeout(3.9) + self.assertEqual(type(self.sock.gettimeout()), type(1.0)) + + def testTypeCheck(self): + # Test type checking by settimeout() + self.sock.settimeout(0) + self.sock.settimeout(0L) + self.sock.settimeout(0.0) + self.sock.settimeout(None) + self.assertRaises(TypeError, self.sock.settimeout, "") + self.assertRaises(TypeError, self.sock.settimeout, u"") + self.assertRaises(TypeError, self.sock.settimeout, ()) + self.assertRaises(TypeError, self.sock.settimeout, []) + self.assertRaises(TypeError, self.sock.settimeout, {}) + self.assertRaises(TypeError, self.sock.settimeout, 0j) + + def testRangeCheck(self): + # Test range checking by settimeout() + self.assertRaises(ValueError, self.sock.settimeout, -1) + self.assertRaises(ValueError, self.sock.settimeout, -1L) + self.assertRaises(ValueError, self.sock.settimeout, -1.0) + + def testTimeoutThenBlocking(self): + # Test settimeout() followed by setblocking() + self.sock.settimeout(10) + self.sock.setblocking(1) + self.assertEqual(self.sock.gettimeout(), None) + self.sock.setblocking(0) + self.assertEqual(self.sock.gettimeout(), 0.0) + + self.sock.settimeout(10) + self.sock.setblocking(0) + self.assertEqual(self.sock.gettimeout(), 0.0) + self.sock.setblocking(1) + self.assertEqual(self.sock.gettimeout(), None) + + def testBlockingThenTimeout(self): + # Test setblocking() followed by settimeout() + self.sock.setblocking(0) + self.sock.settimeout(1) + self.assertEqual(self.sock.gettimeout(), 1) + + self.sock.setblocking(1) + self.sock.settimeout(1) + self.assertEqual(self.sock.gettimeout(), 1) + + +class TimeoutTestCase(unittest.TestCase): + """Test case for socket.socket() timeout functions""" + + # There are a number of tests here trying to make sure that an operation + # doesn't take too much longer than expected. But competing machine + # activity makes it inevitable that such tests will fail at times. + # When fuzz was at 1.0, I (tim) routinely saw bogus failures on Win2K + # and Win98SE. Boosting it to 2.0 helped a lot, but isn't a real + # solution. + fuzz = 2.0 + + def setUp(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addr_remote = ('www.python.org.', 80) + self.localhost = '127.0.0.1' + + def tearDown(self): + self.sock.close() + + def testConnectTimeout(self): + # Choose a private address that is unlikely to exist to prevent + # failures due to the connect succeeding before the timeout. + # Use a dotted IP address to avoid including the DNS lookup time + # with the connect time. This avoids failing the assertion that + # the timeout occurred fast enough. + addr = ('10.0.0.0', 12345) + + # Test connect() timeout + _timeout = 0.001 + self.sock.settimeout(_timeout) + + _t1 = time.time() + self.assertRaises(socket.error, self.sock.connect, addr) + _t2 = time.time() + + _delta = abs(_t1 - _t2) + self.assertTrue(_delta < _timeout + self.fuzz, + "timeout (%g) is more than %g seconds more than expected (%g)" + %(_delta, self.fuzz, _timeout)) + + def testRecvTimeout(self): + # Test recv() timeout + _timeout = 0.02 + + with test_support.transient_internet(self.addr_remote[0]): + self.sock.connect(self.addr_remote) + self.sock.settimeout(_timeout) + + _t1 = time.time() + self.assertRaises(socket.timeout, self.sock.recv, 1024) + _t2 = time.time() + + _delta = abs(_t1 - _t2) + self.assertTrue(_delta < _timeout + self.fuzz, + "timeout (%g) is %g seconds more than expected (%g)" + %(_delta, self.fuzz, _timeout)) + + def testAcceptTimeout(self): + # Test accept() timeout + _timeout = 2 + self.sock.settimeout(_timeout) + # Prevent "Address already in use" socket exceptions + test_support.bind_port(self.sock, self.localhost) + self.sock.listen(5) + + _t1 = time.time() + self.assertRaises(socket.error, self.sock.accept) + _t2 = time.time() + + _delta = abs(_t1 - _t2) + self.assertTrue(_delta < _timeout + self.fuzz, + "timeout (%g) is %g seconds more than expected (%g)" + %(_delta, self.fuzz, _timeout)) + + def testRecvfromTimeout(self): + # Test recvfrom() timeout + _timeout = 2 + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.settimeout(_timeout) + # Prevent "Address already in use" socket exceptions + test_support.bind_port(self.sock, self.localhost) + + _t1 = time.time() + self.assertRaises(socket.error, self.sock.recvfrom, 8192) + _t2 = time.time() + + _delta = abs(_t1 - _t2) + self.assertTrue(_delta < _timeout + self.fuzz, + "timeout (%g) is %g seconds more than expected (%g)" + %(_delta, self.fuzz, _timeout)) + + @unittest.skip('test not implemented') + def testSend(self): + # Test send() timeout + # couldn't figure out how to test it + pass + + @unittest.skip('test not implemented') + def testSendto(self): + # Test sendto() timeout + # couldn't figure out how to test it + pass + + @unittest.skip('test not implemented') + def testSendall(self): + # Test sendall() timeout + # couldn't figure out how to test it + pass + + +def test_main(): + test_support.requires('network') + test_support.run_unittest(CreationTestCase, TimeoutTestCase) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_urllib.py b/src/greentest/2.7pypy/test_urllib.py new file mode 100644 index 0000000..7b1ef6a --- /dev/null +++ b/src/greentest/2.7pypy/test_urllib.py @@ -0,0 +1,1111 @@ +"""Regression tests for urllib""" + +import collections +import urllib +import httplib +import io +import unittest +import os +import sys +import mimetools +import tempfile + +from test import test_support +from base64 import b64encode + + +def hexescape(char): + """Escape char as RFC 2396 specifies""" + hex_repr = hex(ord(char))[2:].upper() + if len(hex_repr) == 1: + hex_repr = "0%s" % hex_repr + return "%" + hex_repr + + +def fakehttp(fakedata): + class FakeSocket(io.BytesIO): + + def sendall(self, data): + FakeHTTPConnection.buf = data + + def makefile(self, *args, **kwds): + return self + + def read(self, amt=None): + if self.closed: + return b"" + return io.BytesIO.read(self, amt) + + def readline(self, length=None): + if self.closed: + return b"" + return io.BytesIO.readline(self, length) + + class FakeHTTPConnection(httplib.HTTPConnection): + + # buffer to store data for verification in urlopen tests. + buf = "" + + def connect(self): + self.sock = FakeSocket(self.fakedata) + self.__class__.fakesock = self.sock + FakeHTTPConnection.fakedata = fakedata + + return FakeHTTPConnection + + +class FakeHTTPMixin(object): + def fakehttp(self, fakedata): + assert httplib.HTTP._connection_class == httplib.HTTPConnection + + httplib.HTTP._connection_class = fakehttp(fakedata) + + def unfakehttp(self): + httplib.HTTP._connection_class = httplib.HTTPConnection + + +class urlopen_FileTests(unittest.TestCase): + """Test urlopen() opening a temporary file. + + Try to test as much functionality as possible so as to cut down on reliance + on connecting to the Net for testing. + + """ + + def setUp(self): + """Setup of a temp file to use for testing""" + self.text = "test_urllib: %s\n" % self.__class__.__name__ + FILE = file(test_support.TESTFN, 'wb') + try: + FILE.write(self.text) + finally: + FILE.close() + self.pathname = test_support.TESTFN + self.returned_obj = urllib.urlopen("file:%s" % self.pathname) + + def tearDown(self): + """Shut down the open object""" + self.returned_obj.close() + os.remove(test_support.TESTFN) + + def test_interface(self): + # Make sure object returned by urlopen() has the specified methods + for attr in ("read", "readline", "readlines", "fileno", + "close", "info", "geturl", "getcode", "__iter__"): + self.assertTrue(hasattr(self.returned_obj, attr), + "object returned by urlopen() lacks %s attribute" % + attr) + + def test_read(self): + self.assertEqual(self.text, self.returned_obj.read()) + + def test_readline(self): + self.assertEqual(self.text, self.returned_obj.readline()) + self.assertEqual('', self.returned_obj.readline(), + "calling readline() after exhausting the file did not" + " return an empty string") + + def test_readlines(self): + lines_list = self.returned_obj.readlines() + self.assertEqual(len(lines_list), 1, + "readlines() returned the wrong number of lines") + self.assertEqual(lines_list[0], self.text, + "readlines() returned improper text") + + def test_fileno(self): + file_num = self.returned_obj.fileno() + self.assertIsInstance(file_num, int, "fileno() did not return an int") + self.assertEqual(os.read(file_num, len(self.text)), self.text, + "Reading on the file descriptor returned by fileno() " + "did not return the expected text") + + def test_close(self): + # Test close() by calling it hear and then having it be called again + # by the tearDown() method for the test + self.returned_obj.close() + + def test_info(self): + self.assertIsInstance(self.returned_obj.info(), mimetools.Message) + + def test_geturl(self): + self.assertEqual(self.returned_obj.geturl(), self.pathname) + + def test_getcode(self): + self.assertEqual(self.returned_obj.getcode(), None) + + def test_iter(self): + # Test iterator + # Don't need to count number of iterations since test would fail the + # instant it returned anything beyond the first line from the + # comparison + for line in self.returned_obj.__iter__(): + self.assertEqual(line, self.text) + + def test_relativelocalfile(self): + self.assertRaises(ValueError,urllib.urlopen,'./' + self.pathname) + +class ProxyTests(unittest.TestCase): + + def setUp(self): + # Records changes to env vars + self.env = test_support.EnvironmentVarGuard() + # Delete all proxy related env vars + for k in os.environ.keys(): + if 'proxy' in k.lower(): + self.env.unset(k) + + def tearDown(self): + # Restore all proxy related env vars + self.env.__exit__() + del self.env + + def test_getproxies_environment_keep_no_proxies(self): + self.env.set('NO_PROXY', 'localhost') + proxies = urllib.getproxies_environment() + # getproxies_environment use lowered case truncated (no '_proxy') keys + self.assertEqual('localhost', proxies['no']) + # List of no_proxies with space. + self.env.set('NO_PROXY', 'localhost, anotherdomain.com, newdomain.com:1234') + self.assertTrue(urllib.proxy_bypass_environment('anotherdomain.com')) + self.assertTrue(urllib.proxy_bypass_environment('anotherdomain.com:8888')) + self.assertTrue(urllib.proxy_bypass_environment('newdomain.com:1234')) + + def test_proxy_cgi_ignore(self): + try: + self.env.set('HTTP_PROXY', 'http://somewhere:3128') + proxies = urllib.getproxies_environment() + self.assertEqual('http://somewhere:3128', proxies['http']) + self.env.set('REQUEST_METHOD', 'GET') + proxies = urllib.getproxies_environment() + self.assertNotIn('http', proxies) + finally: + self.env.unset('REQUEST_METHOD') + self.env.unset('HTTP_PROXY') + + def test_proxy_bypass_environment_host_match(self): + bypass = urllib.proxy_bypass_environment + self.env.set('NO_PROXY', + 'localhost, anotherdomain.com, newdomain.com:1234') + self.assertTrue(bypass('localhost')) + self.assertTrue(bypass('LocalHost')) # MixedCase + self.assertTrue(bypass('LOCALHOST')) # UPPERCASE + self.assertTrue(bypass('newdomain.com:1234')) + self.assertTrue(bypass('anotherdomain.com:8888')) + self.assertTrue(bypass('www.newdomain.com:1234')) + self.assertFalse(bypass('prelocalhost')) + self.assertFalse(bypass('newdomain.com')) # no port + self.assertFalse(bypass('newdomain.com:1235')) # wrong port + +class ProxyTests_withOrderedEnv(unittest.TestCase): + + def setUp(self): + # We need to test conditions, where variable order _is_ significant + self._saved_env = os.environ + # Monkey patch os.environ, start with empty fake environment + os.environ = collections.OrderedDict() + + def tearDown(self): + os.environ = self._saved_env + + def test_getproxies_environment_prefer_lowercase(self): + # Test lowercase preference with removal + os.environ['no_proxy'] = '' + os.environ['No_Proxy'] = 'localhost' + self.assertFalse(urllib.proxy_bypass_environment('localhost')) + self.assertFalse(urllib.proxy_bypass_environment('arbitrary')) + os.environ['http_proxy'] = '' + os.environ['HTTP_PROXY'] = 'http://somewhere:3128' + proxies = urllib.getproxies_environment() + self.assertEqual({}, proxies) + # Test lowercase preference of proxy bypass and correct matching including ports + os.environ['no_proxy'] = 'localhost, noproxy.com, my.proxy:1234' + os.environ['No_Proxy'] = 'xyz.com' + self.assertTrue(urllib.proxy_bypass_environment('localhost')) + self.assertTrue(urllib.proxy_bypass_environment('noproxy.com:5678')) + self.assertTrue(urllib.proxy_bypass_environment('my.proxy:1234')) + self.assertFalse(urllib.proxy_bypass_environment('my.proxy')) + self.assertFalse(urllib.proxy_bypass_environment('arbitrary')) + # Test lowercase preference with replacement + os.environ['http_proxy'] = 'http://somewhere:3128' + os.environ['Http_Proxy'] = 'http://somewhereelse:3128' + proxies = urllib.getproxies_environment() + self.assertEqual('http://somewhere:3128', proxies['http']) + + +class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin): + """Test urlopen() opening a fake http connection.""" + + def test_read(self): + self.fakehttp('Hello!') + try: + fp = urllib.urlopen("http://python.org/") + self.assertEqual(fp.readline(), 'Hello!') + self.assertEqual(fp.readline(), '') + self.assertEqual(fp.geturl(), 'http://python.org/') + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + def test_url_fragment(self): + # Issue #11703: geturl() omits fragments in the original URL. + url = 'http://docs.python.org/library/urllib.html#OK' + self.fakehttp('Hello!') + try: + fp = urllib.urlopen(url) + self.assertEqual(fp.geturl(), url) + finally: + self.unfakehttp() + + def test_read_bogus(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp('''HTTP/1.1 401 Authentication Required +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Type: text/html; charset=iso-8859-1 +''') + try: + self.assertRaises(IOError, urllib.urlopen, "http://python.org/") + finally: + self.unfakehttp() + + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp("""HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file:README +Connection: close +Content-Type: text/html; charset=iso-8859-1 +""") + try: + msg = "Redirection to url 'file:" + with self.assertRaisesRegexp(IOError, msg): + urllib.urlopen("http://python.org/") + finally: + self.unfakehttp() + + def test_redirect_limit_independent(self): + # Ticket #12923: make sure independent requests each use their + # own retry limit. + for i in range(urllib.FancyURLopener().maxtries): + self.fakehttp(b'''HTTP/1.1 302 Found +Location: file://guidocomputer.athome.com:/python/license +Connection: close +''') + try: + self.assertRaises(IOError, urllib.urlopen, + "http://something") + finally: + self.unfakehttp() + + def test_empty_socket(self): + # urlopen() raises IOError if the underlying socket does not send any + # data. (#1680230) + self.fakehttp('') + try: + self.assertRaises(IOError, urllib.urlopen, 'http://something') + finally: + self.unfakehttp() + + def test_missing_localfile(self): + self.assertRaises(IOError, urllib.urlopen, + 'file://localhost/a/missing/file.py') + fd, tmp_file = tempfile.mkstemp() + tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + self.assertTrue(os.path.exists(tmp_file)) + try: + fp = urllib.urlopen(tmp_fileurl) + fp.close() + finally: + os.close(fd) + os.unlink(tmp_file) + + self.assertFalse(os.path.exists(tmp_file)) + self.assertRaises(IOError, urllib.urlopen, tmp_fileurl) + + def test_ftp_nonexisting(self): + self.assertRaises(IOError, urllib.urlopen, + 'ftp://localhost/not/existing/file.py') + + + def test_userpass_inurl(self): + self.fakehttp('Hello!') + try: + fakehttp_wrapper = httplib.HTTP._connection_class + fp = urllib.urlopen("http://user:pass@python.org/") + authorization = ("Authorization: Basic %s\r\n" % + b64encode('user:pass')) + # The authorization header must be in place + self.assertIn(authorization, fakehttp_wrapper.buf) + self.assertEqual(fp.readline(), "Hello!") + self.assertEqual(fp.readline(), "") + self.assertEqual(fp.geturl(), 'http://user:pass@python.org/') + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + def test_userpass_with_spaces_inurl(self): + self.fakehttp('Hello!') + try: + url = "http://a b:c d@python.org/" + fakehttp_wrapper = httplib.HTTP._connection_class + authorization = ("Authorization: Basic %s\r\n" % + b64encode('a b:c d')) + fp = urllib.urlopen(url) + # The authorization header must be in place + self.assertIn(authorization, fakehttp_wrapper.buf) + self.assertEqual(fp.readline(), "Hello!") + self.assertEqual(fp.readline(), "") + # the spaces are quoted in URL so no match + self.assertNotEqual(fp.geturl(), url) + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + +class urlretrieve_FileTests(unittest.TestCase): + """Test urllib.urlretrieve() on local files""" + + def setUp(self): + # Create a list of temporary files. Each item in the list is a file + # name (absolute path or relative to the current working directory). + # All files in this list will be deleted in the tearDown method. Note, + # this only helps to makes sure temporary files get deleted, but it + # does nothing about trying to close files that may still be open. It + # is the responsibility of the developer to properly close files even + # when exceptional conditions occur. + self.tempFiles = [] + + # Create a temporary file. + self.registerFileForCleanUp(test_support.TESTFN) + self.text = 'testing urllib.urlretrieve' + try: + FILE = file(test_support.TESTFN, 'wb') + FILE.write(self.text) + FILE.close() + finally: + try: FILE.close() + except: pass + + def tearDown(self): + # Delete the temporary files. + for each in self.tempFiles: + try: os.remove(each) + except: pass + + def constructLocalFileUrl(self, filePath): + return "file://%s" % urllib.pathname2url(os.path.abspath(filePath)) + + def createNewTempFile(self, data=""): + """Creates a new temporary file containing the specified data, + registers the file for deletion during the test fixture tear down, and + returns the absolute path of the file.""" + + newFd, newFilePath = tempfile.mkstemp() + try: + self.registerFileForCleanUp(newFilePath) + newFile = os.fdopen(newFd, "wb") + newFile.write(data) + newFile.close() + finally: + try: newFile.close() + except: pass + return newFilePath + + def registerFileForCleanUp(self, fileName): + self.tempFiles.append(fileName) + + def test_basic(self): + # Make sure that a local file just gets its own location returned and + # a headers value is returned. + result = urllib.urlretrieve("file:%s" % test_support.TESTFN) + self.assertEqual(result[0], test_support.TESTFN) + self.assertIsInstance(result[1], mimetools.Message, + "did not get a mimetools.Message instance as " + "second returned value") + + def test_copy(self): + # Test that setting the filename argument works. + second_temp = "%s.2" % test_support.TESTFN + self.registerFileForCleanUp(second_temp) + result = urllib.urlretrieve(self.constructLocalFileUrl( + test_support.TESTFN), second_temp) + self.assertEqual(second_temp, result[0]) + self.assertTrue(os.path.exists(second_temp), "copy of the file was not " + "made") + FILE = file(second_temp, 'rb') + try: + text = FILE.read() + FILE.close() + finally: + try: FILE.close() + except: pass + self.assertEqual(self.text, text) + + def test_reporthook(self): + # Make sure that the reporthook works. + def hooktester(count, block_size, total_size, count_holder=[0]): + self.assertIsInstance(count, int) + self.assertIsInstance(block_size, int) + self.assertIsInstance(total_size, int) + self.assertEqual(count, count_holder[0]) + count_holder[0] = count_holder[0] + 1 + second_temp = "%s.2" % test_support.TESTFN + self.registerFileForCleanUp(second_temp) + urllib.urlretrieve(self.constructLocalFileUrl(test_support.TESTFN), + second_temp, hooktester) + + def test_reporthook_0_bytes(self): + # Test on zero length file. Should call reporthook only 1 time. + report = [] + def hooktester(count, block_size, total_size, _report=report): + _report.append((count, block_size, total_size)) + srcFileName = self.createNewTempFile() + urllib.urlretrieve(self.constructLocalFileUrl(srcFileName), + test_support.TESTFN, hooktester) + self.assertEqual(len(report), 1) + self.assertEqual(report[0][2], 0) + + def test_reporthook_5_bytes(self): + # Test on 5 byte file. Should call reporthook only 2 times (once when + # the "network connection" is established and once when the block is + # read). Since the block size is 8192 bytes, only one block read is + # required to read the entire file. + report = [] + def hooktester(count, block_size, total_size, _report=report): + _report.append((count, block_size, total_size)) + srcFileName = self.createNewTempFile("x" * 5) + urllib.urlretrieve(self.constructLocalFileUrl(srcFileName), + test_support.TESTFN, hooktester) + self.assertEqual(len(report), 2) + self.assertEqual(report[0][1], 8192) + self.assertEqual(report[0][2], 5) + + def test_reporthook_8193_bytes(self): + # Test on 8193 byte file. Should call reporthook only 3 times (once + # when the "network connection" is established, once for the next 8192 + # bytes, and once for the last byte). + report = [] + def hooktester(count, block_size, total_size, _report=report): + _report.append((count, block_size, total_size)) + srcFileName = self.createNewTempFile("x" * 8193) + urllib.urlretrieve(self.constructLocalFileUrl(srcFileName), + test_support.TESTFN, hooktester) + self.assertEqual(len(report), 3) + self.assertEqual(report[0][1], 8192) + self.assertEqual(report[0][2], 8193) + + +class urlretrieve_HttpTests(unittest.TestCase, FakeHTTPMixin): + """Test urllib.urlretrieve() using fake http connections""" + + def test_short_content_raises_ContentTooShortError(self): + self.fakehttp('''HTTP/1.1 200 OK +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Length: 100 +Content-Type: text/html; charset=iso-8859-1 + +FF +''') + + def _reporthook(par1, par2, par3): + pass + + try: + self.assertRaises(urllib.ContentTooShortError, urllib.urlretrieve, + 'http://example.com', reporthook=_reporthook) + finally: + self.unfakehttp() + + def test_short_content_raises_ContentTooShortError_without_reporthook(self): + self.fakehttp('''HTTP/1.1 200 OK +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Length: 100 +Content-Type: text/html; charset=iso-8859-1 + +FF +''') + try: + self.assertRaises(urllib.ContentTooShortError, urllib.urlretrieve, 'http://example.com/') + finally: + self.unfakehttp() + +class QuotingTests(unittest.TestCase): + """Tests for urllib.quote() and urllib.quote_plus() + + According to RFC 2396 ("Uniform Resource Identifiers), to escape a + character you write it as '%' + <2 character US-ASCII hex value>. The Python + code of ``'%' + hex(ord())[2:]`` escapes a character properly. + Case does not matter on the hex letters. + + The various character sets specified are: + + Reserved characters : ";/?:@&=+$," + Have special meaning in URIs and must be escaped if not being used for + their special meaning + Data characters : letters, digits, and "-_.!~*'()" + Unreserved and do not need to be escaped; can be, though, if desired + Control characters : 0x00 - 0x1F, 0x7F + Have no use in URIs so must be escaped + space : 0x20 + Must be escaped + Delimiters : '<>#%"' + Must be escaped + Unwise : "{}|\^[]`" + Must be escaped + + """ + + def test_never_quote(self): + # Make sure quote() does not quote letters, digits, and "_,.-" + do_not_quote = '' .join(["ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "abcdefghijklmnopqrstuvwxyz", + "0123456789", + "_.-"]) + result = urllib.quote(do_not_quote) + self.assertEqual(do_not_quote, result, + "using quote(): %s != %s" % (do_not_quote, result)) + result = urllib.quote_plus(do_not_quote) + self.assertEqual(do_not_quote, result, + "using quote_plus(): %s != %s" % (do_not_quote, result)) + + def test_default_safe(self): + # Test '/' is default value for 'safe' parameter + self.assertEqual(urllib.quote.func_defaults[0], '/') + + def test_safe(self): + # Test setting 'safe' parameter does what it should do + quote_by_default = "<>" + result = urllib.quote(quote_by_default, safe=quote_by_default) + self.assertEqual(quote_by_default, result, + "using quote(): %s != %s" % (quote_by_default, result)) + result = urllib.quote_plus(quote_by_default, safe=quote_by_default) + self.assertEqual(quote_by_default, result, + "using quote_plus(): %s != %s" % + (quote_by_default, result)) + + def test_default_quoting(self): + # Make sure all characters that should be quoted are by default sans + # space (separate test for that). + should_quote = [chr(num) for num in range(32)] # For 0x00 - 0x1F + should_quote.append('<>#%"{}|\^[]`') + should_quote.append(chr(127)) # For 0x7F + should_quote = ''.join(should_quote) + for char in should_quote: + result = urllib.quote(char) + self.assertEqual(hexescape(char), result, + "using quote(): %s should be escaped to %s, not %s" % + (char, hexescape(char), result)) + result = urllib.quote_plus(char) + self.assertEqual(hexescape(char), result, + "using quote_plus(): " + "%s should be escapes to %s, not %s" % + (char, hexescape(char), result)) + del should_quote + partial_quote = "ab[]cd" + expected = "ab%5B%5Dcd" + result = urllib.quote(partial_quote) + self.assertEqual(expected, result, + "using quote(): %s != %s" % (expected, result)) + result = urllib.quote_plus(partial_quote) + self.assertEqual(expected, result, + "using quote_plus(): %s != %s" % (expected, result)) + self.assertRaises(TypeError, urllib.quote, None) + + def test_quoting_space(self): + # Make sure quote() and quote_plus() handle spaces as specified in + # their unique way + result = urllib.quote(' ') + self.assertEqual(result, hexescape(' '), + "using quote(): %s != %s" % (result, hexescape(' '))) + result = urllib.quote_plus(' ') + self.assertEqual(result, '+', + "using quote_plus(): %s != +" % result) + given = "a b cd e f" + expect = given.replace(' ', hexescape(' ')) + result = urllib.quote(given) + self.assertEqual(expect, result, + "using quote(): %s != %s" % (expect, result)) + expect = given.replace(' ', '+') + result = urllib.quote_plus(given) + self.assertEqual(expect, result, + "using quote_plus(): %s != %s" % (expect, result)) + + def test_quoting_plus(self): + self.assertEqual(urllib.quote_plus('alpha+beta gamma'), + 'alpha%2Bbeta+gamma') + self.assertEqual(urllib.quote_plus('alpha+beta gamma', '+'), + 'alpha+beta+gamma') + +class UnquotingTests(unittest.TestCase): + """Tests for unquote() and unquote_plus() + + See the doc string for quoting_Tests for details on quoting and such. + + """ + + def test_unquoting(self): + # Make sure unquoting of all ASCII values works + escape_list = [] + for num in range(128): + given = hexescape(chr(num)) + expect = chr(num) + result = urllib.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %s != %s" % (expect, result)) + result = urllib.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %s != %s" % + (expect, result)) + escape_list.append(given) + escape_string = ''.join(escape_list) + del escape_list + result = urllib.unquote(escape_string) + self.assertEqual(result.count('%'), 1, + "using quote(): not all characters escaped; %s" % + result) + result = urllib.unquote(escape_string) + self.assertEqual(result.count('%'), 1, + "using unquote(): not all characters escaped: " + "%s" % result) + + def test_unquoting_badpercent(self): + # Test unquoting on bad percent-escapes + given = '%xab' + expect = given + result = urllib.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + given = '%x' + expect = given + result = urllib.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + given = '%' + expect = given + result = urllib.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + + def test_unquoting_mixed_case(self): + # Test unquoting on mixed-case hex digits in the percent-escapes + given = '%Ab%eA' + expect = '\xab\xea' + result = urllib.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + + def test_unquoting_parts(self): + # Make sure unquoting works when have non-quoted characters + # interspersed + given = 'ab%sd' % hexescape('c') + expect = "abcd" + result = urllib.unquote(given) + self.assertEqual(expect, result, + "using quote(): %s != %s" % (expect, result)) + result = urllib.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %s != %s" % (expect, result)) + + def test_unquoting_plus(self): + # Test difference between unquote() and unquote_plus() + given = "are+there+spaces..." + expect = given + result = urllib.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %s != %s" % (expect, result)) + expect = given.replace('+', ' ') + result = urllib.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %s != %s" % (expect, result)) + + def test_unquote_with_unicode(self): + r = urllib.unquote(u'br%C3%BCckner_sapporo_20050930.doc') + self.assertEqual(r, u'br\xc3\xbcckner_sapporo_20050930.doc') + +class urlencode_Tests(unittest.TestCase): + """Tests for urlencode()""" + + def help_inputtype(self, given, test_type): + """Helper method for testing different input types. + + 'given' must lead to only the pairs: + * 1st, 1 + * 2nd, 2 + * 3rd, 3 + + Test cannot assume anything about order. Docs make no guarantee and + have possible dictionary input. + + """ + expect_somewhere = ["1st=1", "2nd=2", "3rd=3"] + result = urllib.urlencode(given) + for expected in expect_somewhere: + self.assertIn(expected, result, + "testing %s: %s not found in %s" % + (test_type, expected, result)) + self.assertEqual(result.count('&'), 2, + "testing %s: expected 2 '&'s; got %s" % + (test_type, result.count('&'))) + amp_location = result.index('&') + on_amp_left = result[amp_location - 1] + on_amp_right = result[amp_location + 1] + self.assertTrue(on_amp_left.isdigit() and on_amp_right.isdigit(), + "testing %s: '&' not located in proper place in %s" % + (test_type, result)) + self.assertEqual(len(result), (5 * 3) + 2, #5 chars per thing and amps + "testing %s: " + "unexpected number of characters: %s != %s" % + (test_type, len(result), (5 * 3) + 2)) + + def test_using_mapping(self): + # Test passing in a mapping object as an argument. + self.help_inputtype({"1st":'1', "2nd":'2', "3rd":'3'}, + "using dict as input type") + + def test_using_sequence(self): + # Test passing in a sequence of two-item sequences as an argument. + self.help_inputtype([('1st', '1'), ('2nd', '2'), ('3rd', '3')], + "using sequence of two-item tuples as input") + + def test_quoting(self): + # Make sure keys and values are quoted using quote_plus() + given = {"&":"="} + expect = "%s=%s" % (hexescape('&'), hexescape('=')) + result = urllib.urlencode(given) + self.assertEqual(expect, result) + given = {"key name":"A bunch of pluses"} + expect = "key+name=A+bunch+of+pluses" + result = urllib.urlencode(given) + self.assertEqual(expect, result) + + def test_doseq(self): + # Test that passing True for 'doseq' parameter works correctly + given = {'sequence':['1', '2', '3']} + expect = "sequence=%s" % urllib.quote_plus(str(['1', '2', '3'])) + result = urllib.urlencode(given) + self.assertEqual(expect, result) + result = urllib.urlencode(given, True) + for value in given["sequence"]: + expect = "sequence=%s" % value + self.assertIn(expect, result) + self.assertEqual(result.count('&'), 2, + "Expected 2 '&'s, got %s" % result.count('&')) + +class Pathname_Tests(unittest.TestCase): + """Test pathname2url() and url2pathname()""" + + def test_basic(self): + # Make sure simple tests pass + expected_path = os.path.join("parts", "of", "a", "path") + expected_url = "parts/of/a/path" + result = urllib.pathname2url(expected_path) + self.assertEqual(expected_url, result, + "pathname2url() failed; %s != %s" % + (result, expected_url)) + result = urllib.url2pathname(expected_url) + self.assertEqual(expected_path, result, + "url2pathame() failed; %s != %s" % + (result, expected_path)) + + def test_quoting(self): + # Test automatic quoting and unquoting works for pathnam2url() and + # url2pathname() respectively + given = os.path.join("needs", "quot=ing", "here") + expect = "needs/%s/here" % urllib.quote("quot=ing") + result = urllib.pathname2url(given) + self.assertEqual(expect, result, + "pathname2url() failed; %s != %s" % + (expect, result)) + expect = given + result = urllib.url2pathname(result) + self.assertEqual(expect, result, + "url2pathname() failed; %s != %s" % + (expect, result)) + given = os.path.join("make sure", "using_quote") + expect = "%s/using_quote" % urllib.quote("make sure") + result = urllib.pathname2url(given) + self.assertEqual(expect, result, + "pathname2url() failed; %s != %s" % + (expect, result)) + given = "make+sure/using_unquote" + expect = os.path.join("make+sure", "using_unquote") + result = urllib.url2pathname(given) + self.assertEqual(expect, result, + "url2pathname() failed; %s != %s" % + (expect, result)) + + @unittest.skipUnless(sys.platform == 'win32', + 'test specific to the nturl2path library') + def test_ntpath(self): + given = ('/C:/', '///C:/', '/C|//') + expect = 'C:\\' + for url in given: + result = urllib.url2pathname(url) + self.assertEqual(expect, result, + 'nturl2path.url2pathname() failed; %s != %s' % + (expect, result)) + given = '///C|/path' + expect = 'C:\\path' + result = urllib.url2pathname(given) + self.assertEqual(expect, result, + 'nturl2path.url2pathname() failed; %s != %s' % + (expect, result)) + +class Utility_Tests(unittest.TestCase): + """Testcase to test the various utility functions in the urllib.""" + # In Python 3 this test class is moved to test_urlparse. + + def test_splittype(self): + splittype = urllib.splittype + self.assertEqual(splittype('type:opaquestring'), ('type', 'opaquestring')) + self.assertEqual(splittype('opaquestring'), (None, 'opaquestring')) + self.assertEqual(splittype(':opaquestring'), (None, ':opaquestring')) + self.assertEqual(splittype('type:'), ('type', '')) + self.assertEqual(splittype('type:opaque:string'), ('type', 'opaque:string')) + + def test_splithost(self): + splithost = urllib.splithost + self.assertEqual(splithost('//www.example.org:80/foo/bar/baz.html'), + ('www.example.org:80', '/foo/bar/baz.html')) + self.assertEqual(splithost('//www.example.org:80'), + ('www.example.org:80', '')) + self.assertEqual(splithost('/foo/bar/baz.html'), + (None, '/foo/bar/baz.html')) + + def test_splituser(self): + splituser = urllib.splituser + self.assertEqual(splituser('User:Pass@www.python.org:080'), + ('User:Pass', 'www.python.org:080')) + self.assertEqual(splituser('@www.python.org:080'), + ('', 'www.python.org:080')) + self.assertEqual(splituser('www.python.org:080'), + (None, 'www.python.org:080')) + self.assertEqual(splituser('User:Pass@'), + ('User:Pass', '')) + self.assertEqual(splituser('User@example.com:Pass@www.python.org:080'), + ('User@example.com:Pass', 'www.python.org:080')) + + def test_splitpasswd(self): + # Some of the password examples are not sensible, but it is added to + # confirming to RFC2617 and addressing issue4675. + splitpasswd = urllib.splitpasswd + self.assertEqual(splitpasswd('user:ab'), ('user', 'ab')) + self.assertEqual(splitpasswd('user:a\nb'), ('user', 'a\nb')) + self.assertEqual(splitpasswd('user:a\tb'), ('user', 'a\tb')) + self.assertEqual(splitpasswd('user:a\rb'), ('user', 'a\rb')) + self.assertEqual(splitpasswd('user:a\fb'), ('user', 'a\fb')) + self.assertEqual(splitpasswd('user:a\vb'), ('user', 'a\vb')) + self.assertEqual(splitpasswd('user:a:b'), ('user', 'a:b')) + self.assertEqual(splitpasswd('user:a b'), ('user', 'a b')) + self.assertEqual(splitpasswd('user 2:ab'), ('user 2', 'ab')) + self.assertEqual(splitpasswd('user+1:a+b'), ('user+1', 'a+b')) + self.assertEqual(splitpasswd('user:'), ('user', '')) + self.assertEqual(splitpasswd('user'), ('user', None)) + self.assertEqual(splitpasswd(':ab'), ('', 'ab')) + + def test_splitport(self): + splitport = urllib.splitport + self.assertEqual(splitport('parrot:88'), ('parrot', '88')) + self.assertEqual(splitport('parrot'), ('parrot', None)) + self.assertEqual(splitport('parrot:'), ('parrot', None)) + self.assertEqual(splitport('127.0.0.1'), ('127.0.0.1', None)) + self.assertEqual(splitport('parrot:cheese'), ('parrot:cheese', None)) + self.assertEqual(splitport('[::1]:88'), ('[::1]', '88')) + self.assertEqual(splitport('[::1]'), ('[::1]', None)) + self.assertEqual(splitport(':88'), ('', '88')) + + def test_splitnport(self): + splitnport = urllib.splitnport + self.assertEqual(splitnport('parrot:88'), ('parrot', 88)) + self.assertEqual(splitnport('parrot'), ('parrot', -1)) + self.assertEqual(splitnport('parrot', 55), ('parrot', 55)) + self.assertEqual(splitnport('parrot:'), ('parrot', -1)) + self.assertEqual(splitnport('parrot:', 55), ('parrot', 55)) + self.assertEqual(splitnport('127.0.0.1'), ('127.0.0.1', -1)) + self.assertEqual(splitnport('127.0.0.1', 55), ('127.0.0.1', 55)) + self.assertEqual(splitnport('parrot:cheese'), ('parrot', None)) + self.assertEqual(splitnport('parrot:cheese', 55), ('parrot', None)) + + def test_splitquery(self): + # Normal cases are exercised by other tests; ensure that we also + # catch cases with no port specified (testcase ensuring coverage) + splitquery = urllib.splitquery + self.assertEqual(splitquery('http://python.org/fake?foo=bar'), + ('http://python.org/fake', 'foo=bar')) + self.assertEqual(splitquery('http://python.org/fake?foo=bar?'), + ('http://python.org/fake?foo=bar', '')) + self.assertEqual(splitquery('http://python.org/fake'), + ('http://python.org/fake', None)) + self.assertEqual(splitquery('?foo=bar'), ('', 'foo=bar')) + + def test_splittag(self): + splittag = urllib.splittag + self.assertEqual(splittag('http://example.com?foo=bar#baz'), + ('http://example.com?foo=bar', 'baz')) + self.assertEqual(splittag('http://example.com?foo=bar#'), + ('http://example.com?foo=bar', '')) + self.assertEqual(splittag('#baz'), ('', 'baz')) + self.assertEqual(splittag('http://example.com?foo=bar'), + ('http://example.com?foo=bar', None)) + self.assertEqual(splittag('http://example.com?foo=bar#baz#boo'), + ('http://example.com?foo=bar#baz', 'boo')) + + def test_splitattr(self): + splitattr = urllib.splitattr + self.assertEqual(splitattr('/path;attr1=value1;attr2=value2'), + ('/path', ['attr1=value1', 'attr2=value2'])) + self.assertEqual(splitattr('/path;'), ('/path', [''])) + self.assertEqual(splitattr(';attr1=value1;attr2=value2'), + ('', ['attr1=value1', 'attr2=value2'])) + self.assertEqual(splitattr('/path'), ('/path', [])) + + def test_splitvalue(self): + # Normal cases are exercised by other tests; test pathological cases + # with no key/value pairs. (testcase ensuring coverage) + splitvalue = urllib.splitvalue + self.assertEqual(splitvalue('foo=bar'), ('foo', 'bar')) + self.assertEqual(splitvalue('foo='), ('foo', '')) + self.assertEqual(splitvalue('=bar'), ('', 'bar')) + self.assertEqual(splitvalue('foobar'), ('foobar', None)) + self.assertEqual(splitvalue('foo=bar=baz'), ('foo', 'bar=baz')) + + def test_toBytes(self): + result = urllib.toBytes(u'http://www.python.org') + self.assertEqual(result, 'http://www.python.org') + self.assertRaises(UnicodeError, urllib.toBytes, + test_support.u(r'http://www.python.org/medi\u00e6val')) + + def test_unwrap(self): + url = urllib.unwrap('') + self.assertEqual(url, 'type://host/path') + + +class URLopener_Tests(unittest.TestCase): + """Testcase to test the open method of URLopener class.""" + + def test_quoted_open(self): + class DummyURLopener(urllib.URLopener): + def open_spam(self, url): + return url + + self.assertEqual(DummyURLopener().open( + 'spam://example/ /'),'//example/%20/') + + # test the safe characters are not quoted by urlopen + self.assertEqual(DummyURLopener().open( + "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"), + "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/") + + +# Just commented them out. +# Can't really tell why keep failing in windows and sparc. +# Everywhere else they work ok, but on those machines, sometimes +# fail in one of the tests, sometimes in other. I have a linux, and +# the tests go ok. +# If anybody has one of the problematic environments, please help! +# . Facundo +# +# def server(evt): +# import socket, time +# serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +# serv.settimeout(3) +# serv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +# serv.bind(("", 9093)) +# serv.listen(5) +# try: +# conn, addr = serv.accept() +# conn.send("1 Hola mundo\n") +# cantdata = 0 +# while cantdata < 13: +# data = conn.recv(13-cantdata) +# cantdata += len(data) +# time.sleep(.3) +# conn.send("2 No more lines\n") +# conn.close() +# except socket.timeout: +# pass +# finally: +# serv.close() +# evt.set() +# +# class FTPWrapperTests(unittest.TestCase): +# +# def setUp(self): +# import ftplib, time, threading +# ftplib.FTP.port = 9093 +# self.evt = threading.Event() +# threading.Thread(target=server, args=(self.evt,)).start() +# time.sleep(.1) +# +# def tearDown(self): +# self.evt.wait() +# +# def testBasic(self): +# # connects +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# ftp.close() +# +# def testTimeoutNone(self): +# # global default timeout is ignored +# import socket +# self.assertIsNone(socket.getdefaulttimeout()) +# socket.setdefaulttimeout(30) +# try: +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# finally: +# socket.setdefaulttimeout(None) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() +# +# def testTimeoutDefault(self): +# # global default timeout is used +# import socket +# self.assertIsNone(socket.getdefaulttimeout()) +# socket.setdefaulttimeout(30) +# try: +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# finally: +# socket.setdefaulttimeout(None) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() +# +# def testTimeoutValue(self): +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, [], +# timeout=30) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() + + + +def test_main(): + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', ".*urllib\.urlopen.*Python 3.0", + DeprecationWarning) + test_support.run_unittest( + urlopen_FileTests, + urlopen_HttpTests, + urlretrieve_FileTests, + urlretrieve_HttpTests, + ProxyTests, + QuotingTests, + UnquotingTests, + urlencode_Tests, + Pathname_Tests, + Utility_Tests, + URLopener_Tests, + ProxyTests, + ProxyTests_withOrderedEnv, + #FTPWrapperTests, + ) + + + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/2.7pypy/test_urllib2.py b/src/greentest/2.7pypy/test_urllib2.py new file mode 100644 index 0000000..6783ba4 --- /dev/null +++ b/src/greentest/2.7pypy/test_urllib2.py @@ -0,0 +1,1428 @@ +import unittest +from test import test_support +from test import test_urllib + +import os +import socket +import StringIO + +import urllib2 +from urllib2 import Request, OpenerDirector, AbstractDigestAuthHandler +import httplib + +try: + import ssl +except ImportError: + ssl = None + +# XXX +# Request +# CacheFTPHandler (hard to write) +# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler + +class TrivialTests(unittest.TestCase): + def test_trivial(self): + # A couple trivial tests + + self.assertRaises(ValueError, urllib2.urlopen, 'bogus url') + + # XXX Name hacking to get this to work on Windows. + fname = os.path.abspath(urllib2.__file__).replace(os.sep, '/') + + # And more hacking to get it to work on MacOS. This assumes + # urllib.pathname2url works, unfortunately... + if os.name == 'riscos': + import string + fname = os.expand(fname) + fname = fname.translate(string.maketrans("/.", "./")) + + if os.name == 'nt': + file_url = "file:///%s" % fname + else: + file_url = "file://%s" % fname + + f = urllib2.urlopen(file_url) + + buf = f.read() + f.close() + + def test_parse_http_list(self): + tests = [('a,b,c', ['a', 'b', 'c']), + ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']), + ('a, b, "c", "d", "e,f", g, h', ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']), + ('a="b\\"c", d="e\\,f", g="h\\\\i"', ['a="b"c"', 'd="e,f"', 'g="h\\i"'])] + for string, list in tests: + self.assertEqual(urllib2.parse_http_list(string), list) + + @unittest.skipUnless(ssl, "ssl module required") + def test_cafile_and_context(self): + context = ssl.create_default_context() + with self.assertRaises(ValueError): + urllib2.urlopen( + "https://localhost", cafile="/nonexistent/path", context=context + ) + + +def test_request_headers_dict(): + """ + The Request.headers dictionary is not a documented interface. It should + stay that way, because the complete set of headers are only accessible + through the .get_header(), .has_header(), .header_items() interface. + However, .headers pre-dates those methods, and so real code will be using + the dictionary. + + The introduction in 2.4 of those methods was a mistake for the same reason: + code that previously saw all (urllib2 user)-provided headers in .headers + now sees only a subset (and the function interface is ugly and incomplete). + A better change would have been to replace .headers dict with a dict + subclass (or UserDict.DictMixin instance?) that preserved the .headers + interface and also provided access to the "unredirected" headers. It's + probably too late to fix that, though. + + + Check .capitalize() case normalization: + + >>> url = "http://example.com" + >>> Request(url, headers={"Spam-eggs": "blah"}).headers["Spam-eggs"] + 'blah' + >>> Request(url, headers={"spam-EggS": "blah"}).headers["Spam-eggs"] + 'blah' + + Currently, Request(url, "Spam-eggs").headers["Spam-Eggs"] raises KeyError, + but that could be changed in future. + + """ + +def test_request_headers_methods(): + """ + Note the case normalization of header names here, to .capitalize()-case. + This should be preserved for backwards-compatibility. (In the HTTP case, + normalization to .title()-case is done by urllib2 before sending headers to + httplib). + + >>> url = "http://example.com" + >>> r = Request(url, headers={"Spam-eggs": "blah"}) + >>> r.has_header("Spam-eggs") + True + >>> r.header_items() + [('Spam-eggs', 'blah')] + >>> r.add_header("Foo-Bar", "baz") + >>> items = r.header_items() + >>> items.sort() + >>> items + [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')] + + Note that e.g. r.has_header("spam-EggS") is currently False, and + r.get_header("spam-EggS") returns None, but that could be changed in + future. + + >>> r.has_header("Not-there") + False + >>> print r.get_header("Not-there") + None + >>> r.get_header("Not-there", "default") + 'default' + + """ + + +def test_password_manager(self): + """ + >>> mgr = urllib2.HTTPPasswordMgr() + >>> add = mgr.add_password + >>> add("Some Realm", "http://example.com/", "joe", "password") + >>> add("Some Realm", "http://example.com/ni", "ni", "ni") + >>> add("c", "http://example.com/foo", "foo", "ni") + >>> add("c", "http://example.com/bar", "bar", "nini") + >>> add("b", "http://example.com/", "first", "blah") + >>> add("b", "http://example.com/", "second", "spam") + >>> add("a", "http://example.com", "1", "a") + >>> add("Some Realm", "http://c.example.com:3128", "3", "c") + >>> add("Some Realm", "d.example.com", "4", "d") + >>> add("Some Realm", "e.example.com:3128", "5", "e") + + >>> mgr.find_user_password("Some Realm", "example.com") + ('joe', 'password') + >>> mgr.find_user_password("Some Realm", "http://example.com") + ('joe', 'password') + >>> mgr.find_user_password("Some Realm", "http://example.com/") + ('joe', 'password') + >>> mgr.find_user_password("Some Realm", "http://example.com/spam") + ('joe', 'password') + >>> mgr.find_user_password("Some Realm", "http://example.com/spam/spam") + ('joe', 'password') + >>> mgr.find_user_password("c", "http://example.com/foo") + ('foo', 'ni') + >>> mgr.find_user_password("c", "http://example.com/bar") + ('bar', 'nini') + + Actually, this is really undefined ATM +## Currently, we use the highest-level path where more than one match: + +## >>> mgr.find_user_password("Some Realm", "http://example.com/ni") +## ('joe', 'password') + + Use latest add_password() in case of conflict: + + >>> mgr.find_user_password("b", "http://example.com/") + ('second', 'spam') + + No special relationship between a.example.com and example.com: + + >>> mgr.find_user_password("a", "http://example.com/") + ('1', 'a') + >>> mgr.find_user_password("a", "http://a.example.com/") + (None, None) + + Ports: + + >>> mgr.find_user_password("Some Realm", "c.example.com") + (None, None) + >>> mgr.find_user_password("Some Realm", "c.example.com:3128") + ('3', 'c') + >>> mgr.find_user_password("Some Realm", "http://c.example.com:3128") + ('3', 'c') + >>> mgr.find_user_password("Some Realm", "d.example.com") + ('4', 'd') + >>> mgr.find_user_password("Some Realm", "e.example.com:3128") + ('5', 'e') + + """ + pass + + +def test_password_manager_default_port(self): + """ + >>> mgr = urllib2.HTTPPasswordMgr() + >>> add = mgr.add_password + + The point to note here is that we can't guess the default port if there's + no scheme. This applies to both add_password and find_user_password. + + >>> add("f", "http://g.example.com:80", "10", "j") + >>> add("g", "http://h.example.com", "11", "k") + >>> add("h", "i.example.com:80", "12", "l") + >>> add("i", "j.example.com", "13", "m") + >>> mgr.find_user_password("f", "g.example.com:100") + (None, None) + >>> mgr.find_user_password("f", "g.example.com:80") + ('10', 'j') + >>> mgr.find_user_password("f", "g.example.com") + (None, None) + >>> mgr.find_user_password("f", "http://g.example.com:100") + (None, None) + >>> mgr.find_user_password("f", "http://g.example.com:80") + ('10', 'j') + >>> mgr.find_user_password("f", "http://g.example.com") + ('10', 'j') + >>> mgr.find_user_password("g", "h.example.com") + ('11', 'k') + >>> mgr.find_user_password("g", "h.example.com:80") + ('11', 'k') + >>> mgr.find_user_password("g", "http://h.example.com:80") + ('11', 'k') + >>> mgr.find_user_password("h", "i.example.com") + (None, None) + >>> mgr.find_user_password("h", "i.example.com:80") + ('12', 'l') + >>> mgr.find_user_password("h", "http://i.example.com:80") + ('12', 'l') + >>> mgr.find_user_password("i", "j.example.com") + ('13', 'm') + >>> mgr.find_user_password("i", "j.example.com:80") + (None, None) + >>> mgr.find_user_password("i", "http://j.example.com") + ('13', 'm') + >>> mgr.find_user_password("i", "http://j.example.com:80") + (None, None) + + """ + +class MockOpener: + addheaders = [] + def open(self, req, data=None,timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.req, self.data, self.timeout = req, data, timeout + def error(self, proto, *args): + self.proto, self.args = proto, args + +class MockFile: + def read(self, count=None): pass + def readline(self, count=None): pass + def close(self): pass + +class MockHeaders(dict): + def getheaders(self, name): + return self.values() + +class MockResponse(StringIO.StringIO): + def __init__(self, code, msg, headers, data, url=None): + StringIO.StringIO.__init__(self, data) + self.code, self.msg, self.headers, self.url = code, msg, headers, url + def info(self): + return self.headers + def geturl(self): + return self.url + +class MockCookieJar: + def add_cookie_header(self, request): + self.ach_req = request + def extract_cookies(self, response, request): + self.ec_req, self.ec_r = request, response + +class FakeMethod: + def __init__(self, meth_name, action, handle): + self.meth_name = meth_name + self.handle = handle + self.action = action + def __call__(self, *args): + return self.handle(self.meth_name, self.action, *args) + +class MockHTTPResponse: + def __init__(self, fp, msg, status, reason): + self.fp = fp + self.msg = msg + self.status = status + self.reason = reason + def read(self): + return '' + def _reuse(self): pass + def _drop(self): pass + +class MockHTTPClass: + def __init__(self): + self.req_headers = [] + self.data = None + self.raise_on_endheaders = False + self.sock = None + self._tunnel_headers = {} + + def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.host = host + self.timeout = timeout + return self + + def set_debuglevel(self, level): + self.level = level + + def set_tunnel(self, host, port=None, headers=None): + self._tunnel_host = host + self._tunnel_port = port + if headers: + self._tunnel_headers = headers + else: + self._tunnel_headers.clear() + + def request(self, method, url, body=None, headers=None): + self.method = method + self.selector = url + if headers is not None: + self.req_headers += headers.items() + self.req_headers.sort() + if body: + self.data = body + if self.raise_on_endheaders: + import socket + raise socket.error() + + def getresponse(self): + return MockHTTPResponse(MockFile(), {}, 200, "OK") + + def close(self): + pass + +class MockHandler: + # useful for testing handler machinery + # see add_ordered_mock_handlers() docstring + handler_order = 500 + def __init__(self, methods): + self._define_methods(methods) + def _define_methods(self, methods): + for spec in methods: + if len(spec) == 2: name, action = spec + else: name, action = spec, None + meth = FakeMethod(name, action, self.handle) + setattr(self.__class__, name, meth) + def handle(self, fn_name, action, *args, **kwds): + self.parent.calls.append((self, fn_name, args, kwds)) + if action is None: + return None + elif action == "return self": + return self + elif action == "return response": + res = MockResponse(200, "OK", {}, "") + return res + elif action == "return request": + return Request("http://blah/") + elif action.startswith("error"): + code = action[action.rfind(" ")+1:] + try: + code = int(code) + except ValueError: + pass + res = MockResponse(200, "OK", {}, "") + return self.parent.error("http", args[0], res, code, "", {}) + elif action == "raise": + raise urllib2.URLError("blah") + assert False + def close(self): pass + def add_parent(self, parent): + self.parent = parent + self.parent.calls = [] + def __lt__(self, other): + if not hasattr(other, "handler_order"): + # No handler_order, leave in original order. Yuck. + return True + return self.handler_order < other.handler_order + +def add_ordered_mock_handlers(opener, meth_spec): + """Create MockHandlers and add them to an OpenerDirector. + + meth_spec: list of lists of tuples and strings defining methods to define + on handlers. eg: + + [["http_error", "ftp_open"], ["http_open"]] + + defines methods .http_error() and .ftp_open() on one handler, and + .http_open() on another. These methods just record their arguments and + return None. Using a tuple instead of a string causes the method to + perform some action (see MockHandler.handle()), eg: + + [["http_error"], [("http_open", "return request")]] + + defines .http_error() on one handler (which simply returns None), and + .http_open() on another handler, which returns a Request object. + + """ + handlers = [] + count = 0 + for meths in meth_spec: + class MockHandlerSubclass(MockHandler): pass + h = MockHandlerSubclass(meths) + h.handler_order += count + h.add_parent(opener) + count = count + 1 + handlers.append(h) + opener.add_handler(h) + return handlers + +def build_test_opener(*handler_instances): + opener = OpenerDirector() + for h in handler_instances: + opener.add_handler(h) + return opener + +class MockHTTPHandler(urllib2.BaseHandler): + # useful for testing redirections and auth + # sends supplied headers and code as first response + # sends 200 OK as second response + def __init__(self, code, headers): + self.code = code + self.headers = headers + self.reset() + def reset(self): + self._count = 0 + self.requests = [] + def http_open(self, req): + import mimetools, copy + from StringIO import StringIO + self.requests.append(copy.deepcopy(req)) + if self._count == 0: + self._count = self._count + 1 + name = httplib.responses[self.code] + msg = mimetools.Message(StringIO(self.headers)) + return self.parent.error( + "http", req, MockFile(), self.code, name, msg) + else: + self.req = req + msg = mimetools.Message(StringIO("\r\n\r\n")) + return MockResponse(200, "OK", msg, "", req.get_full_url()) + +class MockHTTPSHandler(urllib2.AbstractHTTPHandler): + # Useful for testing the Proxy-Authorization request by verifying the + # properties of httpcon + + def __init__(self): + urllib2.AbstractHTTPHandler.__init__(self) + self.httpconn = MockHTTPClass() + + def https_open(self, req): + return self.do_open(self.httpconn, req) + +class MockPasswordManager: + def add_password(self, realm, uri, user, password): + self.realm = realm + self.url = uri + self.user = user + self.password = password + def find_user_password(self, realm, authuri): + self.target_realm = realm + self.target_url = authuri + return self.user, self.password + + +class OpenerDirectorTests(unittest.TestCase): + + def test_add_non_handler(self): + class NonHandler(object): + pass + self.assertRaises(TypeError, + OpenerDirector().add_handler, NonHandler()) + + def test_badly_named_methods(self): + # test work-around for three methods that accidentally follow the + # naming conventions for handler methods + # (*_open() / *_request() / *_response()) + + # These used to call the accidentally-named methods, causing a + # TypeError in real code; here, returning self from these mock + # methods would either cause no exception, or AttributeError. + + from urllib2 import URLError + + o = OpenerDirector() + meth_spec = [ + [("do_open", "return self"), ("proxy_open", "return self")], + [("redirect_request", "return self")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + o.add_handler(urllib2.UnknownHandler()) + for scheme in "do", "proxy", "redirect": + self.assertRaises(URLError, o.open, scheme+"://example.com/") + + def test_handled(self): + # handler returning non-None means no more handlers will be called + o = OpenerDirector() + meth_spec = [ + ["http_open", "ftp_open", "http_error_302"], + ["ftp_open"], + [("http_open", "return self")], + [("http_open", "return self")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + r = o.open(req) + # Second .http_open() gets called, third doesn't, since second returned + # non-None. Handlers without .http_open() never get any methods called + # on them. + # In fact, second mock handler defining .http_open() returns self + # (instead of response), which becomes the OpenerDirector's return + # value. + self.assertEqual(r, handlers[2]) + calls = [(handlers[0], "http_open"), (handlers[2], "http_open")] + for expected, got in zip(calls, o.calls): + handler, name, args, kwds = got + self.assertEqual((handler, name), expected) + self.assertEqual(args, (req,)) + + def test_handler_order(self): + o = OpenerDirector() + handlers = [] + for meths, handler_order in [ + ([("http_open", "return self")], 500), + (["http_open"], 0), + ]: + class MockHandlerSubclass(MockHandler): pass + h = MockHandlerSubclass(meths) + h.handler_order = handler_order + handlers.append(h) + o.add_handler(h) + + r = o.open("http://example.com/") + # handlers called in reverse order, thanks to their sort order + self.assertEqual(o.calls[0][0], handlers[1]) + self.assertEqual(o.calls[1][0], handlers[0]) + + def test_raise(self): + # raising URLError stops processing of request + o = OpenerDirector() + meth_spec = [ + [("http_open", "raise")], + [("http_open", "return self")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + self.assertRaises(urllib2.URLError, o.open, req) + self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) + +## def test_error(self): +## # XXX this doesn't actually seem to be used in standard library, +## # but should really be tested anyway... + + def test_http_error(self): + # XXX http_error_default + # http errors are a special case + o = OpenerDirector() + meth_spec = [ + [("http_open", "error 302")], + [("http_error_400", "raise"), "http_open"], + [("http_error_302", "return response"), "http_error_303", + "http_error"], + [("http_error_302")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + class Unknown: + def __eq__(self, other): return True + + req = Request("http://example.com/") + r = o.open(req) + assert len(o.calls) == 2 + calls = [(handlers[0], "http_open", (req,)), + (handlers[2], "http_error_302", + (req, Unknown(), 302, "", {}))] + for expected, got in zip(calls, o.calls): + handler, method_name, args = expected + self.assertEqual((handler, method_name), got[:2]) + self.assertEqual(args, got[2]) + + def test_processors(self): + # *_request / *_response methods get called appropriately + o = OpenerDirector() + meth_spec = [ + [("http_request", "return request"), + ("http_response", "return response")], + [("http_request", "return request"), + ("http_response", "return response")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + r = o.open(req) + # processor methods are called on *all* handlers that define them, + # not just the first handler that handles the request + calls = [ + (handlers[0], "http_request"), (handlers[1], "http_request"), + (handlers[0], "http_response"), (handlers[1], "http_response")] + + for i, (handler, name, args, kwds) in enumerate(o.calls): + if i < 2: + # *_request + self.assertEqual((handler, name), calls[i]) + self.assertEqual(len(args), 1) + self.assertIsInstance(args[0], Request) + else: + # *_response + self.assertEqual((handler, name), calls[i]) + self.assertEqual(len(args), 2) + self.assertIsInstance(args[0], Request) + # response from opener.open is None, because there's no + # handler that defines http_open to handle it + if args[1] is not None: + self.assertIsInstance(args[1], MockResponse) + + +def sanepathname2url(path): + import urllib + urlpath = urllib.pathname2url(path) + if os.name == "nt" and urlpath.startswith("///"): + urlpath = urlpath[2:] + # XXX don't ask me about the mac... + return urlpath + +class HandlerTests(unittest.TestCase): + + def test_ftp(self): + class MockFTPWrapper: + def __init__(self, data): self.data = data + def retrfile(self, filename, filetype): + self.filename, self.filetype = filename, filetype + return StringIO.StringIO(self.data), len(self.data) + def close(self): pass + + class NullFTPHandler(urllib2.FTPHandler): + def __init__(self, data): self.data = data + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.user, self.passwd = user, passwd + self.host, self.port = host, port + self.dirs = dirs + self.ftpwrapper = MockFTPWrapper(self.data) + return self.ftpwrapper + + import ftplib + data = "rheum rhaponicum" + h = NullFTPHandler(data) + o = h.parent = MockOpener() + + for url, host, port, user, passwd, type_, dirs, filename, mimetype in [ + ("ftp://localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://%25parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "%parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://%2542parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "%42parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://localhost:80/foo/bar/", + "localhost", 80, "", "", "D", + ["foo", "bar"], "", None), + ("ftp://localhost/baz.gif;type=a", + "localhost", ftplib.FTP_PORT, "", "", "A", + [], "baz.gif", None), # XXX really this should guess image/gif + ]: + req = Request(url) + req.timeout = None + r = h.ftp_open(req) + # ftp authentication not yet implemented by FTPHandler + self.assertEqual(h.user, user) + self.assertEqual(h.passwd, passwd) + self.assertEqual(h.host, socket.gethostbyname(host)) + self.assertEqual(h.port, port) + self.assertEqual(h.dirs, dirs) + self.assertEqual(h.ftpwrapper.filename, filename) + self.assertEqual(h.ftpwrapper.filetype, type_) + headers = r.info() + self.assertEqual(headers.get("Content-type"), mimetype) + self.assertEqual(int(headers["Content-length"]), len(data)) + + def test_file(self): + import rfc822, socket + h = urllib2.FileHandler() + o = h.parent = MockOpener() + + TESTFN = test_support.TESTFN + urlpath = sanepathname2url(os.path.abspath(TESTFN)) + towrite = "hello, world\n" + urls = [ + "file://localhost%s" % urlpath, + "file://%s" % urlpath, + "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), + ] + try: + localaddr = socket.gethostbyname(socket.gethostname()) + except socket.gaierror: + localaddr = '' + if localaddr: + urls.append("file://%s%s" % (localaddr, urlpath)) + + for url in urls: + f = open(TESTFN, "wb") + try: + try: + f.write(towrite) + finally: + f.close() + + r = h.file_open(Request(url)) + try: + data = r.read() + headers = r.info() + respurl = r.geturl() + finally: + r.close() + stats = os.stat(TESTFN) + modified = rfc822.formatdate(stats.st_mtime) + finally: + os.remove(TESTFN) + self.assertEqual(data, towrite) + self.assertEqual(headers["Content-type"], "text/plain") + self.assertEqual(headers["Content-length"], "13") + self.assertEqual(headers["Last-modified"], modified) + self.assertEqual(respurl, url) + + for url in [ + "file://localhost:80%s" % urlpath, + "file:///file_does_not_exist.txt", + "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), + os.getcwd(), TESTFN), + "file://somerandomhost.ontheinternet.com%s/%s" % + (os.getcwd(), TESTFN), + ]: + try: + f = open(TESTFN, "wb") + try: + f.write(towrite) + finally: + f.close() + + self.assertRaises(urllib2.URLError, + h.file_open, Request(url)) + finally: + os.remove(TESTFN) + + h = urllib2.FileHandler() + o = h.parent = MockOpener() + # XXXX why does // mean ftp (and /// mean not ftp!), and where + # is file: scheme specified? I think this is really a bug, and + # what was intended was to distinguish between URLs like: + # file:/blah.txt (a file) + # file://localhost/blah.txt (a file) + # file:///blah.txt (a file) + # file://ftp.example.com/blah.txt (an ftp URL) + for url, ftp in [ + ("file://ftp.example.com//foo.txt", True), + ("file://ftp.example.com///foo.txt", False), +# XXXX bug: fails with OSError, should be URLError + ("file://ftp.example.com/foo.txt", False), + ("file://somehost//foo/something.txt", True), + ("file://localhost//foo/something.txt", False), + ]: + req = Request(url) + try: + h.file_open(req) + # XXXX remove OSError when bug fixed + except (urllib2.URLError, OSError): + self.assertTrue(not ftp) + else: + self.assertTrue(o.req is req) + self.assertEqual(req.type, "ftp") + self.assertEqual(req.type == "ftp", ftp) + + def test_http(self): + + h = urllib2.AbstractHTTPHandler() + o = h.parent = MockOpener() + + url = "http://example.com/" + for method, data in [("GET", None), ("POST", "blah")]: + req = Request(url, data, {"Foo": "bar"}) + req.timeout = None + req.add_unredirected_header("Spam", "eggs") + http = MockHTTPClass() + r = h.do_open(http, req) + + # result attributes + r.read; r.readline # wrapped MockFile methods + r.info; r.geturl # addinfourl methods + r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply() + hdrs = r.info() + hdrs.get; hdrs.has_key # r.info() gives dict from .getreply() + self.assertEqual(r.geturl(), url) + + self.assertEqual(http.host, "example.com") + self.assertEqual(http.level, 0) + self.assertEqual(http.method, method) + self.assertEqual(http.selector, "/") + self.assertEqual(http.req_headers, + [("Connection", "close"), + ("Foo", "bar"), ("Spam", "eggs")]) + self.assertEqual(http.data, data) + + # check socket.error converted to URLError + http.raise_on_endheaders = True + self.assertRaises(urllib2.URLError, h.do_open, http, req) + + # check adding of standard headers + o.addheaders = [("Spam", "eggs")] + for data in "", None: # POST, GET + req = Request("http://example.com/", data) + r = MockResponse(200, "OK", {}, "") + newreq = h.do_request_(req) + if data is None: # GET + self.assertNotIn("Content-length", req.unredirected_hdrs) + self.assertNotIn("Content-type", req.unredirected_hdrs) + else: # POST + self.assertEqual(req.unredirected_hdrs["Content-length"], "0") + self.assertEqual(req.unredirected_hdrs["Content-type"], + "application/x-www-form-urlencoded") + # XXX the details of Host could be better tested + self.assertEqual(req.unredirected_hdrs["Host"], "example.com") + self.assertEqual(req.unredirected_hdrs["Spam"], "eggs") + + # don't clobber existing headers + req.add_unredirected_header("Content-length", "foo") + req.add_unredirected_header("Content-type", "bar") + req.add_unredirected_header("Host", "baz") + req.add_unredirected_header("Spam", "foo") + newreq = h.do_request_(req) + self.assertEqual(req.unredirected_hdrs["Content-length"], "foo") + self.assertEqual(req.unredirected_hdrs["Content-type"], "bar") + self.assertEqual(req.unredirected_hdrs["Host"], "baz") + self.assertEqual(req.unredirected_hdrs["Spam"], "foo") + + def test_http_doubleslash(self): + # Checks that the presence of an unnecessary double slash in a url doesn't break anything + # Previously, a double slash directly after the host could cause incorrect parsing of the url + h = urllib2.AbstractHTTPHandler() + o = h.parent = MockOpener() + + data = "" + ds_urls = [ + "http://example.com/foo/bar/baz.html", + "http://example.com//foo/bar/baz.html", + "http://example.com/foo//bar/baz.html", + "http://example.com/foo/bar//baz.html", + ] + + for ds_url in ds_urls: + ds_req = Request(ds_url, data) + + # Check whether host is determined correctly if there is no proxy + np_ds_req = h.do_request_(ds_req) + self.assertEqual(np_ds_req.unredirected_hdrs["Host"],"example.com") + + # Check whether host is determined correctly if there is a proxy + ds_req.set_proxy("someproxy:3128",None) + p_ds_req = h.do_request_(ds_req) + self.assertEqual(p_ds_req.unredirected_hdrs["Host"],"example.com") + + def test_fixpath_in_weirdurls(self): + # Issue4493: urllib2 to supply '/' when to urls where path does not + # start with'/' + + h = urllib2.AbstractHTTPHandler() + o = h.parent = MockOpener() + + weird_url = 'http://www.python.org?getspam' + req = Request(weird_url) + newreq = h.do_request_(req) + self.assertEqual(newreq.get_host(),'www.python.org') + self.assertEqual(newreq.get_selector(),'/?getspam') + + url_without_path = 'http://www.python.org' + req = Request(url_without_path) + newreq = h.do_request_(req) + self.assertEqual(newreq.get_host(),'www.python.org') + self.assertEqual(newreq.get_selector(),'') + + def test_errors(self): + h = urllib2.HTTPErrorProcessor() + o = h.parent = MockOpener() + + url = "http://example.com/" + req = Request(url) + # all 2xx are passed through + r = MockResponse(200, "OK", {}, "", url) + newr = h.http_response(req, r) + self.assertTrue(r is newr) + self.assertTrue(not hasattr(o, "proto")) # o.error not called + r = MockResponse(202, "Accepted", {}, "", url) + newr = h.http_response(req, r) + self.assertTrue(r is newr) + self.assertTrue(not hasattr(o, "proto")) # o.error not called + r = MockResponse(206, "Partial content", {}, "", url) + newr = h.http_response(req, r) + self.assertTrue(r is newr) + self.assertTrue(not hasattr(o, "proto")) # o.error not called + # anything else calls o.error (and MockOpener returns None, here) + r = MockResponse(502, "Bad gateway", {}, "", url) + self.assertTrue(h.http_response(req, r) is None) + self.assertEqual(o.proto, "http") # o.error called + self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) + + def test_cookies(self): + cj = MockCookieJar() + h = urllib2.HTTPCookieProcessor(cj) + o = h.parent = MockOpener() + + req = Request("http://example.com/") + r = MockResponse(200, "OK", {}, "") + newreq = h.http_request(req) + self.assertTrue(cj.ach_req is req is newreq) + self.assertEqual(req.get_origin_req_host(), "example.com") + self.assertTrue(not req.is_unverifiable()) + newr = h.http_response(req, r) + self.assertTrue(cj.ec_req is req) + self.assertTrue(cj.ec_r is r is newr) + + def test_redirect(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib2.HTTPRedirectHandler() + o = h.parent = MockOpener() + + # ordinary redirect behaviour + for code in 301, 302, 303, 307: + for data in None, "blah\nblah\n": + method = getattr(h, "http_error_%s" % code) + req = Request(from_url, data) + req.add_header("Nonsense", "viking=withhold") + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + if data is not None: + req.add_header("Content-Length", str(len(data))) + req.add_unredirected_header("Spam", "spam") + try: + method(req, MockFile(), code, "Blah", + MockHeaders({"location": to_url})) + except urllib2.HTTPError: + # 307 in response to POST requires user OK + self.assertEqual(code, 307) + self.assertIsNotNone(data) + self.assertEqual(o.req.get_full_url(), to_url) + try: + self.assertEqual(o.req.get_method(), "GET") + except AttributeError: + self.assertTrue(not o.req.has_data()) + + # now it's a GET, there should not be headers regarding content + # (possibly dragged from before being a POST) + headers = [x.lower() for x in o.req.headers] + self.assertNotIn("content-length", headers) + self.assertNotIn("content-type", headers) + + self.assertEqual(o.req.headers["Nonsense"], + "viking=withhold") + self.assertNotIn("Spam", o.req.headers) + self.assertNotIn("Spam", o.req.unredirected_hdrs) + + # loop detection + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + def redirect(h, req, url=to_url): + h.http_error_302(req, MockFile(), 302, "Blah", + MockHeaders({"location": url})) + # Note that the *original* request shares the same record of + # redirections with the sub-requests caused by the redirections. + + # detect infinite loop redirect of a URL to itself + req = Request(from_url, origin_req_host="example.com") + count = 0 + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + try: + while 1: + redirect(h, req, "http://example.com/") + count = count + 1 + except urllib2.HTTPError: + # don't stop until max_repeats, because cookies may introduce state + self.assertEqual(count, urllib2.HTTPRedirectHandler.max_repeats) + + # detect endless non-repeating chain of redirects + req = Request(from_url, origin_req_host="example.com") + count = 0 + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + try: + while 1: + redirect(h, req, "http://example.com/%d" % count) + count = count + 1 + except urllib2.HTTPError: + self.assertEqual(count, + urllib2.HTTPRedirectHandler.max_redirections) + + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http', 'https', 'ftp'] + invalid_schemes = ['file', 'imap', 'ldap'] + schemeless_url = "example.com/b.html" + h = urllib2.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib2.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + + def test_cookie_redirect(self): + # cookies shouldn't leak into redirected requests + from cookielib import CookieJar + + from test.test_cookielib import interact_netscape + + cj = CookieJar() + interact_netscape(cj, "http://www.example.com/", "spam=eggs") + hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n") + hdeh = urllib2.HTTPDefaultErrorHandler() + hrh = urllib2.HTTPRedirectHandler() + cp = urllib2.HTTPCookieProcessor(cj) + o = build_test_opener(hh, hdeh, hrh, cp) + o.open("http://www.example.com/") + self.assertTrue(not hh.req.has_header("Cookie")) + + def test_redirect_fragment(self): + redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' + hh = MockHTTPHandler(302, 'Location: ' + redirected_url) + hdeh = urllib2.HTTPDefaultErrorHandler() + hrh = urllib2.HTTPRedirectHandler() + o = build_test_opener(hh, hdeh, hrh) + fp = o.open('http://www.example.com') + self.assertEqual(fp.geturl(), redirected_url.strip()) + + def test_redirect_no_path(self): + # Issue 14132: Relative redirect strips original path + real_class = httplib.HTTPConnection + response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n" + httplib.HTTPConnection = test_urllib.fakehttp(response1) + self.addCleanup(setattr, httplib, "HTTPConnection", real_class) + urls = iter(("/path", "/path?query")) + def request(conn, method, url, *pos, **kw): + self.assertEqual(url, next(urls)) + real_class.request(conn, method, url, *pos, **kw) + # Change response for subsequent connection + conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!" + httplib.HTTPConnection.request = request + fp = urllib2.urlopen("http://python.org/path") + self.assertEqual(fp.geturl(), "http://python.org/path?query") + + def test_proxy(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(http="proxy.example.com:3128")) + o.add_handler(ph) + meth_spec = [ + [("http_open", "return response")] + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://acme.example.com/") + self.assertEqual(req.get_host(), "acme.example.com") + r = o.open(req) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + + self.assertEqual([(handlers[0], "http_open")], + [tup[0:2] for tup in o.calls]) + + def test_proxy_no_proxy(self): + os.environ['no_proxy'] = 'python.org' + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(http="proxy.example.com")) + o.add_handler(ph) + req = Request("http://www.perl.org/") + self.assertEqual(req.get_host(), "www.perl.org") + r = o.open(req) + self.assertEqual(req.get_host(), "proxy.example.com") + req = Request("http://www.python.org") + self.assertEqual(req.get_host(), "www.python.org") + r = o.open(req) + self.assertEqual(req.get_host(), "www.python.org") + del os.environ['no_proxy'] + + + def test_proxy_https(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(https='proxy.example.com:3128')) + o.add_handler(ph) + meth_spec = [ + [("https_open","return response")] + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + req = Request("https://www.example.com/") + self.assertEqual(req.get_host(), "www.example.com") + r = o.open(req) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + self.assertEqual([(handlers[0], "https_open")], + [tup[0:2] for tup in o.calls]) + + def test_proxy_https_proxy_authorization(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(https='proxy.example.com:3128')) + o.add_handler(ph) + https_handler = MockHTTPSHandler() + o.add_handler(https_handler) + req = Request("https://www.example.com/") + req.add_header("Proxy-Authorization","FooBar") + req.add_header("User-Agent","Grail") + self.assertEqual(req.get_host(), "www.example.com") + self.assertIsNone(req._tunnel_host) + r = o.open(req) + # Verify Proxy-Authorization gets tunneled to request. + # httpsconn req_headers do not have the Proxy-Authorization header but + # the req will have. + self.assertNotIn(("Proxy-Authorization","FooBar"), + https_handler.httpconn.req_headers) + self.assertIn(("User-Agent","Grail"), + https_handler.httpconn.req_headers) + self.assertIsNotNone(req._tunnel_host) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + self.assertEqual(req.get_header("Proxy-authorization"),"FooBar") + + def test_basic_auth(self, quote_char='"'): + opener = OpenerDirector() + password_manager = MockPasswordManager() + auth_handler = urllib2.HTTPBasicAuthHandler(password_manager) + realm = "ACME Widget Store" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' % + (quote_char, realm, quote_char) ) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + self._test_basic_auth(opener, auth_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected" + ) + + def test_basic_auth_with_single_quoted_realm(self): + self.test_basic_auth(quote_char="'") + + def test_basic_auth_with_unquoted_realm(self): + opener = OpenerDirector() + password_manager = MockPasswordManager() + auth_handler = urllib2.HTTPBasicAuthHandler(password_manager) + realm = "ACME Widget Store" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + msg = "Basic Auth Realm was unquoted" + with test_support.check_warnings((msg, UserWarning)): + self._test_basic_auth(opener, auth_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected" + ) + + + def test_proxy_basic_auth(self): + opener = OpenerDirector() + ph = urllib2.ProxyHandler(dict(http="proxy.example.com:3128")) + opener.add_handler(ph) + password_manager = MockPasswordManager() + auth_handler = urllib2.ProxyBasicAuthHandler(password_manager) + realm = "ACME Networks" + http_handler = MockHTTPHandler( + 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + self._test_basic_auth(opener, auth_handler, "Proxy-authorization", + realm, http_handler, password_manager, + "http://acme.example.com:3128/protected", + "proxy.example.com:3128", + ) + + def test_basic_and_digest_auth_handlers(self): + # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* + # response (http://python.org/sf/1479302), where it should instead + # return None to allow another handler (especially + # HTTPBasicAuthHandler) to handle the response. + + # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must + # try digest first (since it's the strongest auth scheme), so we record + # order of calls here to check digest comes first: + class RecordingOpenerDirector(OpenerDirector): + def __init__(self): + OpenerDirector.__init__(self) + self.recorded = [] + def record(self, info): + self.recorded.append(info) + class TestDigestAuthHandler(urllib2.HTTPDigestAuthHandler): + def http_error_401(self, *args, **kwds): + self.parent.record("digest") + urllib2.HTTPDigestAuthHandler.http_error_401(self, + *args, **kwds) + class TestBasicAuthHandler(urllib2.HTTPBasicAuthHandler): + def http_error_401(self, *args, **kwds): + self.parent.record("basic") + urllib2.HTTPBasicAuthHandler.http_error_401(self, + *args, **kwds) + + opener = RecordingOpenerDirector() + password_manager = MockPasswordManager() + digest_handler = TestDigestAuthHandler(password_manager) + basic_handler = TestBasicAuthHandler(password_manager) + realm = "ACME Networks" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) + opener.add_handler(basic_handler) + opener.add_handler(digest_handler) + opener.add_handler(http_handler) + + # check basic auth isn't blocked by digest handler failing + self._test_basic_auth(opener, basic_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected", + ) + # check digest was tried before basic (twice, because + # _test_basic_auth called .open() twice) + self.assertEqual(opener.recorded, ["digest", "basic"]*2) + + def _test_basic_auth(self, opener, auth_handler, auth_header, + realm, http_handler, password_manager, + request_url, protected_url): + import base64 + user, password = "wile", "coyote" + + # .add_password() fed through to password manager + auth_handler.add_password(realm, request_url, user, password) + self.assertEqual(realm, password_manager.realm) + self.assertEqual(request_url, password_manager.url) + self.assertEqual(user, password_manager.user) + self.assertEqual(password, password_manager.password) + + r = opener.open(request_url) + + # should have asked the password manager for the username/password + self.assertEqual(password_manager.target_realm, realm) + self.assertEqual(password_manager.target_url, protected_url) + + # expect one request without authorization, then one with + self.assertEqual(len(http_handler.requests), 2) + self.assertFalse(http_handler.requests[0].has_header(auth_header)) + userpass = '%s:%s' % (user, password) + auth_hdr_value = 'Basic '+base64.encodestring(userpass).strip() + self.assertEqual(http_handler.requests[1].get_header(auth_header), + auth_hdr_value) + self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header], + auth_hdr_value) + # if the password manager can't find a password, the handler won't + # handle the HTTP auth error + password_manager.user = password_manager.password = None + http_handler.reset() + r = opener.open(request_url) + self.assertEqual(len(http_handler.requests), 1) + self.assertFalse(http_handler.requests[0].has_header(auth_header)) + +class MiscTests(unittest.TestCase): + + def test_build_opener(self): + class MyHTTPHandler(urllib2.HTTPHandler): pass + class FooHandler(urllib2.BaseHandler): + def foo_open(self): pass + class BarHandler(urllib2.BaseHandler): + def bar_open(self): pass + + build_opener = urllib2.build_opener + + o = build_opener(FooHandler, BarHandler) + self.opener_has_handler(o, FooHandler) + self.opener_has_handler(o, BarHandler) + + # can take a mix of classes and instances + o = build_opener(FooHandler, BarHandler()) + self.opener_has_handler(o, FooHandler) + self.opener_has_handler(o, BarHandler) + + # subclasses of default handlers override default handlers + o = build_opener(MyHTTPHandler) + self.opener_has_handler(o, MyHTTPHandler) + + # a particular case of overriding: default handlers can be passed + # in explicitly + o = build_opener() + self.opener_has_handler(o, urllib2.HTTPHandler) + o = build_opener(urllib2.HTTPHandler) + self.opener_has_handler(o, urllib2.HTTPHandler) + o = build_opener(urllib2.HTTPHandler()) + self.opener_has_handler(o, urllib2.HTTPHandler) + + # Issue2670: multiple handlers sharing the same base class + class MyOtherHTTPHandler(urllib2.HTTPHandler): pass + o = build_opener(MyHTTPHandler, MyOtherHTTPHandler) + self.opener_has_handler(o, MyHTTPHandler) + self.opener_has_handler(o, MyOtherHTTPHandler) + + def opener_has_handler(self, opener, handler_class): + for h in opener.handlers: + if h.__class__ == handler_class: + break + else: + self.assertTrue(False) + + def test_unsupported_algorithm(self): + handler = AbstractDigestAuthHandler() + with self.assertRaises(ValueError) as exc: + handler.get_algorithm_impls('invalid') + self.assertEqual( + str(exc.exception), + "Unsupported digest authentication algorithm 'invalid'" + ) + + +class RequestTests(unittest.TestCase): + + def setUp(self): + self.get = urllib2.Request("http://www.python.org/~jeremy/") + self.post = urllib2.Request("http://www.python.org/~jeremy/", + "data", + headers={"X-Test": "test"}) + + def test_method(self): + self.assertEqual("POST", self.post.get_method()) + self.assertEqual("GET", self.get.get_method()) + + def test_add_data(self): + self.assertTrue(not self.get.has_data()) + self.assertEqual("GET", self.get.get_method()) + self.get.add_data("spam") + self.assertTrue(self.get.has_data()) + self.assertEqual("POST", self.get.get_method()) + + def test_get_full_url(self): + self.assertEqual("http://www.python.org/~jeremy/", + self.get.get_full_url()) + + def test_selector(self): + self.assertEqual("/~jeremy/", self.get.get_selector()) + req = urllib2.Request("http://www.python.org/") + self.assertEqual("/", req.get_selector()) + + def test_get_type(self): + self.assertEqual("http", self.get.get_type()) + + def test_get_host(self): + self.assertEqual("www.python.org", self.get.get_host()) + + def test_get_host_unquote(self): + req = urllib2.Request("http://www.%70ython.org/") + self.assertEqual("www.python.org", req.get_host()) + + def test_proxy(self): + self.assertTrue(not self.get.has_proxy()) + self.get.set_proxy("www.perl.org", "http") + self.assertTrue(self.get.has_proxy()) + self.assertEqual("www.python.org", self.get.get_origin_req_host()) + self.assertEqual("www.perl.org", self.get.get_host()) + + def test_wrapped_url(self): + req = Request("") + self.assertEqual("www.python.org", req.get_host()) + + def test_url_fragment(self): + req = Request("http://www.python.org/?qs=query#fragment=true") + self.assertEqual("/?qs=query", req.get_selector()) + req = Request("http://www.python.org/#fun=true") + self.assertEqual("/", req.get_selector()) + + # Issue 11703: geturl() omits fragment in the original URL. + url = 'http://docs.python.org/library/urllib2.html#OK' + req = Request(url) + self.assertEqual(req.get_full_url(), url) + + def test_private_attributes(self): + self.assertFalse(hasattr(self.get, '_Request__r_xxx')) + # Issue #6500: infinite recursion + self.assertFalse(hasattr(self.get, '_Request__r_method')) + + def test_HTTPError_interface(self): + """ + Issue 13211 reveals that HTTPError didn't implement the URLError + interface even though HTTPError is a subclass of URLError. + + >>> err = urllib2.HTTPError(msg='something bad happened', url=None, code=None, hdrs=None, fp=None) + >>> assert hasattr(err, 'reason') + >>> err.reason + 'something bad happened' + """ + + def test_HTTPError_interface_call(self): + """ + Issue 15701= - HTTPError interface has info method available from URLError. + """ + err = urllib2.HTTPError(msg='something bad happened', url=None, + code=None, hdrs='Content-Length:42', fp=None) + self.assertTrue(hasattr(err, 'reason')) + assert hasattr(err, 'reason') + assert hasattr(err, 'info') + assert callable(err.info) + try: + err.info() + except AttributeError: + self.fail("err.info() failed") + self.assertEqual(err.info(), "Content-Length:42") + +def test_main(verbose=None): + from test import test_urllib2 + test_support.run_doctest(test_urllib2, verbose) + test_support.run_doctest(urllib2, verbose) + tests = (TrivialTests, + OpenerDirectorTests, + HandlerTests, + MiscTests, + RequestTests) + test_support.run_unittest(*tests) + +if __name__ == "__main__": + test_main(verbose=True) diff --git a/src/greentest/2.7pypy/test_urllib2_localnet.py b/src/greentest/2.7pypy/test_urllib2_localnet.py new file mode 100644 index 0000000..9199cb9 --- /dev/null +++ b/src/greentest/2.7pypy/test_urllib2_localnet.py @@ -0,0 +1,717 @@ +import os +import base64 +import urlparse +import urllib2 +import BaseHTTPServer +import unittest +import hashlib + +from test import test_support + +mimetools = test_support.import_module('mimetools', deprecated=True) +threading = test_support.import_module('threading') + +try: + import ssl +except ImportError: + ssl = None + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') + +# Loopback http server infrastructure + +class LoopbackHttpServer(BaseHTTPServer.HTTPServer): + """HTTP server w/ a few modifications that make it useful for + loopback testing purposes. + """ + + def __init__(self, server_address, RequestHandlerClass): + BaseHTTPServer.HTTPServer.__init__(self, + server_address, + RequestHandlerClass) + + # Set the timeout of our listening socket really low so + # that we can stop the server easily. + self.socket.settimeout(0.1) + + def get_request(self): + """BaseHTTPServer method, overridden.""" + + request, client_address = self.socket.accept() + + # It's a loopback connection, so setting the timeout + # really low shouldn't affect anything, but should make + # deadlocks less likely to occur. + request.settimeout(10.0) + + return (request, client_address) + +class LoopbackHttpServerThread(threading.Thread): + """Stoppable thread that runs a loopback http server.""" + + def __init__(self, request_handler): + threading.Thread.__init__(self) + self._stop = False + self.ready = threading.Event() + request_handler.protocol_version = "HTTP/1.0" + self.httpd = LoopbackHttpServer(('127.0.0.1', 0), + request_handler) + #print "Serving HTTP on %s port %s" % (self.httpd.server_name, + # self.httpd.server_port) + self.port = self.httpd.server_port + + def stop(self): + """Stops the webserver if it's currently running.""" + + # Set the stop flag. + self._stop = True + + self.join() + + def run(self): + self.ready.set() + while not self._stop: + self.httpd.handle_request() + +# Authentication infrastructure + + +class BasicAuthHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """Handler for performing Basic Authentication.""" + # Server side values + USER = "testUser" + PASSWD = "testPass" + REALM = "Test" + USER_PASSWD = "%s:%s" % (USER, PASSWD) + ENCODED_AUTH = base64.b64encode(USER_PASSWD) + + def __init__(self, *args, **kwargs): + BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def log_message(self, format, *args): + # Suppress the HTTP Console log output + pass + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_AUTHHEAD(self): + self.send_response(401) + self.send_header("WWW-Authenticate", "Basic realm=\"%s\"" % self.REALM) + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_GET(self): + if self.headers.getheader("Authorization") == None: + self.do_AUTHHEAD() + self.wfile.write("No Auth Header Received") + elif self.headers.getheader( + "Authorization") == "Basic " + self.ENCODED_AUTH: + self.wfile.write("It works!") + else: + # Unauthorized Request + self.do_AUTHHEAD() + + +class DigestAuthHandler: + """Handler for performing digest authentication.""" + + def __init__(self): + self._request_num = 0 + self._nonces = [] + self._users = {} + self._realm_name = "Test Realm" + self._qop = "auth" + + def set_qop(self, qop): + self._qop = qop + + def set_users(self, users): + assert isinstance(users, dict) + self._users = users + + def set_realm(self, realm): + self._realm_name = realm + + def _generate_nonce(self): + self._request_num += 1 + nonce = hashlib.md5(str(self._request_num)).hexdigest() + self._nonces.append(nonce) + return nonce + + def _create_auth_dict(self, auth_str): + first_space_index = auth_str.find(" ") + auth_str = auth_str[first_space_index+1:] + + parts = auth_str.split(",") + + auth_dict = {} + for part in parts: + name, value = part.split("=") + name = name.strip() + if value[0] == '"' and value[-1] == '"': + value = value[1:-1] + else: + value = value.strip() + auth_dict[name] = value + return auth_dict + + def _validate_auth(self, auth_dict, password, method, uri): + final_dict = {} + final_dict.update(auth_dict) + final_dict["password"] = password + final_dict["method"] = method + final_dict["uri"] = uri + HA1_str = "%(username)s:%(realm)s:%(password)s" % final_dict + HA1 = hashlib.md5(HA1_str).hexdigest() + HA2_str = "%(method)s:%(uri)s" % final_dict + HA2 = hashlib.md5(HA2_str).hexdigest() + final_dict["HA1"] = HA1 + final_dict["HA2"] = HA2 + response_str = "%(HA1)s:%(nonce)s:%(nc)s:" \ + "%(cnonce)s:%(qop)s:%(HA2)s" % final_dict + response = hashlib.md5(response_str).hexdigest() + + return response == auth_dict["response"] + + def _return_auth_challenge(self, request_handler): + request_handler.send_response(407, "Proxy Authentication Required") + request_handler.send_header("Content-Type", "text/html") + request_handler.send_header( + 'Proxy-Authenticate', 'Digest realm="%s", ' + 'qop="%s",' + 'nonce="%s", ' % \ + (self._realm_name, self._qop, self._generate_nonce())) + # XXX: Not sure if we're supposed to add this next header or + # not. + #request_handler.send_header('Connection', 'close') + request_handler.end_headers() + request_handler.wfile.write("Proxy Authentication Required.") + return False + + def handle_request(self, request_handler): + """Performs digest authentication on the given HTTP request + handler. Returns True if authentication was successful, False + otherwise. + + If no users have been set, then digest auth is effectively + disabled and this method will always return True. + """ + + if len(self._users) == 0: + return True + + if 'Proxy-Authorization' not in request_handler.headers: + return self._return_auth_challenge(request_handler) + else: + auth_dict = self._create_auth_dict( + request_handler.headers['Proxy-Authorization'] + ) + if auth_dict["username"] in self._users: + password = self._users[ auth_dict["username"] ] + else: + return self._return_auth_challenge(request_handler) + if not auth_dict.get("nonce") in self._nonces: + return self._return_auth_challenge(request_handler) + else: + self._nonces.remove(auth_dict["nonce"]) + + auth_validated = False + + # MSIE uses short_path in its validation, but Python's + # urllib2 uses the full path, so we're going to see if + # either of them works here. + + for path in [request_handler.path, request_handler.short_path]: + if self._validate_auth(auth_dict, + password, + request_handler.command, + path): + auth_validated = True + + if not auth_validated: + return self._return_auth_challenge(request_handler) + return True + +# Proxy test infrastructure + +class FakeProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """This is a 'fake proxy' that makes it look like the entire + internet has gone down due to a sudden zombie invasion. It main + utility is in providing us with authentication support for + testing. + """ + + def __init__(self, digest_auth_handler, *args, **kwargs): + # This has to be set before calling our parent's __init__(), which will + # try to call do_GET(). + self.digest_auth_handler = digest_auth_handler + BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def log_message(self, format, *args): + # Uncomment the next line for debugging. + #sys.stderr.write(format % args) + pass + + def do_GET(self): + (scm, netloc, path, params, query, fragment) = urlparse.urlparse( + self.path, 'http') + self.short_path = path + if self.digest_auth_handler.handle_request(self): + self.send_response(200, "OK") + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write("You've reached %s!
" % self.path) + self.wfile.write("Our apologies, but our server is down due to " + "a sudden zombie invasion.") + +# Test cases + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test_support.threading_setup() + + def tearDown(self): + test_support.threading_cleanup(*self._threads) + + +class BasicAuthTests(BaseTestCase): + USER = "testUser" + PASSWD = "testPass" + INCORRECT_PASSWD = "Incorrect" + REALM = "Test" + + def setUp(self): + super(BasicAuthTests, self).setUp() + # With Basic Authentication + def http_server_with_basic_auth_handler(*args, **kwargs): + return BasicAuthHandler(*args, **kwargs) + self.server = LoopbackHttpServerThread(http_server_with_basic_auth_handler) + self.server_url = 'http://127.0.0.1:%s' % self.server.port + self.server.start() + self.server.ready.wait() + + def tearDown(self): + self.server.stop() + super(BasicAuthTests, self).tearDown() + + def test_basic_auth_success(self): + ah = urllib2.HTTPBasicAuthHandler() + ah.add_password(self.REALM, self.server_url, self.USER, self.PASSWD) + urllib2.install_opener(urllib2.build_opener(ah)) + try: + self.assertTrue(urllib2.urlopen(self.server_url)) + except urllib2.HTTPError: + self.fail("Basic Auth Failed for url: %s" % self.server_url) + except Exception as e: + raise e + + def test_basic_auth_httperror(self): + ah = urllib2.HTTPBasicAuthHandler() + ah.add_password(self.REALM, self.server_url, self.USER, + self.INCORRECT_PASSWD) + urllib2.install_opener(urllib2.build_opener(ah)) + self.assertRaises(urllib2.HTTPError, urllib2.urlopen, self.server_url) + + +class ProxyAuthTests(BaseTestCase): + URL = "http://localhost" + + USER = "tester" + PASSWD = "test123" + REALM = "TestRealm" + + def setUp(self): + super(ProxyAuthTests, self).setUp() + # Ignore proxy bypass settings in the environment. + def restore_environ(old_environ): + os.environ.clear() + os.environ.update(old_environ) + self.addCleanup(restore_environ, os.environ.copy()) + os.environ['NO_PROXY'] = '' + os.environ['no_proxy'] = '' + + self.digest_auth_handler = DigestAuthHandler() + self.digest_auth_handler.set_users({self.USER: self.PASSWD}) + self.digest_auth_handler.set_realm(self.REALM) + # With Digest Authentication + def create_fake_proxy_handler(*args, **kwargs): + return FakeProxyHandler(self.digest_auth_handler, *args, **kwargs) + + self.server = LoopbackHttpServerThread(create_fake_proxy_handler) + self.server.start() + self.server.ready.wait() + proxy_url = "http://127.0.0.1:%d" % self.server.port + handler = urllib2.ProxyHandler({"http" : proxy_url}) + self.proxy_digest_handler = urllib2.ProxyDigestAuthHandler() + self.opener = urllib2.build_opener(handler, self.proxy_digest_handler) + + def tearDown(self): + self.server.stop() + super(ProxyAuthTests, self).tearDown() + + def test_proxy_with_bad_password_raises_httperror(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD+"bad") + self.digest_auth_handler.set_qop("auth") + self.assertRaises(urllib2.HTTPError, + self.opener.open, + self.URL) + + def test_proxy_with_no_password_raises_httperror(self): + self.digest_auth_handler.set_qop("auth") + self.assertRaises(urllib2.HTTPError, + self.opener.open, + self.URL) + + def test_proxy_qop_auth_works(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD) + self.digest_auth_handler.set_qop("auth") + result = self.opener.open(self.URL) + while result.read(): + pass + result.close() + + def test_proxy_qop_auth_int_works_or_throws_urlerror(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD) + self.digest_auth_handler.set_qop("auth-int") + try: + result = self.opener.open(self.URL) + except urllib2.URLError: + # It's okay if we don't support auth-int, but we certainly + # shouldn't receive any kind of exception here other than + # a URLError. + result = None + if result: + while result.read(): + pass + result.close() + + +def GetRequestHandler(responses): + + class FakeHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + + server_version = "TestHTTP/" + requests = [] + headers_received = [] + port = 80 + + def do_GET(self): + body = self.send_head() + if body: + self.wfile.write(body) + + def do_POST(self): + content_length = self.headers['Content-Length'] + post_data = self.rfile.read(int(content_length)) + self.do_GET() + self.requests.append(post_data) + + def send_head(self): + FakeHTTPRequestHandler.headers_received = self.headers + self.requests.append(self.path) + response_code, headers, body = responses.pop(0) + + self.send_response(response_code) + + for (header, value) in headers: + self.send_header(header, value % self.port) + if body: + self.send_header('Content-type', 'text/plain') + self.end_headers() + return body + self.end_headers() + + def log_message(self, *args): + pass + + + return FakeHTTPRequestHandler + + +class TestUrlopen(BaseTestCase): + """Tests urllib2.urlopen using the network. + + These tests are not exhaustive. Assuming that testing using files does a + good job overall of some of the basic interface features. There are no + tests exercising the optional 'data' and 'proxies' arguments. No tests + for transparent redirection have been written. + """ + + def setUp(self): + proxy_handler = urllib2.ProxyHandler({}) + opener = urllib2.build_opener(proxy_handler) + urllib2.install_opener(opener) + super(TestUrlopen, self).setUp() + + def urlopen(self, url, data=None, **kwargs): + l = [] + f = urllib2.urlopen(url, data, **kwargs) + try: + # Exercise various methods + l.extend(f.readlines(200)) + l.append(f.readline()) + l.append(f.read(1024)) + l.append(f.read()) + finally: + f.close() + return b"".join(l) + + def start_server(self, responses): + handler = GetRequestHandler(responses) + + self.server = LoopbackHttpServerThread(handler) + self.server.start() + self.server.ready.wait() + port = self.server.port + handler.port = port + return handler + + def start_https_server(self, responses=None, **kwargs): + if not hasattr(urllib2, 'HTTPSHandler'): + self.skipTest('ssl support required') + from test.ssl_servers import make_https_server + if responses is None: + responses = [(200, [], b"we care a bit")] + handler = GetRequestHandler(responses) + server = make_https_server(self, handler_class=handler, **kwargs) + handler.port = server.port + return handler + + def test_redirection(self): + expected_response = 'We got here...' + responses = [ + (302, [('Location', 'http://localhost:%s/somewhere_else')], ''), + (200, [], expected_response) + ] + + handler = self.start_server(responses) + + try: + f = urllib2.urlopen('http://localhost:%s/' % handler.port) + data = f.read() + f.close() + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ['/', '/somewhere_else']) + finally: + self.server.stop() + + + def test_404(self): + expected_response = 'Bad bad bad...' + handler = self.start_server([(404, [], expected_response)]) + + try: + try: + urllib2.urlopen('http://localhost:%s/weeble' % handler.port) + except urllib2.URLError, f: + pass + else: + self.fail('404 should raise URLError') + + data = f.read() + f.close() + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ['/weeble']) + finally: + self.server.stop() + + + def test_200(self): + expected_response = 'pycon 2008...' + handler = self.start_server([(200, [], expected_response)]) + + try: + f = urllib2.urlopen('http://localhost:%s/bizarre' % handler.port) + data = f.read() + f.close() + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ['/bizarre']) + finally: + self.server.stop() + + def test_200_with_parameters(self): + expected_response = 'pycon 2008...' + handler = self.start_server([(200, [], expected_response)]) + + try: + f = urllib2.urlopen('http://localhost:%s/bizarre' % handler.port, 'get=with_feeling') + data = f.read() + f.close() + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ['/bizarre', 'get=with_feeling']) + finally: + self.server.stop() + + def test_https(self): + handler = self.start_https_server() + context = ssl.create_default_context(cafile=CERT_localhost) + data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context) + self.assertEqual(data, b"we care a bit") + + def test_https_with_cafile(self): + handler = self.start_https_server(certfile=CERT_localhost) + # Good cert + data = self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_localhost) + self.assertEqual(data, b"we care a bit") + # Bad cert + with self.assertRaises(urllib2.URLError): + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_fakehostname) + # Good cert, but mismatching hostname + handler = self.start_https_server(certfile=CERT_fakehostname) + with self.assertRaises(ssl.CertificateError): + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_fakehostname) + + def test_https_with_cadefault(self): + handler = self.start_https_server(certfile=CERT_localhost) + # Self-signed cert should fail verification with system certificate store + with self.assertRaises(urllib2.URLError): + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cadefault=True) + + def test_https_sni(self): + if ssl is None: + self.skipTest("ssl module required") + if not ssl.HAS_SNI: + self.skipTest("SNI support required in OpenSSL") + sni_name = [None] + def cb_sni(ssl_sock, server_name, initial_context): + sni_name[0] = server_name + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.set_servername_callback(cb_sni) + handler = self.start_https_server(context=context, certfile=CERT_localhost) + context = ssl.create_default_context(cafile=CERT_localhost) + self.urlopen("https://localhost:%s" % handler.port, context=context) + self.assertEqual(sni_name[0], "localhost") + + def test_sending_headers(self): + handler = self.start_server([(200, [], "we don't care")]) + + try: + req = urllib2.Request("http://localhost:%s/" % handler.port, + headers={'Range': 'bytes=20-39'}) + urllib2.urlopen(req) + self.assertEqual(handler.headers_received['Range'], 'bytes=20-39') + finally: + self.server.stop() + + def test_basic(self): + handler = self.start_server([(200, [], "we don't care")]) + + try: + open_url = urllib2.urlopen("http://localhost:%s" % handler.port) + for attr in ("read", "close", "info", "geturl"): + self.assertTrue(hasattr(open_url, attr), "object returned from " + "urlopen lacks the %s attribute" % attr) + try: + self.assertTrue(open_url.read(), "calling 'read' failed") + finally: + open_url.close() + finally: + self.server.stop() + + def test_info(self): + handler = self.start_server([(200, [], "we don't care")]) + + try: + open_url = urllib2.urlopen("http://localhost:%s" % handler.port) + info_obj = open_url.info() + self.assertIsInstance(info_obj, mimetools.Message, + "object returned by 'info' is not an " + "instance of mimetools.Message") + self.assertEqual(info_obj.getsubtype(), "plain") + finally: + self.server.stop() + + def test_geturl(self): + # Make sure same URL as opened is returned by geturl. + handler = self.start_server([(200, [], "we don't care")]) + + try: + open_url = urllib2.urlopen("http://localhost:%s" % handler.port) + url = open_url.geturl() + self.assertEqual(url, "http://localhost:%s" % handler.port) + finally: + self.server.stop() + + + def test_bad_address(self): + # Make sure proper exception is raised when connecting to a bogus + # address. + + # as indicated by the comment below, this might fail with some ISP, + # so we run the test only when -unetwork/-uall is specified to + # mitigate the problem a bit (see #17564) + test_support.requires('network') + self.assertRaises(IOError, + # Given that both VeriSign and various ISPs have in + # the past or are presently hijacking various invalid + # domain name requests in an attempt to boost traffic + # to their own sites, finding a domain name to use + # for this test is difficult. RFC2606 leads one to + # believe that '.invalid' should work, but experience + # seemed to indicate otherwise. Single character + # TLDs are likely to remain invalid, so this seems to + # be the best choice. The trailing '.' prevents a + # related problem: The normal DNS resolver appends + # the domain names from the search path if there is + # no '.' the end and, and if one of those domains + # implements a '*' rule a result is returned. + # However, none of this will prevent the test from + # failing if the ISP hijacks all invalid domain + # requests. The real solution would be to be able to + # parameterize the framework with a mock resolver. + urllib2.urlopen, "http://sadflkjsasf.i.nvali.d./") + + def test_iteration(self): + expected_response = "pycon 2008..." + handler = self.start_server([(200, [], expected_response)]) + try: + data = urllib2.urlopen("http://localhost:%s" % handler.port) + for line in data: + self.assertEqual(line, expected_response) + finally: + self.server.stop() + + def ztest_line_iteration(self): + lines = ["We\n", "got\n", "here\n", "verylong " * 8192 + "\n"] + expected_response = "".join(lines) + handler = self.start_server([(200, [], expected_response)]) + try: + data = urllib2.urlopen("http://localhost:%s" % handler.port) + for index, line in enumerate(data): + self.assertEqual(line, lines[index], + "Fetched line number %s doesn't match expected:\n" + " Expected length was %s, got %s" % + (index, len(lines[index]), len(line))) + finally: + self.server.stop() + self.assertEqual(index + 1, len(lines)) + +def test_main(): + # We will NOT depend on the network resource flag + # (Lib/test/regrtest.py -u network) since all tests here are only + # localhost. However, if this is a bad rationale, then uncomment + # the next line. + #test_support.requires("network") + + test_support.run_unittest(BasicAuthTests, ProxyAuthTests, TestUrlopen) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_urllib2net.py b/src/greentest/2.7pypy/test_urllib2net.py new file mode 100644 index 0000000..66cee7b --- /dev/null +++ b/src/greentest/2.7pypy/test_urllib2net.py @@ -0,0 +1,339 @@ +import unittest +from test import test_support +from test.test_urllib2 import sanepathname2url + +import socket +import urllib2 +import os +import sys + +TIMEOUT = 60 # seconds + + +def _retry_thrice(func, exc, *args, **kwargs): + for i in range(3): + try: + return func(*args, **kwargs) + except exc, last_exc: + continue + except: + raise + raise last_exc + +def _wrap_with_retry_thrice(func, exc): + def wrapped(*args, **kwargs): + return _retry_thrice(func, exc, *args, **kwargs) + return wrapped + +# Connecting to remote hosts is flaky. Make it more robust by retrying +# the connection several times. +_urlopen_with_retry = _wrap_with_retry_thrice(urllib2.urlopen, urllib2.URLError) + + +class AuthTests(unittest.TestCase): + """Tests urllib2 authentication features.""" + +## Disabled at the moment since there is no page under python.org which +## could be used to HTTP authentication. +# +# def test_basic_auth(self): +# import httplib +# +# test_url = "http://www.python.org/test/test_urllib2/basic_auth" +# test_hostport = "www.python.org" +# test_realm = 'Test Realm' +# test_user = 'test.test_urllib2net' +# test_password = 'blah' +# +# # failure +# try: +# _urlopen_with_retry(test_url) +# except urllib2.HTTPError, exc: +# self.assertEqual(exc.code, 401) +# else: +# self.fail("urlopen() should have failed with 401") +# +# # success +# auth_handler = urllib2.HTTPBasicAuthHandler() +# auth_handler.add_password(test_realm, test_hostport, +# test_user, test_password) +# opener = urllib2.build_opener(auth_handler) +# f = opener.open('http://localhost/') +# response = _urlopen_with_retry("http://www.python.org/") +# +# # The 'userinfo' URL component is deprecated by RFC 3986 for security +# # reasons, let's not implement it! (it's already implemented for proxy +# # specification strings (that is, URLs or authorities specifying a +# # proxy), so we must keep that) +# self.assertRaises(httplib.InvalidURL, +# urllib2.urlopen, "http://evil:thing@example.com") + + +class CloseSocketTest(unittest.TestCase): + + def test_close(self): + import httplib + + # calling .close() on urllib2's response objects should close the + # underlying socket + + # delve deep into response to fetch socket._socketobject + response = _urlopen_with_retry("http://www.example.com/") + abused_fileobject = response.fp + # self.assertIs(abused_fileobject.__class__, socket._fileobject) # gevent: disable + httpresponse = abused_fileobject._sock + self.assertIs(httpresponse.__class__, httplib.HTTPResponse) + fileobject = httpresponse.fp + # self.assertIs(fileobject.__class__, socket._fileobject) # gevent: disable + + self.assertTrue(not fileobject.closed) + response.close() + self.assertTrue(fileobject.closed) + +class OtherNetworkTests(unittest.TestCase): + def setUp(self): + if 0: # for debugging + import logging + logger = logging.getLogger("test_urllib2net") + logger.addHandler(logging.StreamHandler()) + + # XXX The rest of these tests aren't very good -- they don't check much. + # They do sometimes catch some major disasters, though. + + def test_ftp(self): + urls = [ + 'ftp://ftp.debian.org/debian/README', + ('ftp://ftp.debian.org/debian/non-existent-file', + None, urllib2.URLError), + ] + self._test_urls(urls, self._extra_handlers()) + + def test_file(self): + TESTFN = test_support.TESTFN + f = open(TESTFN, 'w') + try: + f.write('hi there\n') + f.close() + urls = [ + 'file:'+sanepathname2url(os.path.abspath(TESTFN)), + ('file:///nonsensename/etc/passwd', None, urllib2.URLError), + ] + self._test_urls(urls, self._extra_handlers(), retry=True) + finally: + os.remove(TESTFN) + + self.assertRaises(ValueError, urllib2.urlopen,'./relative_path/to/file') + + # XXX Following test depends on machine configurations that are internal + # to CNRI. Need to set up a public server with the right authentication + # configuration for test purposes. + +## def test_cnri(self): +## if socket.gethostname() == 'bitdiddle': +## localhost = 'bitdiddle.cnri.reston.va.us' +## elif socket.gethostname() == 'bitdiddle.concentric.net': +## localhost = 'localhost' +## else: +## localhost = None +## if localhost is not None: +## urls = [ +## 'file://%s/etc/passwd' % localhost, +## 'http://%s/simple/' % localhost, +## 'http://%s/digest/' % localhost, +## 'http://%s/not/found.h' % localhost, +## ] + +## bauth = HTTPBasicAuthHandler() +## bauth.add_password('basic_test_realm', localhost, 'jhylton', +## 'password') +## dauth = HTTPDigestAuthHandler() +## dauth.add_password('digest_test_realm', localhost, 'jhylton', +## 'password') + +## self._test_urls(urls, self._extra_handlers()+[bauth, dauth]) + + def test_urlwithfrag(self): + urlwith_frag = "http://www.pythontest.net/index.html#frag" + with test_support.transient_internet(urlwith_frag): + req = urllib2.Request(urlwith_frag) + res = urllib2.urlopen(req) + self.assertEqual(res.geturl(), + "http://www.pythontest.net/index.html#frag") + + def test_fileno(self): + req = urllib2.Request("http://www.example.com") + opener = urllib2.build_opener() + res = opener.open(req) + try: + res.fileno() + except AttributeError: + self.fail("HTTPResponse object should return a valid fileno") + finally: + res.close() + + def test_custom_headers(self): + url = "http://www.example.com" + with test_support.transient_internet(url): + opener = urllib2.build_opener() + request = urllib2.Request(url) + self.assertFalse(request.header_items()) + opener.open(request) + self.assertTrue(request.header_items()) + self.assertTrue(request.has_header('User-agent')) + request.add_header('User-Agent','Test-Agent') + opener.open(request) + self.assertEqual(request.get_header('User-agent'),'Test-Agent') + + def test_sites_no_connection_close(self): + # Some sites do not send Connection: close header. + # Verify that those work properly. (#issue12576) + + URL = 'http://www.imdb.com' # No Connection:close + with test_support.transient_internet(URL): + req = urllib2.urlopen(URL) + res = req.read() + self.assertTrue(res) + + def _test_urls(self, urls, handlers, retry=True): + import time + import logging + debug = logging.getLogger("test_urllib2").debug + + urlopen = urllib2.build_opener(*handlers).open + if retry: + urlopen = _wrap_with_retry_thrice(urlopen, urllib2.URLError) + + for url in urls: + if isinstance(url, tuple): + url, req, expected_err = url + else: + req = expected_err = None + with test_support.transient_internet(url): + debug(url) + try: + f = urlopen(url, req, TIMEOUT) + except EnvironmentError as err: + debug(err) + if expected_err: + msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % + (expected_err, url, req, type(err), err)) + self.assertIsInstance(err, expected_err, msg) + except urllib2.URLError as err: + if isinstance(err[0], socket.timeout): + print >>sys.stderr, "" % url + continue + else: + raise + else: + try: + with test_support.transient_internet(url): + buf = f.read() + debug("read %d bytes" % len(buf)) + except socket.timeout: + print >>sys.stderr, "" % url + f.close() + debug("******** next url coming up...") + time.sleep(0.1) + + def _extra_handlers(self): + handlers = [] + + cfh = urllib2.CacheFTPHandler() + self.addCleanup(cfh.clear_cache) + cfh.setTimeout(1) + handlers.append(cfh) + + return handlers + + +class TimeoutTest(unittest.TestCase): + def test_http_basic(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with test_support.transient_internet(url, timeout=None): + u = _urlopen_with_retry(url) + self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) + u.close() + + def test_http_default_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with test_support.transient_internet(url): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(url) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60) + u.close() + + def test_http_no_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with test_support.transient_internet(url): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(url, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(u.fp._sock.fp._sock.gettimeout()) + u.close() + + def test_http_timeout(self): + url = "http://www.example.com" + with test_support.transient_internet(url): + u = _urlopen_with_retry(url, timeout=120) + self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) + u.close() + + FTP_HOST = 'ftp://ftp.debian.org/debian/' + + def test_ftp_basic(self): + self.assertIsNone(socket.getdefaulttimeout()) + with test_support.transient_internet(self.FTP_HOST, timeout=None): + u = _urlopen_with_retry(self.FTP_HOST) + self.assertIsNone(u.fp.fp._sock.gettimeout()) + u.close() + + def test_ftp_default_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + with test_support.transient_internet(self.FTP_HOST): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(self.FTP_HOST) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + u.close() + + def test_ftp_no_timeout(self): + self.assertIsNone(socket.getdefaulttimeout(),) + with test_support.transient_internet(self.FTP_HOST): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(self.FTP_HOST, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(u.fp.fp._sock.gettimeout()) + u.close() + + def test_ftp_timeout(self): + with test_support.transient_internet(self.FTP_HOST): + try: + u = _urlopen_with_retry(self.FTP_HOST, timeout=60) + except: + raise + self.assertEqual(u.fp.fp._sock.gettimeout(), 60) + u.close() + + +def test_main(): + test_support.requires("network") + test_support.run_unittest(AuthTests, + OtherNetworkTests, + CloseSocketTest, + TimeoutTest, + ) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/test_wsgiref.py b/src/greentest/2.7pypy/test_wsgiref.py new file mode 100644 index 0000000..2469f67 --- /dev/null +++ b/src/greentest/2.7pypy/test_wsgiref.py @@ -0,0 +1,571 @@ +from unittest import TestCase +from wsgiref.util import setup_testing_defaults +from wsgiref.headers import Headers +from wsgiref.handlers import BaseHandler, BaseCGIHandler +from wsgiref import util +from wsgiref.validate import validator +from wsgiref.simple_server import WSGIServer, WSGIRequestHandler +from wsgiref.simple_server import make_server +from StringIO import StringIO +from SocketServer import BaseServer + +import os +import re +import sys + +from test import test_support + +class MockServer(WSGIServer): + """Non-socket HTTP server""" + + def __init__(self, server_address, RequestHandlerClass): + BaseServer.__init__(self, server_address, RequestHandlerClass) + self.server_bind() + + def server_bind(self): + host, port = self.server_address + self.server_name = host + self.server_port = port + self.setup_environ() + + +class MockHandler(WSGIRequestHandler): + """Non-socket HTTP handler""" + def setup(self): + self.connection = self.request + self.rfile, self.wfile = self.connection + + def finish(self): + pass + + +def hello_app(environ,start_response): + start_response("200 OK", [ + ('Content-Type','text/plain'), + ('Date','Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return ["Hello, world!"] + +def run_amock(app=hello_app, data="GET / HTTP/1.0\n\n"): + server = make_server("", 80, app, MockServer, MockHandler) + inp, out, err, olderr = StringIO(data), StringIO(), StringIO(), sys.stderr + sys.stderr = err + + try: + server.finish_request((inp,out), ("127.0.0.1",8888)) + finally: + sys.stderr = olderr + + return out.getvalue(), err.getvalue() + + +def compare_generic_iter(make_it,match): + """Utility to compare a generic 2.1/2.2+ iterator with an iterable + + If running under Python 2.2+, this tests the iterator using iter()/next(), + as well as __getitem__. 'make_it' must be a function returning a fresh + iterator to be tested (since this may test the iterator twice).""" + + it = make_it() + n = 0 + for item in match: + if not it[n]==item: raise AssertionError + n+=1 + try: + it[n] + except IndexError: + pass + else: + raise AssertionError("Too many items from __getitem__",it) + + try: + iter, StopIteration + except NameError: + pass + else: + # Only test iter mode under 2.2+ + it = make_it() + if not iter(it) is it: raise AssertionError + for item in match: + if not it.next()==item: raise AssertionError + try: + it.next() + except StopIteration: + pass + else: + raise AssertionError("Too many items from .next()",it) + + +class IntegrationTests(TestCase): + + def check_hello(self, out, has_length=True): + self.assertEqual(out, + "HTTP/1.0 200 OK\r\n" + "Server: WSGIServer/0.1 Python/"+sys.version.split()[0]+"\r\n" + "Content-Type: text/plain\r\n" + "Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + + (has_length and "Content-Length: 13\r\n" or "") + + "\r\n" + "Hello, world!" + ) + + def test_plain_hello(self): + out, err = run_amock() + self.check_hello(out) + + def test_request_length(self): + out, err = run_amock(data="GET " + ("x" * 65537) + " HTTP/1.0\n\n") + self.assertEqual(out.splitlines()[0], + "HTTP/1.0 414 Request-URI Too Long") + + def test_validated_hello(self): + out, err = run_amock(validator(hello_app)) + # the middleware doesn't support len(), so content-length isn't there + self.check_hello(out, has_length=False) + + def test_simple_validation_error(self): + def bad_app(environ,start_response): + start_response("200 OK", ('Content-Type','text/plain')) + return ["Hello, world!"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + "A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], + "AssertionError: Headers (('Content-Type', 'text/plain')) must" + " be of type list: " + ) + + +class UtilityTests(TestCase): + + def checkShift(self,sn_in,pi_in,part,sn_out,pi_out): + env = {'SCRIPT_NAME':sn_in,'PATH_INFO':pi_in} + util.setup_testing_defaults(env) + self.assertEqual(util.shift_path_info(env),part) + self.assertEqual(env['PATH_INFO'],pi_out) + self.assertEqual(env['SCRIPT_NAME'],sn_out) + return env + + def checkDefault(self, key, value, alt=None): + # Check defaulting when empty + env = {} + util.setup_testing_defaults(env) + if isinstance(value, StringIO): + self.assertIsInstance(env[key], StringIO) + else: + self.assertEqual(env[key], value) + + # Check existing value + env = {key:alt} + util.setup_testing_defaults(env) + self.assertIs(env[key], alt) + + def checkCrossDefault(self,key,value,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(kw[key],value) + + def checkAppURI(self,uri,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.application_uri(kw),uri) + + def checkReqURI(self,uri,query=1,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.request_uri(kw,query),uri) + + def checkFW(self,text,size,match): + + def make_it(text=text,size=size): + return util.FileWrapper(StringIO(text),size) + + compare_generic_iter(make_it,match) + + it = make_it() + self.assertFalse(it.filelike.closed) + + for item in it: + pass + + self.assertFalse(it.filelike.closed) + + it.close() + self.assertTrue(it.filelike.closed) + + def testSimpleShifts(self): + self.checkShift('','/', '', '/', '') + self.checkShift('','/x', 'x', '/x', '') + self.checkShift('/','', None, '/', '') + self.checkShift('/a','/x/y', 'x', '/a/x', '/y') + self.checkShift('/a','/x/', 'x', '/a/x', '/') + + def testNormalizedShifts(self): + self.checkShift('/a/b', '/../y', '..', '/a', '/y') + self.checkShift('', '/../y', '..', '', '/y') + self.checkShift('/a/b', '//y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '//y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '/./y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '/./y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '///./..//y/.//', '..', '/a', '/y/') + self.checkShift('/a/b', '///', '', '/a/b/', '') + self.checkShift('/a/b', '/.//', '', '/a/b/', '') + self.checkShift('/a/b', '/x//', 'x', '/a/b/x', '/') + self.checkShift('/a/b', '/.', None, '/a/b', '') + + def testDefaults(self): + for key, value in [ + ('SERVER_NAME','127.0.0.1'), + ('SERVER_PORT', '80'), + ('SERVER_PROTOCOL','HTTP/1.0'), + ('HTTP_HOST','127.0.0.1'), + ('REQUEST_METHOD','GET'), + ('SCRIPT_NAME',''), + ('PATH_INFO','/'), + ('wsgi.version', (1,0)), + ('wsgi.run_once', 0), + ('wsgi.multithread', 0), + ('wsgi.multiprocess', 0), + ('wsgi.input', StringIO("")), + ('wsgi.errors', StringIO()), + ('wsgi.url_scheme','http'), + ]: + self.checkDefault(key,value) + + def testCrossDefaults(self): + self.checkCrossDefault('HTTP_HOST',"foo.bar",SERVER_NAME="foo.bar") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="on") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="1") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="yes") + self.checkCrossDefault('wsgi.url_scheme',"http",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"80",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"443",HTTPS="on") + + def testGuessScheme(self): + self.assertEqual(util.guess_scheme({}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"foo"}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"on"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"yes"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"1"}), "https") + + def testAppURIs(self): + self.checkAppURI("http://127.0.0.1/") + self.checkAppURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkAppURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkAppURI("http://spam.example.com:2071/", + HTTP_HOST="spam.example.com:2071", SERVER_PORT="2071") + self.checkAppURI("http://spam.example.com/", + SERVER_NAME="spam.example.com") + self.checkAppURI("http://127.0.0.1/", + HTTP_HOST="127.0.0.1", SERVER_NAME="spam.example.com") + self.checkAppURI("https://127.0.0.1/", HTTPS="on") + self.checkAppURI("http://127.0.0.1:8000/", SERVER_PORT="8000", + HTTP_HOST=None) + + def testReqURIs(self): + self.checkReqURI("http://127.0.0.1/") + self.checkReqURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkReqURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam", + SCRIPT_NAME="/spammity", PATH_INFO="/spam") + self.checkReqURI("http://127.0.0.1/spammity/sp%E4m", + SCRIPT_NAME="/spammity", PATH_INFO="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam;ham", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;ham") + self.checkReqURI("http://127.0.0.1/spammity/spam;cookie=1234,5678", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;cookie=1234,5678") + self.checkReqURI("http://127.0.0.1/spammity/spam?say=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam?s%E4y=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="s%E4y=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam", 0, + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + + def testFileWrapper(self): + self.checkFW("xyz"*50, 120, ["xyz"*40,"xyz"*10]) + + def testHopByHop(self): + for hop in ( + "Connection Keep-Alive Proxy-Authenticate Proxy-Authorization " + "TE Trailers Transfer-Encoding Upgrade" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertTrue(util.is_hop_by_hop(alt)) + + # Not comprehensive, just a few random header names + for hop in ( + "Accept Cache-Control Date Pragma Trailer Via Warning" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertFalse(util.is_hop_by_hop(alt)) + +class HeaderTests(TestCase): + + def testMappingInterface(self): + test = [('x','y')] + self.assertEqual(len(Headers([])),0) + self.assertEqual(len(Headers(test[:])),1) + self.assertEqual(Headers(test[:]).keys(), ['x']) + self.assertEqual(Headers(test[:]).values(), ['y']) + self.assertEqual(Headers(test[:]).items(), test) + self.assertIsNot(Headers(test).items(), test) # must be copy! + + h=Headers([]) + del h['foo'] # should not raise an error + + h['Foo'] = 'bar' + for m in h.has_key, h.__contains__, h.get, h.get_all, h.__getitem__: + self.assertTrue(m('foo')) + self.assertTrue(m('Foo')) + self.assertTrue(m('FOO')) + self.assertFalse(m('bar')) + + self.assertEqual(h['foo'],'bar') + h['foo'] = 'baz' + self.assertEqual(h['FOO'],'baz') + self.assertEqual(h.get_all('foo'),['baz']) + + self.assertEqual(h.get("foo","whee"), "baz") + self.assertEqual(h.get("zoo","whee"), "whee") + self.assertEqual(h.setdefault("foo","whee"), "baz") + self.assertEqual(h.setdefault("zoo","whee"), "whee") + self.assertEqual(h["foo"],"baz") + self.assertEqual(h["zoo"],"whee") + + def testRequireList(self): + self.assertRaises(TypeError, Headers, "foo") + + + def testExtras(self): + h = Headers([]) + self.assertEqual(str(h),'\r\n') + + h.add_header('foo','bar',baz="spam") + self.assertEqual(h['foo'], 'bar; baz="spam"') + self.assertEqual(str(h),'foo: bar; baz="spam"\r\n\r\n') + + h.add_header('Foo','bar',cheese=None) + self.assertEqual(h.get_all('foo'), + ['bar; baz="spam"', 'bar; cheese']) + + self.assertEqual(str(h), + 'foo: bar; baz="spam"\r\n' + 'Foo: bar; cheese\r\n' + '\r\n' + ) + + +class ErrorHandler(BaseCGIHandler): + """Simple handler subclass for testing BaseHandler""" + + # BaseHandler records the OS environment at import time, but envvars + # might have been changed later by other tests, which trips up + # HandlerTests.testEnviron(). + os_environ = dict(os.environ.items()) + + def __init__(self,**kw): + setup_testing_defaults(kw) + BaseCGIHandler.__init__( + self, StringIO(''), StringIO(), StringIO(), kw, + multithread=True, multiprocess=True + ) + +class TestHandler(ErrorHandler): + """Simple handler subclass for testing BaseHandler, w/error passthru""" + + def handle_error(self): + raise # for testing, we want to see what's happening + + +class HandlerTests(TestCase): + + def checkEnvironAttrs(self, handler): + env = handler.environ + for attr in [ + 'version','multithread','multiprocess','run_once','file_wrapper' + ]: + if attr=='file_wrapper' and handler.wsgi_file_wrapper is None: + continue + self.assertEqual(getattr(handler,'wsgi_'+attr),env['wsgi.'+attr]) + + def checkOSEnviron(self,handler): + empty = {}; setup_testing_defaults(empty) + env = handler.environ + from os import environ + for k,v in environ.items(): + if k not in empty: + self.assertEqual(env[k],v) + for k,v in empty.items(): + self.assertIn(k, env) + + def testEnviron(self): + h = TestHandler(X="Y") + h.setup_environ() + self.checkEnvironAttrs(h) + self.checkOSEnviron(h) + self.assertEqual(h.environ["X"],"Y") + + def testCGIEnviron(self): + h = BaseCGIHandler(None,None,None,{}) + h.setup_environ() + for key in 'wsgi.url_scheme', 'wsgi.input', 'wsgi.errors': + self.assertIn(key, h.environ) + + def testScheme(self): + h=TestHandler(HTTPS="on"); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'https') + h=TestHandler(); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'http') + + def testAbstractMethods(self): + h = BaseHandler() + for name in [ + '_flush','get_stdin','get_stderr','add_cgi_vars' + ]: + self.assertRaises(NotImplementedError, getattr(h,name)) + self.assertRaises(NotImplementedError, h._write, "test") + + def testContentLength(self): + # Demo one reason iteration is better than write()... ;) + + def trivial_app1(e,s): + s('200 OK',[]) + return [e['wsgi.url_scheme']] + + def trivial_app2(e,s): + s('200 OK',[])(e['wsgi.url_scheme']) + return [] + + def trivial_app4(e,s): + # Simulate a response to a HEAD request + s('200 OK',[('Content-Length', '12345')]) + return [] + + h = TestHandler() + h.run(trivial_app1) + self.assertEqual(h.stdout.getvalue(), + "Status: 200 OK\r\n" + "Content-Length: 4\r\n" + "\r\n" + "http") + + h = TestHandler() + h.run(trivial_app2) + self.assertEqual(h.stdout.getvalue(), + "Status: 200 OK\r\n" + "\r\n" + "http") + + + h = TestHandler() + h.run(trivial_app4) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 12345\r\n' + b'\r\n') + + def testBasicErrorOutput(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + def error_app(e,s): + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(non_error_app) + self.assertEqual(h.stdout.getvalue(), + "Status: 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n") + self.assertEqual(h.stderr.getvalue(),"") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + "Status: %s\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: %d\r\n" + "\r\n%s" % (h.error_status,len(h.error_body),h.error_body)) + + self.assertNotEqual(h.stderr.getvalue().find("AssertionError"), -1) + + def testErrorAfterOutput(self): + MSG = "Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + "Status: 200 OK\r\n" + "\r\n"+MSG) + self.assertNotEqual(h.stderr.getvalue().find("AssertionError"), -1) + + def testHeaderFormats(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + stdpat = ( + r"HTTP/%s 200 OK\r\n" + r"Date: \w{3}, [ 0123]\d \w{3} \d{4} \d\d:\d\d:\d\d GMT\r\n" + r"%s" r"Content-Length: 0\r\n" r"\r\n" + ) + shortpat = ( + "Status: 200 OK\r\n" "Content-Length: 0\r\n" "\r\n" + ) + + for ssw in "FooBar/1.0", None: + sw = ssw and "Server: %s\r\n" % ssw or "" + + for version in "1.0", "1.1": + for proto in "HTTP/0.9", "HTTP/1.0", "HTTP/1.1": + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = False + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + self.assertEqual(shortpat,h.stdout.getvalue()) + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = True + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + if proto=="HTTP/0.9": + self.assertEqual(h.stdout.getvalue(),"") + else: + self.assertTrue( + re.match(stdpat%(version,sw), h.stdout.getvalue()), + (stdpat%(version,sw), h.stdout.getvalue()) + ) + + def testCloseOnError(self): + side_effects = {'close_called': False} + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + class CrashyIterable(object): + def __iter__(self): + while True: + yield b'blah' + raise AssertionError("This should be caught by handler") + + def close(self): + side_effects['close_called'] = True + return CrashyIterable() + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(side_effects['close_called'], True) + + +def test_main(): + test_support.run_unittest(__name__) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/2.7pypy/version b/src/greentest/2.7pypy/version new file mode 100644 index 0000000..ecc17b8 --- /dev/null +++ b/src/greentest/2.7pypy/version @@ -0,0 +1 @@ +2.7.13 diff --git a/src/greentest/2.7pypy/wrongcert.pem b/src/greentest/2.7pypy/wrongcert.pem new file mode 100644 index 0000000..5f92f9b --- /dev/null +++ b/src/greentest/2.7pypy/wrongcert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH +FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T +f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB +AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq +1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW +7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg +SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe +19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg +ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666 +lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs +ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv +frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk +2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6 ++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK +24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G +A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9 +OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ +fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E +usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+ +43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw +eSHj5jpC8iZKjCHBn+mAi4cQ514= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/badcert.pem b/src/greentest/3.4/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/greentest/3.4/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/badkey.pem b/src/greentest/3.4/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/greentest/3.4/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/capath/0e4015b9.0 b/src/greentest/3.4/capath/0e4015b9.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.4/capath/0e4015b9.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/capath/4e1295a3.0 b/src/greentest/3.4/capath/4e1295a3.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.4/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/capath/5ed36f99.0 b/src/greentest/3.4/capath/5ed36f99.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.4/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/capath/6e88d7b8.0 b/src/greentest/3.4/capath/6e88d7b8.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.4/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/capath/99d0fa06.0 b/src/greentest/3.4/capath/99d0fa06.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.4/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/capath/ce7b8643.0 b/src/greentest/3.4/capath/ce7b8643.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.4/capath/ce7b8643.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/dh1024.pem b/src/greentest/3.4/dh1024.pem new file mode 100644 index 0000000..a391176 --- /dev/null +++ b/src/greentest/3.4/dh1024.pem @@ -0,0 +1,7 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAIbzw1s9CT8SV5yv6L7esdAdZYZjPi3qWFs61CYTFFQnf2s/d09NYaJt +rrvJhIzWavqnue71qXCf83/J3nz3FEwUU/L0mGyheVbsSHiI64wUo3u50wK5Igo0 +RNs/LD0irs7m0icZ//hijafTU+JOBiuA8zMI+oZfU7BGuc9XrUprAgEC +-----END DH PARAMETERS----- + +Generated with: openssl dhparam -out dh1024.pem 1024 diff --git a/src/greentest/3.4/https_svn_python_org_root.pem b/src/greentest/3.4/https_svn_python_org_root.pem new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.4/https_svn_python_org_root.pem @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/keycert.passwd.pem b/src/greentest/3.4/keycert.passwd.pem new file mode 100644 index 0000000..e905748 --- /dev/null +++ b/src/greentest/3.4/keycert.passwd.pem @@ -0,0 +1,33 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/keycert.pem b/src/greentest/3.4/keycert.pem new file mode 100644 index 0000000..64318aa --- /dev/null +++ b/src/greentest/3.4/keycert.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/keycert2.pem b/src/greentest/3.4/keycert2.pem new file mode 100644 index 0000000..e8a9e08 --- /dev/null +++ b/src/greentest/3.4/keycert2.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAJnsJZVrppL+W5I9 +zGQrrawWwE5QJpBK9nWw17mXrZ03R1cD9BamLGivVISbPlRlAVnZBEyh1ATpsB7d +CUQ+WHEvALquvx4+Yw5l+fXeiYRjrLRBYZuVy8yNtXzU3iWcGObcYRkUdiXdOyP7 +sLF2YZHRvQZpzgDBKkrraeQ81w21AgMBAAECgYBEm7n07FMHWlE+0kT0sXNsLYfy +YE+QKZnJw9WkaDN+zFEEPELkhZVt5BjsMraJr6v2fIEqF0gGGJPkbenffVq2B5dC +lWUOxvJHufMK4sM3Cp6s/gOp3LP+QkzVnvJSfAyZU6l+4PGX5pLdUsXYjPxgzjzL +S36tF7/2Uv1WePyLUQJBAMsPhYzUXOPRgmbhcJiqi9A9c3GO8kvSDYTCKt3VMnqz +HBn6MQ4VQasCD1F+7jWTI0FU/3vdw8non/Fj8hhYqZcCQQDCDRdvmZqDiZnpMqDq +L6ZSrLTVtMvZXZbgwForaAD9uHj51TME7+eYT7EG2YCgJTXJ4YvRJEnPNyskwdKt +vTSTAkEAtaaN/vyemEJ82BIGStwONNw0ILsSr5cZ9tBHzqiA/tipY+e36HRFiXhP +QcU9zXlxyWkDH8iz9DSAmE2jbfoqwwJANlMJ65E543cjIlitGcKLMnvtCCLcKpb7 +xSG0XJB6Lo11OKPJ66jp0gcFTSCY1Lx2CXVd+gfJrfwI1Pp562+bhwJBAJ9IfDPU +R8OpO9v1SGd8x33Owm7uXOpB9d63/T70AD1QOXjKUC4eXYbt0WWfWuny/RNPRuyh +w7DXSfUF+kPKolU= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICXTCCAcagAwIBAgIJAIO3upAG445fMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTAeFw0x +MDEwMDkxNTAxMDBaFw0yMDEwMDYxNTAxMDBaMGIxCzAJBgNVBAYTAlhZMRcwFQYD +VQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZv +dW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEAmewllWumkv5bkj3MZCutrBbATlAmkEr2dbDXuZetnTdHVwP0 +FqYsaK9UhJs+VGUBWdkETKHUBOmwHt0JRD5YcS8Auq6/Hj5jDmX59d6JhGOstEFh +m5XLzI21fNTeJZwY5txhGRR2Jd07I/uwsXZhkdG9BmnOAMEqSutp5DzXDbUCAwEA +AaMbMBkwFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBBQUAA4GB +AH+iMClLLGSaKWgwXsmdVo4FhTZZHo8Uprrtg3N9FxEeE50btpDVQysgRt5ias3K +m+bME9zbKwvbVWD5zZdjus4pDgzwF/iHyccL8JyYhxOvS/9zmvAtFXj/APIIbZFp +IT75d9f88ScIGEtknZQejnrdhB64tYki/EqluiuKBqKD +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/keycert3.pem b/src/greentest/3.4/keycert3.pem new file mode 100644 index 0000000..5bfa62c --- /dev/null +++ b/src/greentest/3.4/keycert3.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP +jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM +9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ +aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe +yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j +y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ +AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW +5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL +9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 +1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT +DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh +1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m +JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 +RnJdHOMXWem7/w== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: + 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: + c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: + 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: + f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: + 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: + f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: + af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: + 21:82:a5:3c:88:e5:be:1b:b1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: + e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: + f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: + e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: + d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: + 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: + ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: + 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: + 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: + 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: + 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: + f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: + 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: + a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: + fc:a9:94:71 +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C +tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola +N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 +TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR +iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG +xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo +5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv +mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF +YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh +2EJ36/yplHE= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/keycert4.pem b/src/greentest/3.4/keycert4.pem new file mode 100644 index 0000000..53355c8 --- /dev/null +++ b/src/greentest/3.4/keycert4.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv +L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2 +NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1 +L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L +pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de +R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9 +myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT +drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS +Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx +i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK +Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu +JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN ++/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/ +e83Gq6ffLVfKNQ== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443282 (0xb09264b1f2da21d2) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14: + 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9: + cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a: + b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76: + 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7: + 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a: + d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79: + 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd: + 81:7e:bd:1b:ae:0b:5d:c6:39 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0: + 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f: + 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56: + 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15: + 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1: + 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb: + 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e: + e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18: + d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1: + af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53: + 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68: + 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da: + 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92: + 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa: + 49:12:1e:ce +-----BEGIN CERTIFICATE----- +MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z +dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU +aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0 +ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ +hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v +xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338 +Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP +XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0 +UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz +aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb +oF+6ufu6+kkSHs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/nokia.pem b/src/greentest/3.4/nokia.pem new file mode 100644 index 0000000..0d044df --- /dev/null +++ b/src/greentest/3.4/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/nullbytecert.pem b/src/greentest/3.4/nullbytecert.pem new file mode 100644 index 0000000..447186c --- /dev/null +++ b/src/greentest/3.4/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/nullcert.pem b/src/greentest/3.4/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/greentest/3.4/pycacert.pem b/src/greentest/3.4/pycacert.pem new file mode 100644 index 0000000..09b1f3e --- /dev/null +++ b/src/greentest/3.4/pycacert.pem @@ -0,0 +1,78 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Jan 2 19:47:07 2023 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: + 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: + e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: + e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: + 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: + 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: + a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: + e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: + 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: + 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: + e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: + c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: + cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: + 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: + 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: + 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: + e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: + c5:4d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + X509v3 Authority Key Identifier: + keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: + 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: + a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: + 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: + 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: + 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: + fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: + 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: + 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: + 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: + ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: + 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: + b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: + 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: + 5e:58:c8:9e +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/pycakey.pem b/src/greentest/3.4/pycakey.pem new file mode 100644 index 0000000..fc6effe --- /dev/null +++ b/src/greentest/3.4/pycakey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDn3unjDJ8AtqH9 +K1uW0m/M4L6GuSBe7AN6VavqpOn5SYXSZtXtx3rqVo4tj+dC4mIoqZ/WG47rtbSc +nxSr3+aUi3YdPm0kYe0MvwCKYQzfXMg2cxYAzUe6baSkdIiDIwoZ/AmnPEpL0+cd +LeTqTFQh8ybbiTcY1AK7QDJfpP8tHPfUu+yOz1yCrOZ8CGxIhWEHfyXgXOC8NF/g +uQRHdchHC4281shoXzODYtIgRDWxrYEais28NbBci0fWGOmcGJfMATwpzOge5OTB +uN7nwhEYh1qTNNimJfcUcevkIaLSDy4u1GIANdPW71xgS0ypFOLdFVhGNzMmt+cu +Xe1C5MVNAgMBAAECggEBAJPM7QuUrPn4cLN/Ysd15lwTWn9oHDFFgkYFvCs66gXE +ju/6Kx2BjWE4wTJby09AHM/MqB0DvguT7Mf1Q2j3tPQ1HZowg8OwRDleuwp6KIls +jBbhL0Jdl/5HC67ktWvZ9wNvO/wFG1rQfT6FVajf9LUbWEaSZbOG2SLhHfsHorzu +xjTJaI3bQ/0+79B1exwk5ruwhzFRd/XpY8hls7D/RfPIuHDlBghkW3N59KFWrf5h +6bNEh2THm0+IyGcGqs0FD+QCOXyvsjwSUswqrr2ctLREOeDcd5ReUjSxYgjcJRrm +J7ceIY/+uwDJxw/OlnmBvF6pQMkKwYW2gFztu+g2t4UCgYEA/9yo01Exz4crxXsy +tAlnDJM++nZcm07rtFjTKHUfKY/cCgNTa8udM0svnfwlid/dpgLsI38gx04HHC1i +EZ4acz+ToIWedLxM0nq73//xeRWEazOvCz1mMTZaMldahTWAyzN8qVK2B/625Yy4 +wNYWyweBBwEB8MzaCs73spksXOsCgYEA5/7wvhiofYGFAfMuANeJIwDL2OtBnoOv +mVNfCmi3GC38fzwyi5ZpskWDiS2woJ+LQfs9Qu4EcZbUFLd7gbeOvb5gmFUtYope +LitUUKunIR18MkQ+mQDBpQPQPhk4QJP5reCbWkrfTu7b5o/iS41s6fBTFmuzhLcT +C71vFdCyeKcCgYAiCCqYeOtELDmBOeLDmaCQRqGQ1N96dOPbCBmF/xYXBCCDYG/f +HaUaJnz96YTgstsbcrYP/p/Qgqtlbw/lQf9IpwMuzbcG1ejt8g89OyDWNyt2ytgU +iaUnFJCos3/Byh0Iah/BsdOueo2/OJl2ZMOBW80orlSgv86cs2y037TL4wKBgQDm +OOyW+MlbowhnIvfoBfwlLEkefnej4nKD6WRLZBcue5Qyf355X06Mhsc9foXlH+6G +D9h/bswiHNdhp6N82rdgPGiHQx/CxiUoE/+b/nvgNO5mw6qLE2EXbG1e8pAMJcyE +bHw+YkawggDfELI036fRj5gki8SeUz8nS1nNgElbyQKBgCRDX9Jh+MwSLu4QBWdt +/fi+lv3K6kun/fI7EOV1vCV/j871tICu7pu5BrOLxAHqoVfU9AUX299/2KjCb5pv +kjogiUK6qWCWBlfuqDNWGCoUGt1rhznUva0nNjSMy5rinBhhjpROZC2pw48lOluP +UuvXsaPph7GTqPuy4Kab12YC +-----END PRIVATE KEY----- diff --git a/src/greentest/3.4/revocation.crl b/src/greentest/3.4/revocation.crl new file mode 100644 index 0000000..6d89b08 --- /dev/null +++ b/src/greentest/3.4/revocation.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTEzMTEyMTE3MDg0N1oXDTIzMDkzMDE3MDg0N1qgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQCNJXC2mVKauEeN3LlQ3ZtM5gkH3ExH ++i4bmJjtJn497WwvvoIeUdrmVXgJQR93RtV37hZwN0SXMLlNmUZPH4rHhihayw4m +unCzVj/OhCCY7/TPjKuJ1O/0XhaLBpBVjQN7R/1ujoRKbSia/CD3vcn7Fqxzw7LK +fSRCKRGTj1CZiuxrphtFchwALXSiFDy9mr2ZKhImcyq1PydfgEzU78APpOkMQsIC +UNJ/cf3c9emzf+dUtcMEcejQ3mynBo4eIGg1EW42bz4q4hSjzQlKcBV0muw5qXhc +HOxH2iTFhQ7SrvVuK/dM14rYM4B5mSX3nRC1kNmXpS9j3wJDhuwmjHed +-----END X509 CRL----- diff --git a/src/greentest/3.4/selfsigned_pythontestdotnet.pem b/src/greentest/3.4/selfsigned_pythontestdotnet.pem new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.4/selfsigned_pythontestdotnet.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/sha256.pem b/src/greentest/3.4/sha256.pem new file mode 100644 index 0000000..d3db4b8 --- /dev/null +++ b/src/greentest/3.4/sha256.pem @@ -0,0 +1,128 @@ +# Certificate chain for https://sha256.tbs-internet.com + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC +-----BEGIN CERTIFICATE----- +MIIGXDCCBUSgAwIBAgIRAKpVmHgg9nfCodAVwcP4siwwDQYJKoZIhvcNAQELBQAw +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMB4XDTEyMDEwNDAwMDAwMFoXDTE0MDIxNzIzNTk1OVowgcsxCzAJBgNV +BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV +BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM +VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS +c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKQIX/zdJcyxty0m +PM1XQSoSSifueS3AVcgqMsaIKS/u+rYzsv4hQ/qA6vLn5m5/ewUcZDj7zdi6rBVf +PaVNXJ6YinLX0tkaW8TEjeVuZG5yksGZlhCt1CJ1Ho9XLiLaP4uJ7MCoNUntpJ+E +LfrOdgsIj91kPmwjDJeztVcQCvKzhjVJA/KxdInc0JvOATn7rpaSmQI5bvIjufgo +qVsTPwVFzuUYULXBk7KxRT7MiEqnd5HvviNh0285QC478zl3v0I0Fb5El4yD3p49 +IthcRnxzMKc0UhU5ogi0SbONyBfm/mzONVfSxpM+MlyvZmJqrbuuLoEDzJD+t8PU +xSuzgbcCAwEAAaOCAj4wggI6MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMB0GA1UdDgQWBBT/qTGYdaj+f61c2IRFL/B1eEsM8DAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG +CisGAQQBgjcKAwMGCWCGSAGG+EIEATBLBgNVHSAERDBCMEAGCisGAQQB5TcCBAEw +MjAwBggrBgEFBQcCARYkaHR0cHM6Ly93d3cudGJzLWludGVybmV0LmNvbS9DQS9D +UFM0MG0GA1UdHwRmMGQwMqAwoC6GLGh0dHA6Ly9jcmwudGJzLWludGVybmV0LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMC6gLKAqhihodHRwOi8vY3JsLnRicy14NTA5LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMIGmBggrBgEFBQcBAQSBmTCBljA4BggrBgEFBQcw +AoYsaHR0cDovL2NydC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQVNHQy5jcnQw +NAYIKwYBBQUHMAKGKGh0dHA6Ly9jcnQudGJzLXg1MDkuY29tL1RCU1g1MDlDQVNH +Qy5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLnRicy14NTA5LmNvbTA/BgNV +HREEODA2ghdzaGEyNTYudGJzLWludGVybmV0LmNvbYIbd3d3LnNoYTI1Ni50YnMt +aW50ZXJuZXQuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQA0pOuL8QvAa5yksTbGShzX +ABApagunUGoEydv4YJT1MXy9tTp7DrWaozZSlsqBxrYAXP1d9r2fuKbEniYHxaQ0 +UYaf1VSIlDo1yuC8wE7wxbHDIpQ/E5KAyxiaJ8obtDhFstWAPAH+UoGXq0kj2teN +21sFQ5dXgA95nldvVFsFhrRUNB6xXAcaj0VZFhttI0ZfQZmQwEI/P+N9Jr40OGun +aa+Dn0TMeUH4U20YntfLbu2nDcJcYfyurm+8/0Tr4HznLnedXu9pCPYj0TaddrgT +XO0oFiyy7qGaY6+qKh71yD64Y3ycCJ/HR9Wm39mjZYc9ezYwT4noP6r7Lk8YO7/q +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6 +rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0 +9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ +ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk +owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G +Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk +9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ +MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3 +AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk +ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k +by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw +cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV +VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B +ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN +AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232 +euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY +1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98 +RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz +8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV +v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E= +-----END CERTIFICATE----- + 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT +AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0 +ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB +IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05 +4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6 +2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh +alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv +u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW +xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p +XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd +tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB +BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX +BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov +L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN +AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO +rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd +FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM ++bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI +3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb ++M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g= +-----END CERTIFICATE----- + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG +EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD +VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu +dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 +E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ +D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK +4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq +lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW +bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB +o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT +MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js +LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr +BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB +AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft +Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj +j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH +KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv +2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 +mfnGV/TJVTl4uix5yaaIK/QI +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/ssl_cert.pem b/src/greentest/3.4/ssl_cert.pem new file mode 100644 index 0000000..47a7d7e --- /dev/null +++ b/src/greentest/3.4/ssl_cert.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.4/ssl_key.passwd.pem b/src/greentest/3.4/ssl_key.passwd.pem new file mode 100644 index 0000000..2524672 --- /dev/null +++ b/src/greentest/3.4/ssl_key.passwd.pem @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- diff --git a/src/greentest/3.4/ssl_key.pem b/src/greentest/3.4/ssl_key.pem new file mode 100644 index 0000000..3fd3bbd --- /dev/null +++ b/src/greentest/3.4/ssl_key.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- diff --git a/src/greentest/3.4/test_httplib.py b/src/greentest/3.4/test_httplib.py new file mode 100644 index 0000000..df9a9e3 --- /dev/null +++ b/src/greentest/3.4/test_httplib.py @@ -0,0 +1,1234 @@ +import errno +from http import client +import io +import itertools +import os +import array +import socket + +import unittest +TestCase = unittest.TestCase + +from test import support + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') +# Self-signed cert file for self-signed.pythontest.net +CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') + +HOST = support.HOST + +class FakeSocket: + def __init__(self, text, fileclass=io.BytesIO, host=None, port=None): + if isinstance(text, str): + text = text.encode("ascii") + self.text = text + self.fileclass = fileclass + self.data = b'' + self.sendall_calls = 0 + self.file_closed = False + self.host = host + self.port = port + + def sendall(self, data): + self.sendall_calls += 1 + self.data += data + + def makefile(self, mode, bufsize=None): + if mode != 'r' and mode != 'rb': + raise client.UnimplementedFileMode() + # keep the file around so we can check how much was read from it + self.file = self.fileclass(self.text) + self.file.close = self.file_close #nerf close () + return self.file + + def file_close(self): + self.file_closed = True + + def close(self): + pass + +class EPipeSocket(FakeSocket): + + def __init__(self, text, pipe_trigger): + # When sendall() is called with pipe_trigger, raise EPIPE. + FakeSocket.__init__(self, text) + self.pipe_trigger = pipe_trigger + + def sendall(self, data): + if self.pipe_trigger in data: + raise OSError(errno.EPIPE, "gotcha") + self.data += data + + def close(self): + pass + +class NoEOFBytesIO(io.BytesIO): + """Like BytesIO, but raises AssertionError on EOF. + + This is used below to test that http.client doesn't try to read + more from the underlying file than it should. + """ + def read(self, n=-1): + data = io.BytesIO.read(self, n) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + + def readline(self, length=None): + data = io.BytesIO.readline(self, length) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + +class HeaderTests(TestCase): + def test_auto_headers(self): + # Some headers are added automatically, but should not be added by + # .request() if they are explicitly set. + + class HeaderCountingBuffer(list): + def __init__(self): + self.count = {} + def append(self, item): + kv = item.split(b':') + if len(kv) > 1: + # item is a 'Key: Value' header string + lcKey = kv[0].decode('ascii').lower() + self.count.setdefault(lcKey, 0) + self.count[lcKey] += 1 + list.append(self, item) + + for explicit_header in True, False: + for header in 'Content-length', 'Host', 'Accept-encoding': + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('blahblahblah') + conn._buffer = HeaderCountingBuffer() + + body = 'spamspamspam' + headers = {} + if explicit_header: + headers[header] = str(len(body)) + conn.request('POST', '/', body, headers) + self.assertEqual(conn._buffer.count[header.lower()], 1) + + def test_content_length_0(self): + + class ContentLengthChecker(list): + def __init__(self): + list.__init__(self) + self.content_length = None + def append(self, item): + kv = item.split(b':', 1) + if len(kv) > 1 and kv[0].lower() == b'content-length': + self.content_length = kv[1].strip() + list.append(self, item) + + # Here, we're testing that methods expecting a body get a + # content-length set to zero if the body is empty (either None or '') + bodies = (None, '') + methods_with_body = ('PUT', 'POST', 'PATCH') + for method, body in itertools.product(methods_with_body, bodies): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', body) + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # For these methods, we make sure that content-length is not set when + # the body is None because it might cause unexpected behaviour on the + # server. + methods_without_body = ( + 'GET', 'CONNECT', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', + ) + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', None) + self.assertEqual( + conn._buffer.content_length, None, + 'Header Content-Length set for empty body on {}'.format(method) + ) + + # If the body is set to '', that's considered to be "present but + # empty" rather than "missing", so content length would be set, even + # for methods that don't expect a body. + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', '') + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # If the body is set, make sure Content-Length is set. + for method in itertools.chain(methods_without_body, methods_with_body): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', ' ') + self.assertEqual( + conn._buffer.content_length, b'1', + 'Header Content-Length incorrect on {}'.format(method) + ) + + def test_putheader(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.putrequest('GET','/') + conn.putheader('Content-length', 42) + self.assertIn(b'Content-length: 42', conn._buffer) + + conn.putheader('Foo', ' bar ') + self.assertIn(b'Foo: bar ', conn._buffer) + conn.putheader('Bar', '\tbaz\t') + self.assertIn(b'Bar: \tbaz\t', conn._buffer) + conn.putheader('Authorization', 'Bearer mytoken') + self.assertIn(b'Authorization: Bearer mytoken', conn._buffer) + conn.putheader('IterHeader', 'IterA', 'IterB') + self.assertIn(b'IterHeader: IterA\r\n\tIterB', conn._buffer) + conn.putheader('LatinHeader', b'\xFF') + self.assertIn(b'LatinHeader: \xFF', conn._buffer) + conn.putheader('Utf8Header', b'\xc3\x80') + self.assertIn(b'Utf8Header: \xc3\x80', conn._buffer) + conn.putheader('C1-Control', b'next\x85line') + self.assertIn(b'C1-Control: next\x85line', conn._buffer) + conn.putheader('Embedded-Fold-Space', 'is\r\n allowed') + self.assertIn(b'Embedded-Fold-Space: is\r\n allowed', conn._buffer) + conn.putheader('Embedded-Fold-Tab', 'is\r\n\tallowed') + self.assertIn(b'Embedded-Fold-Tab: is\r\n\tallowed', conn._buffer) + conn.putheader('Key Space', 'value') + self.assertIn(b'Key Space: value', conn._buffer) + conn.putheader('KeySpace ', 'value') + self.assertIn(b'KeySpace : value', conn._buffer) + conn.putheader(b'Nonbreak\xa0Space', 'value') + self.assertIn(b'Nonbreak\xa0Space: value', conn._buffer) + conn.putheader(b'\xa0NonbreakSpace', 'value') + self.assertIn(b'\xa0NonbreakSpace: value', conn._buffer) + + def test_ipv6host_header(self): + # Default host header on IPv6 transaction should wrapped by [] if + # its actual IPv6 address + expected = b'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001::]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + expected = b'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001:102A::]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + def test_malformed_headers_coped_with(self): + # Issue 19996 + body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + + self.assertEqual(resp.getheader('First'), 'val') + self.assertEqual(resp.getheader('Second'), 'val') + + def test_invalid_headers(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/') + + # http://tools.ietf.org/html/rfc7230#section-3.2.4, whitespace is no + # longer allowed in header names + cases = ( + (b'Invalid\r\nName', b'ValidValue'), + (b'Invalid\rName', b'ValidValue'), + (b'Invalid\nName', b'ValidValue'), + (b'\r\nInvalidName', b'ValidValue'), + (b'\rInvalidName', b'ValidValue'), + (b'\nInvalidName', b'ValidValue'), + (b' InvalidName', b'ValidValue'), + (b'\tInvalidName', b'ValidValue'), + (b'Invalid:Name', b'ValidValue'), + (b':InvalidName', b'ValidValue'), + (b'ValidName', b'Invalid\r\nValue'), + (b'ValidName', b'Invalid\rValue'), + (b'ValidName', b'Invalid\nValue'), + (b'ValidName', b'InvalidValue\r\n'), + (b'ValidName', b'InvalidValue\r'), + (b'ValidName', b'InvalidValue\n'), + ) + for name, value in cases: + with self.subTest((name, value)): + with self.assertRaisesRegex(ValueError, 'Invalid header'): + conn.putheader(name, value) + + +class BasicTest(TestCase): + def test_status_lines(self): + # Test HTTP status lines + + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(0), b'') # Issue #20007 + self.assertFalse(resp.isclosed()) + self.assertFalse(resp.closed) + self.assertEqual(resp.read(), b"Text") + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + self.assertRaises(client.BadStatusLine, resp.begin) + + def test_bad_status_repr(self): + exc = client.BadStatusLine('') + self.assertEqual(repr(exc), '''BadStatusLine("\'\'",)''') + + def test_partial_reads(self): + # if we have a length, the system knows when to close itself + # same behaviour than when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos(self): + # if we have a length, the system knows when to close itself + # same behaviour than when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_reads_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + + def test_partial_reads_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + + def test_partial_readintos_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:80", "www.python.org", 80), + ("www.python.org:", "www.python.org", 80), + ("www.python.org", "www.python.org", 80), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", 80)): + c = client.HTTPConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + def test_response_headers(self): + # test response with multiple message headers with the same field name. + text = ('HTTP/1.1 200 OK\r\n' + 'Set-Cookie: Customer="WILE_E_COYOTE"; ' + 'Version="1"; Path="/acme"\r\n' + 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' + ' Path="/acme"\r\n' + '\r\n' + 'No body\r\n') + hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"' + ', ' + 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"') + s = FakeSocket(text) + r = client.HTTPResponse(s) + r.begin() + cookies = r.getheader("Set-Cookie") + self.assertEqual(cookies, hdr) + + def test_read_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + if resp.read(): + self.fail("Did not expect response from HEAD request") + + def test_readinto_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + if resp.readinto(b) != 0: + self.fail("Did not expect response from HEAD request") + self.assertEqual(bytes(b), b'\x00'*5) + + def test_too_many_headers(self): + headers = '\r\n'.join('Header%d: foo' % i + for i in range(client._MAXHEADERS + 1)) + '\r\n' + text = ('HTTP/1.1 200 OK\r\n' + headers) + s = FakeSocket(text) + r = client.HTTPResponse(s) + self.assertRaisesRegex(client.HTTPException, + r"got more than \d+ headers", r.begin) + + def test_send_file(self): + expected = (b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' + b'Accept-Encoding: identity\r\nContent-Length:') + + with open(__file__, 'rb') as body: + conn = client.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertTrue(sock.data.startswith(expected), '%r != %r' % + (sock.data[:len(expected)], expected)) + + def test_send(self): + expected = b'this is a test this is only a test' + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + conn.send(expected) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(array.array('b', expected)) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(io.BytesIO(expected)) + self.assertEqual(expected, sock.data) + + def test_send_updating_file(self): + def data(): + yield 'data' + yield None + yield 'data_two' + + class UpdatingFile(): + mode = 'r' + d = data() + def read(self, blocksize=-1): + return self.d.__next__() + + expected = b'data' + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.send(UpdatingFile()) + self.assertEqual(sock.data, expected) + + + def test_send_iter(self): + expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \ + b'\r\nonetwothree' + + def body(): + yield b"one" + yield b"two" + yield b"three" + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.request('GET', '/foo', body(), {'Content-Length': '11'}) + self.assertEqual(sock.data, expected) + + def test_send_type_error(self): + # See: Issue #12676 + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + with self.assertRaises(TypeError): + conn.request('POST', 'test', conn) + + def test_chunked(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd! \r\n' + '8\r\n' + 'and now \r\n' + '22\r\n' + 'for something completely different\r\n' + ) + expected = b'hello world! and now for something completely different' + sock = FakeSocket(chunked_start + '0\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + '0\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(n) + resp.read(n) + resp.read(), expected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_readinto_chunked(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd! \r\n' + '8\r\n' + 'and now \r\n' + '22\r\n' + 'for something completely different\r\n' + ) + expected = b'hello world! and now for something completely different' + nexpected = len(expected) + b = bytearray(128) + + sock = FakeSocket(chunked_start + '0\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + n = resp.readinto(b) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(n, nexpected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + '0\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + m = memoryview(b) + i = resp.readinto(m[0:n]) + i += resp.readinto(m[i:n + i]) + i += resp.readinto(m[i:]) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(i, nexpected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + n = resp.readinto(b) + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + '0\r\n') + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_readinto_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + '0\r\n') + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertEqual(bytes(b), b'\x00'*5) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_negative_content_length(self): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), b'Hello\r\n') + self.assertTrue(resp.isclosed()) + + def test_incomplete_read(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, b'Hello\r\n') + self.assertEqual(repr(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertEqual(str(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertTrue(resp.isclosed()) + else: + self.fail('IncompleteRead expected') + + def test_epipe(self): + sock = EPipeSocket( + "HTTP/1.0 401 Authorization Required\r\n" + "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"example\"\r\n", + b"Content-Length") + conn = client.HTTPConnection("example.com") + conn.sock = sock + self.assertRaises(OSError, + lambda: conn.request("PUT", "/url", "body")) + resp = conn.getresponse() + self.assertEqual(401, resp.status) + self.assertEqual("Basic realm=\"example\"", + resp.getheader("www-authenticate")) + + # Test lines overflowing the max line size (_MAXLINE in http.client) + + def test_overflowing_status_line(self): + body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n" + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises((client.LineTooLong, client.BadStatusLine), resp.begin) + + def test_overflowing_header_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises(client.LineTooLong, resp.begin) + + def test_overflowing_chunked_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + + '0' * 65536 + 'a\r\n' + 'hello world\r\n' + '0\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertRaises(client.LineTooLong, resp.read) + + def test_early_eof(self): + # Test httpresponse with no \r\n termination, + body = "HTTP/1.1 200 Ok" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_delayed_ack_opt(self): + # Test that Nagle/delayed_ack optimistaion works correctly. + + # For small payloads, it should coalesce the body with + # headers, resulting in a single sendall() call + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + body = b'x' * (conn.mss - 1) + conn.request('POST', '/', body) + self.assertEqual(sock.sendall_calls, 1) + + # For large payloads, it should send the headers and + # then the body, resulting in more than one sendall() + # call + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + body = b'x' * conn.mss + conn.request('POST', '/', body) + self.assertGreater(sock.sendall_calls, 1) + + def test_error_leak(self): + # Test that the socket is not leaked if getresponse() fails + conn = client.HTTPConnection('example.com') + response = None + class Response(client.HTTPResponse): + def __init__(self, *pos, **kw): + nonlocal response + response = self # Avoid garbage collector closing the socket + client.HTTPResponse.__init__(self, *pos, **kw) + conn.response_class = Response + conn.sock = FakeSocket('') # Emulate server dropping connection + conn.request('GET', '/') + self.assertRaises(client.BadStatusLine, conn.getresponse) + self.assertTrue(response.closed) + self.assertTrue(conn.sock.file_closed) + + +class OfflineTest(TestCase): + def test_all(self): + # Documented objects defined in the module should be in __all__ + expected = {"responses"} # White-list documented dict() object + # HTTPMessage, parse_headers(), and the HTTP status code constants are + # intentionally omitted for simplicity + blacklist = {"HTTPMessage", "parse_headers"} + for name in dir(client): + if name in blacklist: + continue + module_object = getattr(client, name) + if getattr(module_object, "__module__", None) == "http.client": + expected.add(name) + self.assertCountEqual(client.__all__, expected) + + def test_responses(self): + self.assertEqual(client.responses[client.NOT_FOUND], "Not Found") + + +class SourceAddressTest(TestCase): + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.source_port = support.find_unused_port() + self.serv.listen(5) + self.conn = None + + def tearDown(self): + if self.conn: + self.conn.close() + self.conn = None + self.serv.close() + self.serv = None + + def testHTTPConnectionSourceAddress(self): + self.conn = client.HTTPConnection(HOST, self.port, + source_address=('', self.source_port)) + self.conn.connect() + self.assertEqual(self.conn.sock.getsockname()[1], self.source_port) + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not defined') + def testHTTPSConnectionSourceAddress(self): + self.conn = client.HTTPSConnection(HOST, self.port, + source_address=('', self.source_port)) + # We don't test anything here other the constructor not barfing as + # this code doesn't deal with setting up an active running SSL server + # for an ssl_wrapped connect() to actually return from. + + +class TimeoutTest(TestCase): + PORT = None + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + TimeoutTest.PORT = support.bind_port(self.serv) + self.serv.listen(5) + + def tearDown(self): + self.serv.close() + self.serv = None + + def testTimeoutAttribute(self): + # This will prove that the timeout gets through HTTPConnection + # and into the socket. + + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + # no timeout -- do not use global socket default + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, + timeout=None) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), None) + httpConn.close() + + # a value + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30) + httpConn.connect() + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + +class HTTPSTest(TestCase): + + def setUp(self): + if not hasattr(client, 'HTTPSConnection'): + self.skipTest('ssl support required') + + def make_server(self, certfile): + from test.ssl_servers import make_https_server + return make_https_server(self, certfile=certfile) + + def test_attributes(self): + # simple test to check it's storing the timeout + h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) + self.assertEqual(h.timeout, 30) + + def test_networked(self): + # Default settings: requires a valid cert from a trusted CA + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + h = client.HTTPSConnection('self-signed.pythontest.net', 443) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_networked_noverification(self): + # Switch off cert verification + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl._create_unverified_context() + h = client.HTTPSConnection('self-signed.pythontest.net', 443, + context=context) + h.request('GET', '/') + resp = h.getresponse() + h.close() + self.assertIn('nginx', resp.getheader('server')) + + @support.system_must_validate_cert + def test_networked_trusted_by_default_cert(self): + # Default settings: requires a valid cert from a trusted CA + support.requires('network') + with support.transient_internet('www.python.org'): + h = client.HTTPSConnection('www.python.org', 443) + h.request('GET', '/') + resp = h.getresponse() + content_type = resp.getheader('content-type') + h.close() + self.assertIn('text/html', content_type) + + def test_networked_good_cert(self): + # We feed the server's cert as a validating cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_selfsigned_pythontestdotnet) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h.request('GET', '/') + resp = h.getresponse() + server_string = resp.getheader('server') + h.close() + self.assertIn('nginx', server_string) + + def test_networked_bad_cert(self): + # We feed a "CA" cert that is unrelated to the server's cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_unknown_cert(self): + # The custom cert isn't known to the default trust bundle + import ssl + server = self.make_server(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_good_hostname(self): + # The (valid) cert validates the HTTP hostname + import ssl + server = self.make_server(CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + + def test_local_bad_hostname(self): + # The (valid) cert doesn't validate the HTTP hostname + import ssl + server = self.make_server(CERT_fakehostname) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(CERT_fakehostname) + h = client.HTTPSConnection('localhost', server.port, context=context) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # Same with explicit check_hostname=True + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # With check_hostname=False, the mismatching is ignored + context.check_hostname = False + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=False) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + # The context's check_hostname setting is used if one isn't passed to + # HTTPSConnection. + context.check_hostname = False + h = client.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + self.assertEqual(h.getresponse().status, 404) + # Passing check_hostname to HTTPSConnection should override the + # context's setting. + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not available') + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPSConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:443", "www.python.org", 443), + ("www.python.org:", "www.python.org", 443), + ("www.python.org", "www.python.org", 443), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 443), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", + 443)): + c = client.HTTPSConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + +class RequestBodyTest(TestCase): + """Test cases where a request includes a message body.""" + + def setUp(self): + self.conn = client.HTTPConnection('example.com') + self.conn.sock = self.sock = FakeSocket("") + self.conn.sock = self.sock + + def get_headers_and_fp(self): + f = io.BytesIO(self.sock.data) + f.readline() # read the request line + message = client.parse_headers(f) + return message, f + + def test_manual_content_length(self): + # Set an incorrect content-length so that we can verify that + # it will not be over-ridden by the library. + self.conn.request("PUT", "/url", "body", + {"Content-Length": "42"}) + message, f = self.get_headers_and_fp() + self.assertEqual("42", message.get("content-length")) + self.assertEqual(4, len(f.read())) + + def test_ascii_body(self): + self.conn.request("PUT", "/url", "body") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("4", message.get("content-length")) + self.assertEqual(b'body', f.read()) + + def test_latin1_body(self): + self.conn.request("PUT", "/url", "body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_bytes_body(self): + self.conn.request("PUT", "/url", b"body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as f: + f.write("body") + with open(support.TESTFN) as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("4", message.get("content-length")) + self.assertEqual(b'body', f.read()) + + def test_binary_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "wb") as f: + f.write(b"body\xc1") + with open(support.TESTFN, "rb") as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + +class HTTPResponseTest(TestCase): + + def setUp(self): + body = "HTTP/1.1 200 Ok\r\nMy-Header: first-value\r\nMy-Header: \ + second-value\r\n\r\nText" + sock = FakeSocket(body) + self.resp = client.HTTPResponse(sock) + self.resp.begin() + + def test_getting_header(self): + header = self.resp.getheader('My-Header') + self.assertEqual(header, 'first-value, second-value') + + header = self.resp.getheader('My-Header', 'some default') + self.assertEqual(header, 'first-value, second-value') + + def test_getting_nonexistent_header_with_string_default(self): + header = self.resp.getheader('No-Such-Header', 'default-value') + self.assertEqual(header, 'default-value') + + def test_getting_nonexistent_header_with_iterable_default(self): + header = self.resp.getheader('No-Such-Header', ['default', 'values']) + self.assertEqual(header, 'default, values') + + header = self.resp.getheader('No-Such-Header', ('default', 'values')) + self.assertEqual(header, 'default, values') + + def test_getting_nonexistent_header_without_default(self): + header = self.resp.getheader('No-Such-Header') + self.assertEqual(header, None) + + def test_getting_header_defaultint(self): + header = self.resp.getheader('No-Such-Header',default=42) + self.assertEqual(header, 42) + +class TunnelTests(TestCase): + def setUp(self): + response_text = ( + 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n' # Reply to HEAD + 'Content-Length: 42\r\n\r\n' + ) + + def create_connection(address, timeout=None, source_address=None): + return FakeSocket(response_text, host=address[0], port=address[1]) + + self.host = 'proxy.com' + self.conn = client.HTTPConnection(self.host) + self.conn._create_connection = create_connection + + def tearDown(self): + self.conn.close() + + def test_set_tunnel_host_port_headers(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers) + + def test_disallow_set_tunnel_after_connect(self): + # Once connected, we shouldn't be able to tunnel anymore + self.conn.connect() + self.assertRaises(RuntimeError, self.conn.set_tunnel, + 'destination.com') + + def test_connect_with_tunnel(self): + self.conn.set_tunnel('destination.com') + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + # issue22095 + self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + # This test should be removed when CONNECT gets the HTTP/1.1 blessing + self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) + + def test_connect_put_request(self): + self.conn.set_tunnel('destination.com') + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + + +@support.reap_threads +def test_main(verbose=None): + support.run_unittest(HeaderTests, OfflineTest, BasicTest, TimeoutTest, + HTTPSTest, RequestBodyTest, SourceAddressTest, + HTTPResponseTest, TunnelTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/3.4/test_select.py b/src/greentest/3.4/test_select.py new file mode 100644 index 0000000..b0d6661 --- /dev/null +++ b/src/greentest/3.4/test_select.py @@ -0,0 +1,85 @@ +import errno +import os +import select +import sys +import unittest +from test import support + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + #from IPython.core.debugger import Tracer; Tracer()() ## DEBUG ## + + try: + select.select([fd], [], [], 0) + except OSError as err: + self.assertEqual(err.errno, errno.EBADF) + else: + self.fail("exception not raised") + + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + def test_select(self): + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print('timeout =', tout) + rfd, wfd, xfd = select.select([p], [], [], tout) + if (rfd, wfd, xfd) == ([], [], []): + continue + if (rfd, wfd, xfd) == ([p], [], []): + line = p.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select():', rfd, wfd, xfd) + p.close() + + # Issue 16230: Crash on select resized list + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + self.assertEqual(select.select([], a, []), ([], a[:5], [])) + +def test_main(): + support.run_unittest(SelectTestCase) + support.reap_children() + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.4/test_selectors.py b/src/greentest/3.4/test_selectors.py new file mode 100644 index 0000000..952fda6 --- /dev/null +++ b/src/greentest/3.4/test_selectors.py @@ -0,0 +1,467 @@ +import errno +import os +import random +import selectors +import signal +import socket +import sys +from test import support +from time import sleep +import unittest +import unittest.mock +try: + from time import monotonic as time +except ImportError: + from time import time as time +try: + import resource +except ImportError: + resource = None + + +if hasattr(socket, 'socketpair'): + socketpair = socket.socketpair +else: + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): + with socket.socket(family, type, proto) as l: + l.bind((support.HOST, 0)) + l.listen(3) + c = socket.socket(family, type, proto) + try: + c.connect(l.getsockname()) + caddr = c.getsockname() + while True: + a, addr = l.accept() + # check that we've got the correct client + if addr == caddr: + return c, a + a.close() + except OSError: + c.close() + raise + + +def find_ready_matching(ready, flag): + match = [] + for key, events in ready: + if events & flag: + match.append(key.fileobj) + return match + + +class BaseSelectorTestCase(unittest.TestCase): + + def make_socketpair(self): + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + return rd, wr + + def test_register(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIsInstance(key, selectors.SelectorKey) + self.assertEqual(key.fileobj, rd) + self.assertEqual(key.fd, rd.fileno()) + self.assertEqual(key.events, selectors.EVENT_READ) + self.assertEqual(key.data, "data") + + # register an unknown event + self.assertRaises(ValueError, s.register, 0, 999999) + + # register an invalid FD + self.assertRaises(ValueError, s.register, -10, selectors.EVENT_READ) + + # register twice + self.assertRaises(KeyError, s.register, rd, selectors.EVENT_READ) + + # register the same FD, but with a different object + self.assertRaises(KeyError, s.register, rd.fileno(), + selectors.EVENT_READ) + + def test_unregister(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + s.unregister(rd) + + # unregister an unknown file obj + self.assertRaises(KeyError, s.unregister, 999999) + + # unregister twice + self.assertRaises(KeyError, s.unregister, rd) + + def test_unregister_after_fd_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + r, w = rd.fileno(), wr.fileno() + s.register(r, selectors.EVENT_READ) + s.register(w, selectors.EVENT_WRITE) + rd.close() + wr.close() + s.unregister(r) + s.unregister(w) + + @unittest.skipUnless(os.name == 'posix', "requires posix") + def test_unregister_after_fd_close_and_reuse(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + r, w = rd.fileno(), wr.fileno() + s.register(r, selectors.EVENT_READ) + s.register(w, selectors.EVENT_WRITE) + rd2, wr2 = self.make_socketpair() + rd.close() + wr.close() + os.dup2(rd2.fileno(), r) + os.dup2(wr2.fileno(), w) + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + s.unregister(r) + s.unregister(w) + + def test_unregister_after_socket_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + rd.close() + wr.close() + s.unregister(rd) + s.unregister(wr) + + def test_modify(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ) + + # modify events + key2 = s.modify(rd, selectors.EVENT_WRITE) + self.assertNotEqual(key.events, key2.events) + self.assertEqual(key2, s.get_key(rd)) + + s.unregister(rd) + + # modify data + d1 = object() + d2 = object() + + key = s.register(rd, selectors.EVENT_READ, d1) + key2 = s.modify(rd, selectors.EVENT_READ, d2) + self.assertEqual(key.events, key2.events) + self.assertNotEqual(key.data, key2.data) + self.assertEqual(key2, s.get_key(rd)) + self.assertEqual(key2.data, d2) + + # modify unknown file obj + self.assertRaises(KeyError, s.modify, 999999, selectors.EVENT_READ) + + # modify use a shortcut + d3 = object() + s.register = unittest.mock.Mock() + s.unregister = unittest.mock.Mock() + + s.modify(rd, selectors.EVENT_READ, d3) + self.assertFalse(s.register.called) + self.assertFalse(s.unregister.called) + + def test_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + mapping = s.get_map() + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + + s.close() + self.assertRaises(KeyError, s.get_key, rd) + self.assertRaises(KeyError, s.get_key, wr) + self.assertRaises(KeyError, mapping.__getitem__, rd) + self.assertRaises(KeyError, mapping.__getitem__, wr) + + def test_get_key(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertEqual(key, s.get_key(rd)) + + # unknown file obj + self.assertRaises(KeyError, s.get_key, 999999) + + def test_get_map(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + keys = s.get_map() + self.assertFalse(keys) + self.assertEqual(len(keys), 0) + self.assertEqual(list(keys), []) + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIn(rd, keys) + self.assertEqual(key, keys[rd]) + self.assertEqual(len(keys), 1) + self.assertEqual(list(keys), [rd.fileno()]) + self.assertEqual(list(keys.values()), [key]) + + # unknown file obj + with self.assertRaises(KeyError): + keys[999999] + + # Read-only mapping + with self.assertRaises(TypeError): + del keys[rd] + + def test_select(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + wr_key = s.register(wr, selectors.EVENT_WRITE) + + result = s.select() + for key, events in result: + self.assertTrue(isinstance(key, selectors.SelectorKey)) + self.assertTrue(events) + self.assertFalse(events & ~(selectors.EVENT_READ | + selectors.EVENT_WRITE)) + + self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result) + + def test_context_manager(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + with s as sel: + sel.register(rd, selectors.EVENT_READ) + sel.register(wr, selectors.EVENT_WRITE) + + self.assertRaises(KeyError, s.get_key, rd) + self.assertRaises(KeyError, s.get_key, wr) + + def test_fileno(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + if hasattr(s, 'fileno'): + fd = s.fileno() + self.assertTrue(isinstance(fd, int)) + self.assertGreaterEqual(fd, 0) + + def test_selector(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + NUM_SOCKETS = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_SOCKETS): + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = s.select() + ready_writers = find_ready_matching(ready, selectors.EVENT_WRITE) + if not ready_writers: + self.fail("no sockets ready for writing") + wr = random.choice(ready_writers) + wr.send(MSG) + + for i in range(10): + ready = s.select() + ready_readers = find_ready_matching(ready, + selectors.EVENT_READ) + if ready_readers: + break + # there might be a delay between the write to the write end and + # the read end is reported ready + sleep(0.1) + else: + self.fail("no sockets ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = rd.recv(MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + s.unregister(r2w[rd]) + s.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_SOCKETS) + + @unittest.skipIf(sys.platform == 'win32', + 'select.select() cannot be used with empty fd sets') + def test_empty_select(self): + # Issue #23009: Make sure EpollSelector.select() works when no FD is + # registered. + s = self.SELECTOR() + self.addCleanup(s.close) + self.assertEqual(s.select(timeout=0), []) + + def test_timeout(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(wr, selectors.EVENT_WRITE) + t = time() + self.assertEqual(1, len(s.select(0))) + self.assertEqual(1, len(s.select(-1))) + self.assertLess(time() - t, 0.5) + + s.unregister(wr) + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(0)) + self.assertFalse(s.select(-1)) + self.assertLess(time() - t, 0.5) + + t0 = time() + self.assertFalse(s.select(1)) + t1 = time() + dt = t1 - t0 + # Tolerate 2.0 seconds for very slow buildbots + self.assertTrue(0.8 <= dt <= 2.0, dt) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_select_interrupt(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + orig_alrm_handler = signal.signal(signal.SIGALRM, lambda *args: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(signal.alarm, 0) + + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(2)) + self.assertLess(time() - t, 2.5) + + +class ScalableSelectorMixIn: + + # see issue #18963 for why it's skipped on older OS X versions + @support.requires_mac_ver(10, 5) + @unittest.skipUnless(resource, "Test needs resource module") + def test_above_fd_setsize(self): + # A scalable implementation should have no problem with more than + # FD_SETSIZE file descriptors. Since we don't know the value, we just + # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling. + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, + (soft, hard)) + NUM_FDS = min(hard, 2**16) + except (OSError, ValueError): + NUM_FDS = soft + + # guard for already allocated FDs (stdin, stdout...) + NUM_FDS -= 32 + + s = self.SELECTOR() + self.addCleanup(s.close) + + for i in range(NUM_FDS // 2): + try: + rd, wr = self.make_socketpair() + except OSError: + # too many FDs, skip - note that we should only catch EMFILE + # here, but apparently *BSD and Solaris can fail upon connect() + # or bind() with EADDRNOTAVAIL, so let's be safe + self.skipTest("FD limit reached") + + try: + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + except OSError as e: + if e.errno == errno.ENOSPC: + # this can be raised by epoll if we go over + # fs.epoll.max_user_watches sysctl + self.skipTest("FD limit reached") + raise + + self.assertEqual(NUM_FDS // 2, len(s.select())) + + +class DefaultSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.DefaultSelector + + +class SelectSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.SelectSelector + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class PollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'PollSelector', None) + + +@unittest.skipUnless(hasattr(selectors, 'EpollSelector'), + "Test needs selectors.EpollSelector") +class EpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'EpollSelector', None) + + +@unittest.skipUnless(hasattr(selectors, 'KqueueSelector'), + "Test needs selectors.KqueueSelector)") +class KqueueSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'KqueueSelector', None) + + +def test_main(): + tests = [DefaultSelectorTestCase, SelectSelectorTestCase, + PollSelectorTestCase, EpollSelectorTestCase, + KqueueSelectorTestCase] + support.run_unittest(*tests) + support.reap_children() + + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.4/test_smtpd.py b/src/greentest/3.4/test_smtpd.py new file mode 100644 index 0000000..93f14c4 --- /dev/null +++ b/src/greentest/3.4/test_smtpd.py @@ -0,0 +1,557 @@ +import unittest +from test import support, mock_socket +import socket +import io +import smtpd +import asyncore + + +class DummyServer(smtpd.SMTPServer): + def __init__(self, localaddr, remoteaddr): + smtpd.SMTPServer.__init__(self, localaddr, remoteaddr) + self.messages = [] + + def process_message(self, peer, mailfrom, rcpttos, data): + self.messages.append((peer, mailfrom, rcpttos, data)) + if data == 'return status': + return '250 Okish' + + +class DummyDispatcherBroken(Exception): + pass + + +class BrokenDummyServer(DummyServer): + def listen(self, num): + raise DummyDispatcherBroken() + + +class SMTPDServerTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def test_process_message_unimplemented(self): + server = smtpd.SMTPServer('a', 'b') + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + + write_line(b'HELO example') + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class SMTPDChannelTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer('a', 'b') + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_broken_connect(self): + self.assertRaises(DummyDispatcherBroken, BrokenDummyServer, 'a', 'b') + + def test_server_accept(self): + self.server.handle_accept() + + def test_missing_data(self): + self.write_line(b'') + self.assertEqual(self.channel.socket.last, + b'500 Error: bad syntax\r\n') + + def test_EHLO(self): + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, b'250 HELP\r\n') + + def test_EHLO_bad_syntax(self): + self.write_line(b'EHLO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: EHLO hostname\r\n') + + def test_EHLO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_EHLO_HELO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO(self): + name = smtpd.socket.getfqdn() + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + '250 {}\r\n'.format(name).encode('ascii')) + + def test_HELO_EHLO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELP(self): + self.write_line(b'HELP') + self.assertEqual(self.channel.socket.last, + b'250 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELP_command(self): + self.write_line(b'HELP MAIL') + self.assertEqual(self.channel.socket.last, + b'250 Syntax: MAIL FROM:
\r\n') + + def test_HELP_command_unknown(self): + self.write_line(b'HELP SPAM') + self.assertEqual(self.channel.socket.last, + b'501 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELO_bad_syntax(self): + self.write_line(b'HELO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: HELO hostname\r\n') + + def test_HELO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO_parameter_rejected_when_extensions_not_enabled(self): + self.extended_smtp = False + self.write_line(b'HELO example') + self.write_line(b'MAIL from: SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_allows_space_after_colon(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_extended_MAIL_allows_space_after_colon(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: size=20') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_NOOP(self): + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_HELO_NOOP(self): + self.write_line(b'HELO example') + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_NOOP_bad_syntax(self): + self.write_line(b'NOOP hi') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: NOOP\r\n') + + def test_QUIT(self): + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_HELO_QUIT(self): + self.write_line(b'HELO example') + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_QUIT_arg_ignored(self): + self.write_line(b'QUIT bye bye') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_bad_state(self): + self.channel.smtp_state = 'BAD STATE' + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'451 Internal confusion\r\n') + + def test_command_too_long(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ' + + b'a' * self.channel.command_size_limit + + b'@example') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_limit_extended_with_SIZE(self): + self.write_line(b'EHLO example') + fill_len = self.channel.command_size_limit - len('MAIL from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 26) + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_data_longer_than_default_data_size_limit(self): + # Hack the default so we don't have to generate so much data. + self.channel.data_size_limit = 1048 + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'A' * self.channel.data_size_limit + + b'A\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + def test_MAIL_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=512') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_MAIL_invalid_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=invalid') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_RCPT_unknown_parameters(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 MAIL FROM parameters not recognized or not implemented\r\n') + + self.write_line(b'MAIL FROM:') + self.write_line(b'RCPT TO: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 RCPT TO parameters not recognized or not implemented\r\n') + + def test_MAIL_size_parameter_larger_than_default_data_size_limit(self): + self.channel.data_size_limit = 1048 + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=2096') + self.assertEqual(self.channel.socket.last, + b'552 Error: message size exceeds fixed maximum message size\r\n') + + def test_need_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'RCPT to:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: need MAIL command\r\n') + + def test_MAIL_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_missing_address(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_chevrons(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_empty_chevrons(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from:<>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_quoted_localpart(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_nested_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:eggs@example') + self.write_line(b'MAIL from:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: nested MAIL command\r\n') + + def test_VRFY(self): + self.write_line(b'VRFY eggs@example') + self.assertEqual(self.channel.socket.last, + b'252 Cannot VRFY user, but will accept message and attempt ' + \ + b'delivery\r\n') + + def test_VRFY_syntax(self): + self.write_line(b'VRFY') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: VRFY
\r\n') + + def test_EXPN_not_implemented(self): + self.write_line(b'EXPN') + self.assertEqual(self.channel.socket.last, + b'502 EXPN not implemented\r\n') + + def test_no_HELO_MAIL(self): + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_need_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'503 Error: need RCPT command\r\n') + + def test_RCPT_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
\r\n') + + def test_RCPT_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
[SP ]\r\n') + + def test_RCPT_lowercase_to_OK(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to: ') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_no_HELO_RCPT(self): + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [('peer', 'eggs@example', ['spam@example'], 'data\nmore')]) + + def test_DATA_syntax(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n') + + def test_no_HELO_DATA(self): + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_transparency_section_4_5_2(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'..\r\n.\r\n') + self.assertEqual(self.channel.received_data, '.') + + def test_multiple_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RCPT To:ham@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [('peer', 'eggs@example', ['spam@example','ham@example'], 'data')]) + + def test_manual_status(self): + # checks that the Channel is able to return a custom status message + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'return status\r\n.') + self.assertEqual(self.channel.socket.last, b'250 Okish\r\n') + + def test_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'MAIL From:foo@example') + self.write_line(b'RCPT To:eggs@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [('peer', 'foo@example', ['eggs@example'], 'data')]) + + def test_HELO_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_RSET_syntax(self): + self.write_line(b'RSET hi') + self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n') + + def test_unknown_command(self): + self.write_line(b'UNKNOWN_CMD') + self.assertEqual(self.channel.socket.last, + b'500 Error: command "UNKNOWN_CMD" not ' + \ + b'recognized\r\n') + + def test_attribute_deprecations(self): + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__server + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__server = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__line + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__line = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__state + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__state = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__greeting + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__greeting = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__mailfrom + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__mailfrom = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__rcpttos + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__rcpttos = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__data + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__data = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__fqdn + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__fqdn = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__peer + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__peer = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__conn + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__conn = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__addr + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__addr = 'spam' + + +class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer('a', 'b') + conn, addr = self.server.accept() + # Set DATA size limit to 32 bytes for easy testing + self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_data_limit_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [('peer', 'eggs@example', ['spam@example'], 'data\nmore')]) + + def test_data_limit_dialog_too_much_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'This message is longer than 32 bytes\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.4/test_socket.py b/src/greentest/3.4/test_socket.py new file mode 100644 index 0000000..140b4be --- /dev/null +++ b/src/greentest/3.4/test_socket.py @@ -0,0 +1,5180 @@ +import unittest +from test import support + +import errno +import io +import itertools +import socket +import select +import tempfile +import time +import traceback +import queue +import sys +import os +import array +import platform +import contextlib +from weakref import proxy +import signal +import math +import pickle +import struct +try: + import multiprocessing +except ImportError: + multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None + +HOST = support.HOST +MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return + +try: + import _thread as thread + import threading +except ImportError: + thread = None + threading = None +try: + import _socket +except ImportError: + _socket = None + + +def _have_socket_can(): + """Check whether CAN sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_rds(): + """Check whether RDS sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +HAVE_SOCKET_CAN = _have_socket_can() + +HAVE_SOCKET_RDS = _have_socket_rds() + +# Size in bytes of the int type +SIZEOF_INT = array.array("i").itemsize + +class SocketTCPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.serv.listen(1) + + def tearDown(self): + self.serv.close() + self.serv = None + +class SocketUDPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.port = support.bind_port(self.serv) + + def tearDown(self): + self.serv.close() + self.serv = None + +class ThreadSafeCleanupTestCase(unittest.TestCase): + """Subclass of unittest.TestCase with thread-safe cleanup methods. + + This subclass protects the addCleanup() and doCleanups() methods + with a recursive lock. + """ + + if threading: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cleanup_lock = threading.RLock() + + def addCleanup(self, *args, **kwargs): + with self._cleanup_lock: + return super().addCleanup(*args, **kwargs) + + def doCleanups(self, *args, **kwargs): + with self._cleanup_lock: + return super().doCleanups(*args, **kwargs) + +class SocketCANTest(unittest.TestCase): + + """To be able to run this test, a `vcan0` CAN interface can be created with + the following commands: + # modprobe vcan + # ip link add dev vcan0 type vcan + # ifconfig vcan0 up + """ + interface = 'vcan0' + bufsize = 128 + + """The CAN frame structure is defined in : + + struct can_frame { + canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ + __u8 can_dlc; /* data length code: 0 .. 8 */ + __u8 data[8] __attribute__((aligned(8))); + }; + """ + can_frame_fmt = "=IB3x8s" + can_frame_size = struct.calcsize(can_frame_fmt) + + """The Broadcast Management Command frame structure is defined + in : + + struct bcm_msg_head { + __u32 opcode; + __u32 flags; + __u32 count; + struct timeval ival1, ival2; + canid_t can_id; + __u32 nframes; + struct can_frame frames[0]; + } + + `bcm_msg_head` must be 8 bytes aligned because of the `frames` member (see + `struct can_frame` definition). Must use native not standard types for packing. + """ + bcm_cmd_msg_fmt = "@3I4l2I" + bcm_cmd_msg_fmt += "x" * (struct.calcsize(bcm_cmd_msg_fmt) % 8) + + def setUp(self): + self.s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + self.addCleanup(self.s.close) + try: + self.s.bind((self.interface,)) + except OSError: + self.skipTest('network interface `%s` does not exist' % + self.interface) + + +class SocketRDSTest(unittest.TestCase): + + """To be able to run this test, the `rds` kernel module must be loaded: + # modprobe rds + """ + bufsize = 8192 + + def setUp(self): + self.serv = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + self.addCleanup(self.serv.close) + try: + self.port = support.bind_port(self.serv) + except OSError: + self.skipTest('unable to bind RDS socket') + + +class ThreadableTest: + """Threadable Test class + + The ThreadableTest class makes it easy to create a threaded + client/server pair from an existing unit test. To create a + new threaded class from an existing unit test, use multiple + inheritance: + + class NewClass (OldClass, ThreadableTest): + pass + + This class defines two new fixture functions with obvious + purposes for overriding: + + clientSetUp () + clientTearDown () + + Any new test functions within the class must then define + tests in pairs, where the test name is preceeded with a + '_' to indicate the client portion of the test. Ex: + + def testFoo(self): + # Server portion + + def _testFoo(self): + # Client portion + + Any exceptions raised by the clients during their tests + are caught and transferred to the main thread to alert + the testing framework. + + Note, the server setup function cannot call any blocking + functions that rely on the client thread during setup, + unless serverExplicitReady() is called just before + the blocking call (such as in setting up a client/server + connection and performing the accept() in setUp(). + """ + + def __init__(self): + # Swap the true setup function + self.__setUp = self.setUp + self.__tearDown = self.tearDown + self.setUp = self._setUp + self.tearDown = self._tearDown + + def serverExplicitReady(self): + """This method allows the server to explicitly indicate that + it wants the client thread to proceed. This is useful if the + server is about to execute a blocking routine that is + dependent upon the client thread during its setup routine.""" + self.server_ready.set() + + def _setUp(self): + self.server_ready = threading.Event() + self.client_ready = threading.Event() + self.done = threading.Event() + self.queue = queue.Queue(1) + self.server_crashed = False + + # Do some munging to start the client test. + methodname = self.id() + i = methodname.rfind('.') + methodname = methodname[i+1:] + test_method = getattr(self, '_' + methodname) + self.client_thread = thread.start_new_thread( + self.clientRun, (test_method,)) + + try: + self.__setUp() + except: + self.server_crashed = True + raise + finally: + self.server_ready.set() + self.client_ready.wait() + + def _tearDown(self): + self.__tearDown() + self.done.wait() + + if self.queue.qsize(): + exc = self.queue.get() + raise exc + + def clientRun(self, test_func): + self.server_ready.wait() + self.clientSetUp() + self.client_ready.set() + if self.server_crashed: + self.clientTearDown() + return + if not hasattr(test_func, '__call__'): + raise TypeError("test_func must be a callable function") + try: + test_func() + except BaseException as e: + self.queue.put(e) + finally: + self.clientTearDown() + + def clientSetUp(self): + raise NotImplementedError("clientSetUp must be implemented.") + + def clientTearDown(self): + self.done.set() + thread.exit() + +class ThreadedTCPSocketTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketUDPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedCANSocketTest(SocketCANTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketCANTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + try: + self.cli.bind((self.interface,)) + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedRDSSocketTest(SocketRDSTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketRDSTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + try: + # RDS sockets must be bound explicitly to send or receive data + self.cli.bind((HOST, 0)) + self.cli_addr = self.cli.getsockname() + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class SocketConnectedTest(ThreadedTCPSocketTest): + """Socket tests for client-server connection. + + self.cli_conn is a client socket connected to the server. The + setUp() method guarantees that it is connected to the server. + """ + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + ThreadedTCPSocketTest.setUp(self) + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + ThreadedTCPSocketTest.tearDown(self) + + def clientSetUp(self): + ThreadedTCPSocketTest.clientSetUp(self) + self.cli.connect((HOST, self.port)) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + ThreadedTCPSocketTest.clientTearDown(self) + +class SocketPairTest(unittest.TestCase, ThreadableTest): + + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.serv, self.cli = socket.socketpair() + + def tearDown(self): + self.serv.close() + self.serv = None + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +# The following classes are used by the sendmsg()/recvmsg() tests. +# Combining, for instance, ConnectedStreamTestMixin and TCPTestBase +# gives a drop-in replacement for SocketConnectedTest, but different +# address families can be used, and the attributes serv_addr and +# cli_addr will be set to the addresses of the endpoints. + +class SocketTestBase(unittest.TestCase): + """A base class for socket tests. + + Subclasses must provide methods newSocket() to return a new socket + and bindSock(sock) to bind it to an unused address. + + Creates a socket self.serv and sets self.serv_addr to its address. + """ + + def setUp(self): + self.serv = self.newSocket() + self.bindServer() + + def bindServer(self): + """Bind server socket and set self.serv_addr to its address.""" + self.bindSock(self.serv) + self.serv_addr = self.serv.getsockname() + + def tearDown(self): + self.serv.close() + self.serv = None + + +class SocketListeningTestMixin(SocketTestBase): + """Mixin to listen on the server socket.""" + + def setUp(self): + super().setUp() + self.serv.listen(1) + + +class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, + ThreadableTest): + """Mixin to add client socket and allow client/server tests. + + Client socket is self.cli and its address is self.cli_addr. See + ThreadableTest for usage information. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = self.newClientSocket() + self.bindClient() + + def newClientSocket(self): + """Return a new socket for use as client.""" + return self.newSocket() + + def bindClient(self): + """Bind client socket and set self.cli_addr to its address.""" + self.bindSock(self.cli) + self.cli_addr = self.cli.getsockname() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +class ConnectedStreamTestMixin(SocketListeningTestMixin, + ThreadedSocketTestMixin): + """Mixin to allow client/server stream tests with connected client. + + Server's socket representing connection to client is self.cli_conn + and client's connection to server is self.serv_conn. (Based on + SocketConnectedTest.) + """ + + def setUp(self): + super().setUp() + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + super().tearDown() + + def clientSetUp(self): + super().clientSetUp() + self.cli.connect(self.serv_addr) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + super().clientTearDown() + + +class UnixSocketTestBase(SocketTestBase): + """Base class for Unix-domain socket tests.""" + + # This class is used for file descriptor passing tests, so we + # create the sockets in a private directory so that other users + # can't send anything that might be problematic for a privileged + # user running the tests. + + def setUp(self): + self.dir_path = tempfile.mkdtemp() + self.addCleanup(os.rmdir, self.dir_path) + super().setUp() + + def bindSock(self, sock): + path = tempfile.mktemp(dir=self.dir_path) + sock.bind(path) + self.addCleanup(support.unlink, path) + +class UnixStreamBase(UnixSocketTestBase): + """Base class for Unix-domain SOCK_STREAM tests.""" + + def newSocket(self): + return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + +class InetTestBase(SocketTestBase): + """Base class for IPv4 socket tests.""" + + host = HOST + + def setUp(self): + super().setUp() + self.port = self.serv_addr[1] + + def bindSock(self, sock): + support.bind_port(sock, host=self.host) + +class TCPTestBase(InetTestBase): + """Base class for TCP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + +class UDPTestBase(InetTestBase): + """Base class for UDP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + +class SCTPStreamBase(InetTestBase): + """Base class for SCTP tests in one-to-one (SOCK_STREAM) mode.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM, + socket.IPPROTO_SCTP) + + +class Inet6TestBase(InetTestBase): + """Base class for IPv6 socket tests.""" + + host = support.HOSTv6 + +class UDP6TestBase(Inet6TestBase): + """Base class for UDP-over-IPv6 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + + +# Test-skipping decorators for use with ThreadableTest. + +def skipWithClientIf(condition, reason): + """Skip decorated test if condition is true, add client_skip decorator. + + If the decorated object is not a class, sets its attribute + "client_skip" to a decorator which will return an empty function + if the test is to be skipped, or the original function if it is + not. This can be used to avoid running the client part of a + skipped test when using ThreadableTest. + """ + def client_pass(*args, **kwargs): + pass + def skipdec(obj): + retval = unittest.skip(reason)(obj) + if not isinstance(obj, type): + retval.client_skip = lambda f: client_pass + return retval + def noskipdec(obj): + if not (isinstance(obj, type) or hasattr(obj, "client_skip")): + obj.client_skip = lambda f: f + return obj + return skipdec if condition else noskipdec + + +def requireAttrs(obj, *attributes): + """Skip decorated test if obj is missing any of the given attributes. + + Sets client_skip attribute as skipWithClientIf() does. + """ + missing = [name for name in attributes if not hasattr(obj, name)] + return skipWithClientIf( + missing, "don't have " + ", ".join(name for name in missing)) + + +def requireSocket(*args): + """Skip decorated test if a socket cannot be created with given arguments. + + When an argument is given as a string, will use the value of that + attribute of the socket module, or skip the test if it doesn't + exist. Sets client_skip attribute as skipWithClientIf() does. + """ + err = None + missing = [obj for obj in args if + isinstance(obj, str) and not hasattr(socket, obj)] + if missing: + err = "don't have " + ", ".join(name for name in missing) + else: + callargs = [getattr(socket, obj) if isinstance(obj, str) else obj + for obj in args] + try: + s = socket.socket(*callargs) + except OSError as e: + # XXX: check errno? + err = str(e) + else: + s.close() + return skipWithClientIf( + err is not None, + "can't create socket({0}): {1}".format( + ", ".join(str(o) for o in args), err)) + + +####################################################################### +## Begin Tests + +class GeneralModuleTests(unittest.TestCase): + + def test_SocketType_is_socketobject(self): + import _socket + self.assertTrue(socket.SocketType is _socket.socket) + s = socket.socket() + self.assertIsInstance(s, socket.SocketType) + s.close() + + def test_repr(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with s: + self.assertIn('fd=%i' % s.fileno(), repr(s)) + self.assertIn('family=%s' % socket.AF_INET, repr(s)) + self.assertIn('type=%s' % socket.SOCK_STREAM, repr(s)) + self.assertIn('proto=0', repr(s)) + self.assertNotIn('raddr', repr(s)) + s.bind(('127.0.0.1', 0)) + self.assertIn('laddr', repr(s)) + self.assertIn(str(s.getsockname()), repr(s)) + self.assertIn('[closed]', repr(s)) + self.assertNotIn('laddr', repr(s)) + + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_csocket_repr(self): + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + try: + expected = ('' + % (s.fileno(), s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + finally: + s.close() + expected = ('' + % (s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + + def test_weakref(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + p = proxy(s) + self.assertEqual(p.fileno(), s.fileno()) + s.close() + s = None + try: + p.fileno() + except ReferenceError: + pass + else: + self.fail('Socket proxy still exists') + + def testSocketError(self): + # Testing socket module exceptions + msg = "Error raising socket exception (%s)." + with self.assertRaises(OSError, msg=msg % 'OSError'): + raise OSError + with self.assertRaises(OSError, msg=msg % 'socket.herror'): + raise socket.herror + with self.assertRaises(OSError, msg=msg % 'socket.gaierror'): + raise socket.gaierror + + def testSendtoErrors(self): + # Testing that sendto doesn't masks failures. See #10169. + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind(('', 0)) + sockname = s.getsockname() + # 2 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', sockname) + self.assertEqual(str(cm.exception), + "'str' does not support the buffer interface") + with self.assertRaises(TypeError) as cm: + s.sendto(5j, sockname) + self.assertEqual(str(cm.exception), + "'complex' does not support the buffer interface") + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None) + self.assertIn('not NoneType',str(cm.exception)) + # 3 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', 0, sockname) + self.assertEqual(str(cm.exception), + "'str' does not support the buffer interface") + with self.assertRaises(TypeError) as cm: + s.sendto(5j, 0, sockname) + self.assertEqual(str(cm.exception), + "'complex' does not support the buffer interface") + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, None) + self.assertIn('not NoneType', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 'bar', sockname) + self.assertIn('an integer is required', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None, None) + self.assertIn('an integer is required', str(cm.exception)) + # wrong number of args + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo') + self.assertIn('(1 given)', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, sockname, 4) + self.assertIn('(4 given)', str(cm.exception)) + + def testCrucialConstants(self): + # Testing for mission critical constants + socket.AF_INET + socket.SOCK_STREAM + socket.SOCK_DGRAM + socket.SOCK_RAW + socket.SOCK_RDM + socket.SOCK_SEQPACKET + socket.SOL_SOCKET + socket.SO_REUSEADDR + + def testHostnameRes(self): + # Testing hostname resolution mechanisms + hostname = socket.gethostname() + try: + ip = socket.gethostbyname(hostname) + except OSError: + # Probably name lookup wasn't set up right; skip this test + self.skipTest('name lookup failure') + self.assertTrue(ip.find('.') >= 0, "Error resolving host to ip.") + try: + hname, aliases, ipaddrs = socket.gethostbyaddr(ip) + except OSError: + # Probably a similar problem as above; skip this test + self.skipTest('name lookup failure') + all_host_names = [hostname, hname] + aliases + fqhn = socket.getfqdn(ip) + if not fqhn in all_host_names: + self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + + def test_host_resolution(self): + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + self.assertRaises(OSError, socket.gethostbyname, addr) + self.assertRaises(OSError, socket.gethostbyaddr, addr) + + for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: + self.assertEqual(socket.gethostbyname(addr), addr) + + # we don't test support.HOSTv6 because there's a chance it doesn't have + # a matching name entry (e.g. 'ip6-localhost') + for host in [support.HOST]: + self.assertIn(host, socket.gethostbyaddr(host)[2]) + + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") + @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") + def test_sethostname(self): + oldhn = socket.gethostname() + try: + socket.sethostname('new') + except OSError as e: + if e.errno == errno.EPERM: + self.skipTest("test should be run as root") + else: + raise + try: + # running test as root! + self.assertEqual(socket.gethostname(), 'new') + # Should work with bytes objects too + socket.sethostname(b'bar') + self.assertEqual(socket.gethostname(), 'bar') + finally: + socket.sethostname(oldhn) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInterfaceNameIndex(self): + interfaces = socket.if_nameindex() + for index, name in interfaces: + self.assertIsInstance(index, int) + self.assertIsInstance(name, str) + # interface indices are non-zero integers + self.assertGreater(index, 0) + _index = socket.if_nametoindex(name) + self.assertIsInstance(_index, int) + self.assertEqual(index, _index) + _name = socket.if_indextoname(index) + self.assertIsInstance(_name, str) + self.assertEqual(name, _name) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInvalidInterfaceNameIndex(self): + # test nonexistent interface index/name + self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') + # test with invalid values + self.assertRaises(TypeError, socket.if_nametoindex, 0) + self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') + def testRefCountGetNameInfo(self): + # Testing reference count for getnameinfo + try: + # On some versions, this loses a reference + orig = sys.getrefcount(__name__) + socket.getnameinfo(__name__,0) + except TypeError: + if sys.getrefcount(__name__) != orig: + self.fail("socket.getnameinfo loses a reference") + + def testInterpreterCrash(self): + # Making sure getnameinfo doesn't crash the interpreter + try: + # On some versions, this crashes the interpreter. + socket.getnameinfo(('x', 0, 0, 0), 0) + except OSError: + pass + + def testNtoH(self): + # This just checks that htons etc. are their own inverse, + # when looking at the lower 16 or 32 bits. + sizes = {socket.htonl: 32, socket.ntohl: 32, + socket.htons: 16, socket.ntohs: 16} + for func, size in sizes.items(): + mask = (1<") + + def test_unusable_closed_socketio(self): + with socket.socket() as sock: + fp = sock.makefile("rb", buffering=0) + self.assertTrue(fp.readable()) + self.assertFalse(fp.writable()) + self.assertFalse(fp.seekable()) + fp.close() + self.assertRaises(ValueError, fp.readable) + self.assertRaises(ValueError, fp.writable) + self.assertRaises(ValueError, fp.seekable) + + def test_pickle(self): + sock = socket.socket() + with sock: + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, sock, protocol) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + family = pickle.loads(pickle.dumps(socket.AF_INET, protocol)) + self.assertEqual(family, socket.AF_INET) + type = pickle.loads(pickle.dumps(socket.SOCK_STREAM, protocol)) + self.assertEqual(type, socket.SOCK_STREAM) + + def test_listen_backlog(self): + for backlog in 0, -1: + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.bind((HOST, 0)) + srv.listen(backlog) + srv.close() + + @support.cpython_only + def test_listen_backlog_overflow(self): + # Issue 15989 + import _testcapi + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.bind((HOST, 0)) + self.assertRaises(OverflowError, srv.listen, _testcapi.INT_MAX + 1) + srv.close() + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + def test_flowinfo(self): + self.assertRaises(OverflowError, socket.getnameinfo, + (support.HOSTv6, 0, 0xffffffff), 0) + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + self.assertRaises(OverflowError, s.bind, (support.HOSTv6, 0, -10)) + + def test_str_for_enums(self): + # Make sure that the AF_* and SOCK_* constants have enum-like string + # reprs. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + self.assertEqual(str(s.family), 'AddressFamily.AF_INET') + self.assertEqual(str(s.type), 'SocketKind.SOCK_STREAM') + + @unittest.skipIf(os.name == 'nt', 'Will not work on Windows') + def test_uknown_socket_family_repr(self): + # Test that when created with a family that's not one of the known + # AF_*/SOCK_* constants, socket.family just returns the number. + # + # To do this we fool socket.socket into believing it already has an + # open fd because on this path it doesn't actually verify the family and + # type and populates the socket object. + # + # On Windows this trick won't work, so the test is skipped. + fd, _ = tempfile.mkstemp() + with socket.socket(family=42424, type=13331, fileno=fd) as s: + self.assertEqual(s.family, 42424) + self.assertEqual(s.type, 13331) + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +class BasicCANTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_CAN + socket.PF_CAN + socket.CAN_RAW + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCMConstants(self): + socket.CAN_BCM + + # opcodes + socket.CAN_BCM_TX_SETUP # create (cyclic) transmission task + socket.CAN_BCM_TX_DELETE # remove (cyclic) transmission task + socket.CAN_BCM_TX_READ # read properties of (cyclic) transmission task + socket.CAN_BCM_TX_SEND # send one CAN frame + socket.CAN_BCM_RX_SETUP # create RX content filter subscription + socket.CAN_BCM_RX_DELETE # remove RX content filter subscription + socket.CAN_BCM_RX_READ # read properties of RX content filter subscription + socket.CAN_BCM_TX_STATUS # reply to TX_READ request + socket.CAN_BCM_TX_EXPIRED # notification on performed transmissions (count=0) + socket.CAN_BCM_RX_STATUS # reply to RX_READ request + socket.CAN_BCM_RX_TIMEOUT # cyclic message is absent + socket.CAN_BCM_RX_CHANGED # updated CAN frame (detected content change) + + def testCreateSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + pass + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testCreateBCMSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) as s: + pass + + def testBindAny(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.bind(('', )) + + def testTooLongInterfaceName(self): + # most systems limit IFNAMSIZ to 16, take 1024 to be sure + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + self.assertRaisesRegex(OSError, 'interface name too long', + s.bind, ('x' * 1024,)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_LOOPBACK"), + 'socket.CAN_RAW_LOOPBACK required for this test.') + def testLoopback(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + for loopback in (0, 1): + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK, + loopback) + self.assertEqual(loopback, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_FILTER"), + 'socket.CAN_RAW_FILTER required for this test.') + def testFilter(self): + can_id, can_mask = 0x200, 0x700 + can_filter = struct.pack("=II", can_id, can_mask) + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, can_filter) + self.assertEqual(can_filter, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, 8)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class CANTest(ThreadedCANSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedCANSocketTest.__init__(self, methodName=methodName) + + @classmethod + def build_can_frame(cls, can_id, data): + """Build a CAN frame.""" + can_dlc = len(data) + data = data.ljust(8, b'\x00') + return struct.pack(cls.can_frame_fmt, can_id, can_dlc, data) + + @classmethod + def dissect_can_frame(cls, frame): + """Dissect a CAN frame.""" + can_id, can_dlc, data = struct.unpack(cls.can_frame_fmt, frame) + return (can_id, can_dlc, data[:can_dlc]) + + def testSendFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + self.assertEqual(addr[0], self.interface) + self.assertEqual(addr[1], socket.AF_CAN) + + def _testSendFrame(self): + self.cf = self.build_can_frame(0x00, b'\x01\x02\x03\x04\x05') + self.cli.send(self.cf) + + def testSendMaxFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + + def _testSendMaxFrame(self): + self.cf = self.build_can_frame(0x00, b'\x07' * 8) + self.cli.send(self.cf) + + def testSendMultiFrames(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf1, cf) + + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf2, cf) + + def _testSendMultiFrames(self): + self.cf1 = self.build_can_frame(0x07, b'\x44\x33\x22\x11') + self.cli.send(self.cf1) + + self.cf2 = self.build_can_frame(0x12, b'\x99\x22\x33') + self.cli.send(self.cf2) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def _testBCM(self): + cf, addr = self.cli.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + can_id, can_dlc, data = self.dissect_can_frame(cf) + self.assertEqual(self.can_id, can_id) + self.assertEqual(self.data, data) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCM(self): + bcm = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) + self.addCleanup(bcm.close) + bcm.connect((self.interface,)) + self.can_id = 0x123 + self.data = bytes([0xc0, 0xff, 0xee]) + self.cf = self.build_can_frame(self.can_id, self.data) + opcode = socket.CAN_BCM_TX_SEND + flags = 0 + count = 0 + ival1_seconds = ival1_usec = ival2_seconds = ival2_usec = 0 + bcm_can_id = 0x0222 + nframes = 1 + assert len(self.cf) == 16 + header = struct.pack(self.bcm_cmd_msg_fmt, + opcode, + flags, + count, + ival1_seconds, + ival1_usec, + ival2_seconds, + ival2_usec, + bcm_can_id, + nframes, + ) + header_plus_frame = header + self.cf + bytes_sent = bcm.send(header_plus_frame) + self.assertEqual(bytes_sent, len(header_plus_frame)) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +class BasicRDSTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_RDS + socket.PF_RDS + + def testCreateSocket(self): + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + pass + + def testSocketBufferSize(self): + bufsize = 16384 + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, bufsize) + s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, bufsize) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class RDSTest(ThreadedRDSSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedRDSSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + super().setUp() + self.evt = threading.Event() + + def testSendAndRecv(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + self.assertEqual(self.cli_addr, addr) + + def _testSendAndRecv(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testPeek(self): + data, addr = self.serv.recvfrom(self.bufsize, socket.MSG_PEEK) + self.assertEqual(self.data, data) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testPeek(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + @requireAttrs(socket.socket, 'recvmsg') + def testSendAndRecvMsg(self): + data, ancdata, msg_flags, addr = self.serv.recvmsg(self.bufsize) + self.assertEqual(self.data, data) + + @requireAttrs(socket.socket, 'sendmsg') + def _testSendAndRecvMsg(self): + self.data = b'hello ' * 10 + self.cli.sendmsg([self.data], (), 0, (HOST, self.port)) + + def testSendAndRecvMulti(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data1, data) + + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data2, data) + + def _testSendAndRecvMulti(self): + self.data1 = b'bacon' + self.cli.sendto(self.data1, 0, (HOST, self.port)) + + self.data2 = b'egg' + self.cli.sendto(self.data2, 0, (HOST, self.port)) + + def testSelect(self): + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testSelect(self): + self.data = b'select' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testCongestion(self): + # wait until the sender is done + self.evt.wait() + + def _testCongestion(self): + # test the behavior in case of congestion + self.data = b'fill' + self.cli.setblocking(False) + try: + # try to lower the receiver's socket buffer size + self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384) + except OSError: + pass + with self.assertRaises(OSError) as cm: + try: + # fill the receiver's socket buffer + while True: + self.cli.sendto(self.data, 0, (HOST, self.port)) + finally: + # signal the receiver we're done + self.evt.set() + # sendto() should have failed with ENOBUFS + self.assertEqual(cm.exception.errno, errno.ENOBUFS) + # and we should have received a congestion notification through poll + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicTCPTest(SocketConnectedTest): + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecv(self): + # Testing large receive over TCP + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.serv_conn.send(MSG) + + def testOverFlowRecv(self): + # Testing receive in chunks over TCP + seg1 = self.cli_conn.recv(len(MSG) - 3) + seg2 = self.cli_conn.recv(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecv(self): + self.serv_conn.send(MSG) + + def testRecvFrom(self): + # Testing large recvfrom() over TCP + msg, addr = self.cli_conn.recvfrom(1024) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.serv_conn.send(MSG) + + def testOverFlowRecvFrom(self): + # Testing recvfrom() in chunks over TCP + seg1, addr = self.cli_conn.recvfrom(len(MSG)-3) + seg2, addr = self.cli_conn.recvfrom(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecvFrom(self): + self.serv_conn.send(MSG) + + def testSendAll(self): + # Testing sendall() with a 2048 byte string over TCP + msg = b'' + while 1: + read = self.cli_conn.recv(1024) + if not read: + break + msg += read + self.assertEqual(msg, b'f' * 2048) + + def _testSendAll(self): + big_chunk = b'f' * 2048 + self.serv_conn.sendall(big_chunk) + + def testFromFd(self): + # Testing fromfd() + fd = self.cli_conn.fileno() + sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + self.assertIsInstance(sock, socket.socket) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testFromFd(self): + self.serv_conn.send(MSG) + + def testDup(self): + # Testing dup() + sock = self.cli_conn.dup() + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDup(self): + self.serv_conn.send(MSG) + + def testShutdown(self): + # Testing shutdown() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + # wait for _testShutdown to finish: on OS X, when the server + # closes the connection the client also becomes disconnected, + # and the client's shutdown call will fail. (Issue #4397.) + self.done.wait() + + def _testShutdown(self): + self.serv_conn.send(MSG) + self.serv_conn.shutdown(2) + + testShutdown_overflow = support.cpython_only(testShutdown) + + @support.cpython_only + def _testShutdown_overflow(self): + import _testcapi + self.serv_conn.send(MSG) + # Issue 15989 + self.assertRaises(OverflowError, self.serv_conn.shutdown, + _testcapi.INT_MAX + 1) + self.assertRaises(OverflowError, self.serv_conn.shutdown, + 2 + (_testcapi.UINT_MAX + 1)) + self.serv_conn.shutdown(2) + + def testDetach(self): + # Testing detach() + fileno = self.cli_conn.fileno() + f = self.cli_conn.detach() + self.assertEqual(f, fileno) + # cli_conn cannot be used anymore... + self.assertTrue(self.cli_conn._closed) + self.assertRaises(OSError, self.cli_conn.recv, 1024) + self.cli_conn.close() + # ...but we can create another socket using the (still open) + # file descriptor + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=f) + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDetach(self): + self.serv_conn.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicUDPTest(ThreadedUDPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedUDPSocketTest.__init__(self, methodName=methodName) + + def testSendtoAndRecv(self): + # Testing sendto() and Recv() over UDP + msg = self.serv.recv(len(MSG)) + self.assertEqual(msg, MSG) + + def _testSendtoAndRecv(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFrom(self): + # Testing recvfrom() over UDP + msg, addr = self.serv.recvfrom(len(MSG)) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFromNegative(self): + # Negative lengths passed to recvfrom should give ValueError. + self.assertRaises(ValueError, self.serv.recvfrom, -1) + + def _testRecvFromNegative(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + +# Tests for the sendmsg()/recvmsg() interface. Where possible, the +# same test code is used with different families and types of socket +# (e.g. stream, datagram), and tests using recvmsg() are repeated +# using recvmsg_into(). +# +# The generic test classes such as SendmsgTests and +# RecvmsgGenericTests inherit from SendrecvmsgBase and expect to be +# supplied with sockets cli_sock and serv_sock representing the +# client's and the server's end of the connection respectively, and +# attributes cli_addr and serv_addr holding their (numeric where +# appropriate) addresses. +# +# The final concrete test classes combine these with subclasses of +# SocketTestBase which set up client and server sockets of a specific +# type, and with subclasses of SendrecvmsgBase such as +# SendrecvmsgDgramBase and SendrecvmsgConnectedBase which map these +# sockets to cli_sock and serv_sock and override the methods and +# attributes of SendrecvmsgBase to fill in destination addresses if +# needed when sending, check for specific flags in msg_flags, etc. +# +# RecvmsgIntoMixin provides a version of doRecvmsg() implemented using +# recvmsg_into(). + +# XXX: like the other datagram (UDP) tests in this module, the code +# here assumes that datagram delivery on the local machine will be +# reliable. + +class SendrecvmsgBase(ThreadSafeCleanupTestCase): + # Base class for sendmsg()/recvmsg() tests. + + # Time in seconds to wait before considering a test failed, or + # None for no timeout. Not all tests actually set a timeout. + fail_timeout = 3.0 + + def setUp(self): + self.misc_event = threading.Event() + super().setUp() + + def sendToServer(self, msg): + # Send msg to the server. + return self.cli_sock.send(msg) + + # Tuple of alternative default arguments for sendmsg() when called + # via sendmsgToServer() (e.g. to include a destination address). + sendmsg_to_server_defaults = () + + def sendmsgToServer(self, *args): + # Call sendmsg() on self.cli_sock with the given arguments, + # filling in any arguments which are not supplied with the + # corresponding items of self.sendmsg_to_server_defaults, if + # any. + return self.cli_sock.sendmsg( + *(args + self.sendmsg_to_server_defaults[len(args):])) + + def doRecvmsg(self, sock, bufsize, *args): + # Call recvmsg() on sock with given arguments and return its + # result. Should be used for tests which can use either + # recvmsg() or recvmsg_into() - RecvmsgIntoMixin overrides + # this method with one which emulates it using recvmsg_into(), + # thus allowing the same test to be used for both methods. + result = sock.recvmsg(bufsize, *args) + self.registerRecvmsgResult(result) + return result + + def registerRecvmsgResult(self, result): + # Called by doRecvmsg() with the return value of recvmsg() or + # recvmsg_into(). Can be overridden to arrange cleanup based + # on the returned ancillary data, for instance. + pass + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer. + self.assertEqual(addr1, addr2) + + # Flags that are normally unset in msg_flags + msg_flags_common_unset = 0 + for name in ("MSG_CTRUNC", "MSG_OOB"): + msg_flags_common_unset |= getattr(socket, name, 0) + + # Flags that are normally set + msg_flags_common_set = 0 + + # Flags set when a complete record has been received (e.g. MSG_EOR + # for SCTP) + msg_flags_eor_indicator = 0 + + # Flags set when a complete record has not been received + # (e.g. MSG_TRUNC for datagram sockets) + msg_flags_non_eor_indicator = 0 + + def checkFlags(self, flags, eor=None, checkset=0, checkunset=0, ignore=0): + # Method to check the value of msg_flags returned by recvmsg[_into](). + # + # Checks that all bits in msg_flags_common_set attribute are + # set in "flags" and all bits in msg_flags_common_unset are + # unset. + # + # The "eor" argument specifies whether the flags should + # indicate that a full record (or datagram) has been received. + # If "eor" is None, no checks are done; otherwise, checks + # that: + # + # * if "eor" is true, all bits in msg_flags_eor_indicator are + # set and all bits in msg_flags_non_eor_indicator are unset + # + # * if "eor" is false, all bits in msg_flags_non_eor_indicator + # are set and all bits in msg_flags_eor_indicator are unset + # + # If "checkset" and/or "checkunset" are supplied, they require + # the given bits to be set or unset respectively, overriding + # what the attributes require for those bits. + # + # If any bits are set in "ignore", they will not be checked, + # regardless of the other inputs. + # + # Will raise Exception if the inputs require a bit to be both + # set and unset, and it is not ignored. + + defaultset = self.msg_flags_common_set + defaultunset = self.msg_flags_common_unset + + if eor: + defaultset |= self.msg_flags_eor_indicator + defaultunset |= self.msg_flags_non_eor_indicator + elif eor is not None: + defaultset |= self.msg_flags_non_eor_indicator + defaultunset |= self.msg_flags_eor_indicator + + # Function arguments override defaults + defaultset &= ~checkunset + defaultunset &= ~checkset + + # Merge arguments with remaining defaults, and check for conflicts + checkset |= defaultset + checkunset |= defaultunset + inboth = checkset & checkunset & ~ignore + if inboth: + raise Exception("contradictory set, unset requirements for flags " + "{0:#x}".format(inboth)) + + # Compare with given msg_flags value + mask = (checkset | checkunset) & ~ignore + self.assertEqual(flags & mask, checkset & mask) + + +class RecvmsgIntoMixin(SendrecvmsgBase): + # Mixin to implement doRecvmsg() using recvmsg_into(). + + def doRecvmsg(self, sock, bufsize, *args): + buf = bytearray(bufsize) + result = sock.recvmsg_into([buf], *args) + self.registerRecvmsgResult(result) + self.assertGreaterEqual(result[0], 0) + self.assertLessEqual(result[0], bufsize) + return (bytes(buf[:result[0]]),) + result[1:] + + +class SendrecvmsgDgramFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for datagram sockets. + + @property + def msg_flags_non_eor_indicator(self): + return super().msg_flags_non_eor_indicator | socket.MSG_TRUNC + + +class SendrecvmsgSCTPFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for SCTP sockets. + + @property + def msg_flags_eor_indicator(self): + return super().msg_flags_eor_indicator | socket.MSG_EOR + + +class SendrecvmsgConnectionlessBase(SendrecvmsgBase): + # Base class for tests on connectionless-mode sockets. Users must + # supply sockets on attributes cli and serv to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.serv + + @property + def cli_sock(self): + return self.cli + + @property + def sendmsg_to_server_defaults(self): + return ([], [], 0, self.serv_addr) + + def sendToServer(self, msg): + return self.cli_sock.sendto(msg, self.serv_addr) + + +class SendrecvmsgConnectedBase(SendrecvmsgBase): + # Base class for tests on connected sockets. Users must supply + # sockets on attributes serv_conn and cli_conn (representing the + # connections *to* the server and the client), to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.cli_conn + + @property + def cli_sock(self): + return self.serv_conn + + def checkRecvmsgAddress(self, addr1, addr2): + # Address is currently "unspecified" for a connected socket, + # so we don't examine it + pass + + +class SendrecvmsgServerTimeoutBase(SendrecvmsgBase): + # Base class to set a timeout on server's socket. + + def setUp(self): + super().setUp() + self.serv_sock.settimeout(self.fail_timeout) + + +class SendmsgTests(SendrecvmsgServerTimeoutBase): + # Tests for sendmsg() which can use any socket type and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsg(self): + # Send a simple message with sendmsg(). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG]), len(MSG)) + + def testSendmsgDataGenerator(self): + # Send from buffer obtained from a generator (not a sequence). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgDataGenerator(self): + self.assertEqual(self.sendmsgToServer((o for o in [MSG])), + len(MSG)) + + def testSendmsgAncillaryGenerator(self): + # Gather (empty) ancillary data from a generator. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgAncillaryGenerator(self): + self.assertEqual(self.sendmsgToServer([MSG], (o for o in [])), + len(MSG)) + + def testSendmsgArray(self): + # Send data from an array instead of the usual bytes object. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgArray(self): + self.assertEqual(self.sendmsgToServer([array.array("B", MSG)]), + len(MSG)) + + def testSendmsgGather(self): + # Send message data from more than one buffer (gather write). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgGather(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + def testSendmsgBadArgs(self): + # Check that sendmsg() rejects invalid arguments. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadArgs(self): + self.assertRaises(TypeError, self.cli_sock.sendmsg) + self.assertRaises(TypeError, self.sendmsgToServer, + b"not in an iterable") + self.assertRaises(TypeError, self.sendmsgToServer, + object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG, object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], 0, object()) + self.sendToServer(b"done") + + def testSendmsgBadCmsg(self): + # Check that invalid ancillary data items are rejected. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(object(), 0, b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, object(), b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, object())]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0)]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b"data", 42)]) + self.sendToServer(b"done") + + @requireAttrs(socket, "CMSG_SPACE") + def testSendmsgBadMultiCmsg(self): + # Check that invalid ancillary data items are rejected when + # more than one item is present. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + @testSendmsgBadMultiCmsg.client_skip + def _testSendmsgBadMultiCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [0, 0, b""]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b""), object()]) + self.sendToServer(b"done") + + def testSendmsgExcessCmsgReject(self): + # Check that sendmsg() rejects excess ancillary data items + # when the number that can be sent is limited. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgExcessCmsgReject(self): + if not hasattr(socket, "CMSG_SPACE"): + # Can only send one item + with self.assertRaises(OSError) as cm: + self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")]) + self.assertIsNone(cm.exception.errno) + self.sendToServer(b"done") + + def testSendmsgAfterClose(self): + # Check that sendmsg() fails on a closed socket. + pass + + def _testSendmsgAfterClose(self): + self.cli_sock.close() + self.assertRaises(OSError, self.sendmsgToServer, [MSG]) + + +class SendmsgStreamTests(SendmsgTests): + # Tests for sendmsg() which require a stream socket and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsgExplicitNoneAddr(self): + # Check that peer address can be specified as None. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgExplicitNoneAddr(self): + self.assertEqual(self.sendmsgToServer([MSG], [], 0, None), len(MSG)) + + def testSendmsgTimeout(self): + # Check that timeout works with sendmsg(). + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + def _testSendmsgTimeout(self): + try: + self.cli_sock.settimeout(0.03) + with self.assertRaises(socket.timeout): + while True: + self.sendmsgToServer([b"a"*512]) + finally: + self.misc_event.set() + + # XXX: would be nice to have more tests for sendmsg flags argument. + + # Linux supports MSG_DONTWAIT when sending, but in general, it + # only works when receiving. Could add other platforms if they + # support it too. + @skipWithClientIf(sys.platform not in {"linux"}, + "MSG_DONTWAIT not known to work on this platform when " + "sending") + def testSendmsgDontWait(self): + # Check that MSG_DONTWAIT in flags causes non-blocking behaviour. + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @testSendmsgDontWait.client_skip + def _testSendmsgDontWait(self): + try: + with self.assertRaises(OSError) as cm: + while True: + self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT) + self.assertIn(cm.exception.errno, + (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + self.misc_event.set() + + +class SendmsgConnectionlessTests(SendmsgTests): + # Tests for sendmsg() which require a connectionless-mode + # (e.g. datagram) socket, and do not involve recvmsg() or + # recvmsg_into(). + + def testSendmsgNoDestAddr(self): + # Check that sendmsg() fails when no destination address is + # given for unconnected socket. + pass + + def _testSendmsgNoDestAddr(self): + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG]) + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG], [], 0, None) + + +class RecvmsgGenericTests(SendrecvmsgBase): + # Tests for recvmsg() which can also be emulated using + # recvmsg_into(), and can use any socket type. + + def testRecvmsg(self): + # Receive a simple message with recvmsg[_into](). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsg(self): + self.sendToServer(MSG) + + def testRecvmsgExplicitDefaults(self): + # Test recvmsg[_into]() with default arguments provided explicitly. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgExplicitDefaults(self): + self.sendToServer(MSG) + + def testRecvmsgShorter(self): + # Receive a message smaller than buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) + 42) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShorter(self): + self.sendToServer(MSG) + + # FreeBSD < 8 doesn't always set the MSG_TRUNC flag when a truncated + # datagram is received (issue #13001). + @support.requires_freebsd_version(8) + def testRecvmsgTrunc(self): + # Receive part of message, check for truncation indicators. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + @support.requires_freebsd_version(8) + def _testRecvmsgTrunc(self): + self.sendToServer(MSG) + + def testRecvmsgShortAncillaryBuf(self): + # Test ancillary data buffer too small to hold any ancillary data. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 1) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShortAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgLongAncillaryBuf(self): + # Test large ancillary data buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgLongAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgAfterClose(self): + # Check that recvmsg[_into]() fails on a closed socket. + self.serv_sock.close() + self.assertRaises(OSError, self.doRecvmsg, self.serv_sock, 1024) + + def _testRecvmsgAfterClose(self): + pass + + def testRecvmsgTimeout(self): + # Check that timeout works. + try: + self.serv_sock.settimeout(0.03) + self.assertRaises(socket.timeout, + self.doRecvmsg, self.serv_sock, len(MSG)) + finally: + self.misc_event.set() + + def _testRecvmsgTimeout(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @requireAttrs(socket, "MSG_PEEK") + def testRecvmsgPeek(self): + # Check that MSG_PEEK in flags enables examination of pending + # data without consuming it. + + # Receive part of data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3, 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + # Ignoring MSG_TRUNC here (so this test is the same for stream + # and datagram sockets). Some wording in POSIX seems to + # suggest that it needn't be set when peeking, but that may + # just be a slip. + self.checkFlags(flags, eor=False, + ignore=getattr(socket, "MSG_TRUNC", 0)) + + # Receive all data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + # Check that the same data can still be received normally. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgPeek.client_skip + def _testRecvmsgPeek(self): + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + def testRecvmsgFromSendmsg(self): + # Test receiving with recvmsg[_into]() when message is sent + # using sendmsg(). + self.serv_sock.settimeout(self.fail_timeout) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgFromSendmsg.client_skip + def _testRecvmsgFromSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + +class RecvmsgGenericStreamTests(RecvmsgGenericTests): + # Tests which require a stream socket and can use either recvmsg() + # or recvmsg_into(). + + def testRecvmsgEOF(self): + # Receive end-of-stream indicator (b"", peer socket closed). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.assertEqual(msg, b"") + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=None) # Might not have end-of-record marker + + def _testRecvmsgEOF(self): + self.cli_sock.close() + + def testRecvmsgOverflow(self): + # Receive a message in more than one chunk. + seg1, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + seg2, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testRecvmsgOverflow(self): + self.sendToServer(MSG) + + +class RecvmsgTests(RecvmsgGenericTests): + # Tests for recvmsg() which can use any socket type. + + def testRecvmsgBadArgs(self): + # Check that recvmsg() rejects invalid arguments. + self.assertRaises(TypeError, self.serv_sock.recvmsg) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + -1, 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + len(MSG), -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + [bytearray(10)], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + object(), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), 0, object()) + + msg, ancdata, flags, addr = self.serv_sock.recvmsg(len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgBadArgs(self): + self.sendToServer(MSG) + + +class RecvmsgIntoTests(RecvmsgIntoMixin, RecvmsgGenericTests): + # Tests for recvmsg_into() which can use any socket type. + + def testRecvmsgIntoBadArgs(self): + # Check that recvmsg_into() rejects invalid arguments. + buf = bytearray(len(MSG)) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + len(MSG), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + buf, 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [object()], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [b"I'm not writable"], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf, object()], 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg_into, + [buf], -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], 0, object()) + + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf], 0, 0) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoBadArgs(self): + self.sendToServer(MSG) + + def testRecvmsgIntoGenerator(self): + # Receive into buffer obtained from a generator (not a sequence). + buf = bytearray(len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + (o for o in [buf])) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoGenerator(self): + self.sendToServer(MSG) + + def testRecvmsgIntoArray(self): + # Receive into an array rather than the usual bytearray. + buf = array.array("B", [0] * len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf]) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf.tobytes(), MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoArray(self): + self.sendToServer(MSG) + + def testRecvmsgIntoScatter(self): + # Receive into multiple buffers (scatter write). + b1 = bytearray(b"----") + b2 = bytearray(b"0123456789") + b3 = bytearray(b"--------------") + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + [b1, memoryview(b2)[2:9], b3]) + self.assertEqual(nbytes, len(b"Mary had a little lamb")) + self.assertEqual(b1, bytearray(b"Mary")) + self.assertEqual(b2, bytearray(b"01 had a 9")) + self.assertEqual(b3, bytearray(b"little lamb---")) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoScatter(self): + self.sendToServer(b"Mary had a little lamb") + + +class CmsgMacroTests(unittest.TestCase): + # Test the functions CMSG_LEN() and CMSG_SPACE(). Tests + # assumptions used by sendmsg() and recvmsg[_into](), which share + # code with these functions. + + # Match the definition in socketmodule.c + try: + import _testcapi + except ImportError: + socklen_t_limit = 0x7fffffff + else: + socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) + + @requireAttrs(socket, "CMSG_LEN") + def testCMSG_LEN(self): + # Test CMSG_LEN() with various valid and invalid values, + # checking the assumptions used by recvmsg() and sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_LEN(0) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(socket.CMSG_LEN(0), array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_LEN(n) + # This is how recvmsg() calculates the data size + self.assertEqual(ret - socket.CMSG_LEN(0), n) + self.assertLessEqual(ret, self.socklen_t_limit) + + self.assertRaises(OverflowError, socket.CMSG_LEN, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_LEN, toobig) + self.assertRaises(OverflowError, socket.CMSG_LEN, sys.maxsize) + + @requireAttrs(socket, "CMSG_SPACE") + def testCMSG_SPACE(self): + # Test CMSG_SPACE() with various valid and invalid values, + # checking the assumptions used by sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + last = socket.CMSG_SPACE(0) + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(last, array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_SPACE(n) + self.assertGreaterEqual(ret, last) + self.assertGreaterEqual(ret, socket.CMSG_LEN(n)) + self.assertGreaterEqual(ret, n + socket.CMSG_LEN(0)) + self.assertLessEqual(ret, self.socklen_t_limit) + last = ret + + self.assertRaises(OverflowError, socket.CMSG_SPACE, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_SPACE, toobig) + self.assertRaises(OverflowError, socket.CMSG_SPACE, sys.maxsize) + + +class SCMRightsTest(SendrecvmsgServerTimeoutBase): + # Tests for file descriptor passing on Unix-domain sockets. + + # Invalid file descriptor value that's unlikely to evaluate to a + # real FD even if one of its bytes is replaced with a different + # value (which shouldn't actually happen). + badfd = -0x5555 + + def newFDs(self, n): + # Return a list of n file descriptors for newly-created files + # containing their list indices as ASCII numbers. + fds = [] + for i in range(n): + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + self.addCleanup(os.close, fd) + os.write(fd, str(i).encode()) + fds.append(fd) + return fds + + def checkFDs(self, fds): + # Check that the file descriptors in the given list contain + # their correct list indices as ASCII numbers. + for n, fd in enumerate(fds): + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(os.read(fd, 1024), str(n).encode()) + + def registerRecvmsgResult(self, result): + self.addCleanup(self.closeRecvmsgFDs, result) + + def closeRecvmsgFDs(self, recvmsg_result): + # Close all file descriptors specified in the ancillary data + # of the given return value from recvmsg() or recvmsg_into(). + for cmsg_level, cmsg_type, cmsg_data in recvmsg_result[1]: + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + for fd in fds: + os.close(fd) + + def createAndSendFDs(self, n): + # Send n new file descriptors created by newFDs() to the + # server, with the constant MSG as the non-ancillary data. + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(n)))]), + len(MSG)) + + def checkRecvmsgFDs(self, numfds, result, maxcmsgs=1, ignoreflags=0): + # Check that constant MSG was received with numfds file + # descriptors in a maximum of maxcmsgs control messages (which + # must contain only complete integers). By default, check + # that MSG_CTRUNC is unset, but ignore any flags in + # ignoreflags. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertIsInstance(ancdata, list) + self.assertLessEqual(len(ancdata), maxcmsgs) + fds = array.array("i") + for item in ancdata: + self.assertIsInstance(item, tuple) + cmsg_level, cmsg_type, cmsg_data = item + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data) % SIZEOF_INT, 0) + fds.frombytes(cmsg_data) + + self.assertEqual(len(fds), numfds) + self.checkFDs(fds) + + def testFDPassSimple(self): + # Pass a single FD (array read from bytes object). + self.checkRecvmsgFDs(1, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testFDPassSimple(self): + self.assertEqual( + self.sendmsgToServer( + [MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(1)).tobytes())]), + len(MSG)) + + def testMultipleFDPass(self): + # Pass multiple FDs in a single array. + self.checkRecvmsgFDs(4, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testMultipleFDPass(self): + self.createAndSendFDs(4) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassCMSG_SPACE(self): + # Test using CMSG_SPACE() to calculate ancillary buffer size. + self.checkRecvmsgFDs( + 4, self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(4 * SIZEOF_INT))) + + @testFDPassCMSG_SPACE.client_skip + def _testFDPassCMSG_SPACE(self): + self.createAndSendFDs(4) + + def testFDPassCMSG_LEN(self): + # Test using CMSG_LEN() to calculate ancillary buffer size. + self.checkRecvmsgFDs(1, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(4 * SIZEOF_INT)), + # RFC 3542 says implementations may set + # MSG_CTRUNC if there isn't enough space + # for trailing padding. + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassCMSG_LEN(self): + self.createAndSendFDs(1) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparate(self): + # Pass two FDs in two separate arrays. Arrays may be combined + # into a single control message by the OS. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), 10240), + maxcmsgs=2) + + @testFDPassSeparate.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparate(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparateMinSpace(self): + # Pass two FDs in two separate arrays, receiving them into the + # minimum space for two arrays. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(SIZEOF_INT)), + maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) + + @testFDPassSeparateMinSpace.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparateMinSpace(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + def sendAncillaryIfPossible(self, msg, ancdata): + # Try to send msg and ancdata to server, but if the system + # call fails, just send msg with no ancillary data. + try: + nbytes = self.sendmsgToServer([msg], ancdata) + except OSError as e: + # Check that it was the system call that failed + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer([msg]) + self.assertEqual(nbytes, len(msg)) + + def testFDPassEmpty(self): + # Try to pass an empty FD array. Can receive either no array + # or an empty array. + self.checkRecvmsgFDs(0, self.doRecvmsg(self.serv_sock, + len(MSG), 10240), + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassEmpty(self): + self.sendAncillaryIfPossible(MSG, [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + b"")]) + + def testFDPassPartialInt(self): + # Try to pass a truncated FD array. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 1) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + def _testFDPassPartialInt(self): + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [self.badfd]).tobytes()[:-1])]) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassPartialIntInMiddle(self): + # Try to pass two FD arrays, the first of which is truncated. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 2) + fds = array.array("i") + # Arrays may have been combined in a single control message + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.assertLessEqual(len(fds), 2) + self.checkFDs(fds) + + @testFDPassPartialIntInMiddle.client_skip + def _testFDPassPartialIntInMiddle(self): + fd0, fd1 = self.newFDs(2) + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0, self.badfd]).tobytes()[:-1]), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]) + + def checkTruncatedHeader(self, result, ignoreflags=0): + # Check that no ancillary data items are returned when data is + # truncated inside the cmsghdr structure. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no buffer size + # is specified. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG)), + # BSD seems to set MSG_CTRUNC only + # if an item has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTruncNoBufSize(self): + self.createAndSendFDs(1) + + def testCmsgTrunc0(self): + # Check that no ancillary data is received when buffer size is 0. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTrunc0(self): + self.createAndSendFDs(1) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + def testCmsgTrunc1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + + def _testCmsgTrunc1(self): + self.createAndSendFDs(1) + + def testCmsgTrunc2Int(self): + # The cmsghdr structure has at least three members, two of + # which are ints, so we still shouldn't see any ancillary + # data. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + SIZEOF_INT * 2)) + + def _testCmsgTrunc2Int(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Minus1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(0) - 1)) + + def _testCmsgTruncLen0Minus1(self): + self.createAndSendFDs(1) + + # The following tests try to truncate the control message in the + # middle of the FD array. + + def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): + # Check that file descriptor data is truncated to between + # mindata and maxdata bytes when received with buffer size + # ancbuf, and that any complete file descriptor numbers are + # valid. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + if mindata == 0 and ancdata == []: + return + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertGreaterEqual(len(cmsg_data), mindata) + self.assertLessEqual(len(cmsg_data), maxdata) + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.checkFDs(fds) + + def testCmsgTruncLen0(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + + def _testCmsgTruncLen0(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Plus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + + def _testCmsgTruncLen0Plus1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), + maxdata=SIZEOF_INT) + + def _testCmsgTruncLen1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen2Minus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, + maxdata=(2 * SIZEOF_INT) - 1) + + def _testCmsgTruncLen2Minus1(self): + self.createAndSendFDs(2) + + +class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase): + # Test sendmsg() and recvmsg[_into]() using the ancillary data + # features of the RFC 3542 Advanced Sockets API for IPv6. + # Currently we can only handle certain data items (e.g. traffic + # class, hop limit, MTU discovery and fragmentation settings) + # without resorting to unportable means such as the struct module, + # but the tests here are aimed at testing the ancillary data + # handling in sendmsg() and recvmsg() rather than the IPv6 API + # itself. + + # Test value to use when setting hop limit of packet + hop_limit = 2 + + # Test value to use when setting traffic class of packet. + # -1 means "use kernel default". + traffic_class = -1 + + def ancillaryMapping(self, ancdata): + # Given ancillary data list ancdata, return a mapping from + # pairs (cmsg_level, cmsg_type) to corresponding cmsg_data. + # Check that no (level, type) pair appears more than once. + d = {} + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertNotIn((cmsg_level, cmsg_type), d) + d[(cmsg_level, cmsg_type)] = cmsg_data + return d + + def checkHopLimit(self, ancbufsize, maxhop=255, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space. Check that data is MSG, ancillary data is not + # truncated (but ignore any flags in ignoreflags), and hop + # limit is between 0 and maxhop inclusive. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + self.assertIsInstance(ancdata[0], tuple) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimit(self): + # Test receiving the packet hop limit as ancillary data. + self.checkHopLimit(ancbufsize=10240) + + @testRecvHopLimit.client_skip + def _testRecvHopLimit(self): + # Need to wait until server has asked to receive ancillary + # data, as implementations are not required to buffer it + # otherwise. + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimitCMSG_SPACE(self): + # Test receiving hop limit, using CMSG_SPACE to calculate buffer size. + self.checkHopLimit(ancbufsize=socket.CMSG_SPACE(SIZEOF_INT)) + + @testRecvHopLimitCMSG_SPACE.client_skip + def _testRecvHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Could test receiving into buffer sized using CMSG_LEN, but RFC + # 3542 says portable applications must provide space for trailing + # padding. Implementations may set MSG_CTRUNC if there isn't + # enough space for the padding. + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSetHopLimit(self): + # Test setting hop limit on outgoing packet and receiving it + # at the other end. + self.checkHopLimit(ancbufsize=10240, maxhop=self.hop_limit) + + @testSetHopLimit.client_skip + def _testSetHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + def checkTrafficClassAndHopLimit(self, ancbufsize, maxhop=255, + ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space. Check that data is MSG, ancillary + # data is not truncated (but ignore any flags in ignoreflags), + # and traffic class and hop limit are in range (hop limit no + # more than maxhop). + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + self.assertEqual(len(ancdata), 2) + ancmap = self.ancillaryMapping(ancdata) + + tcdata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_TCLASS)] + self.assertEqual(len(tcdata), SIZEOF_INT) + a = array.array("i") + a.frombytes(tcdata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + hldata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT)] + self.assertEqual(len(hldata), SIZEOF_INT) + a = array.array("i") + a.frombytes(hldata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimit(self): + # Test receiving traffic class and hop limit as ancillary data. + self.checkTrafficClassAndHopLimit(ancbufsize=10240) + + @testRecvTrafficClassAndHopLimit.client_skip + def _testRecvTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + # Test receiving traffic class and hop limit, using + # CMSG_SPACE() to calculate buffer size. + self.checkTrafficClassAndHopLimit( + ancbufsize=socket.CMSG_SPACE(SIZEOF_INT) * 2) + + @testRecvTrafficClassAndHopLimitCMSG_SPACE.client_skip + def _testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSetTrafficClassAndHopLimit(self): + # Test setting traffic class and hop limit on outgoing packet, + # and receiving them at the other end. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testSetTrafficClassAndHopLimit.client_skip + def _testSetTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testOddCmsgSize(self): + # Try to send ancillary data with first item one byte too + # long. Fall back to sending with correct size if this fails, + # and check that second item was handled correctly. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testOddCmsgSize.client_skip + def _testOddCmsgSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + try: + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class]).tobytes() + b"\x00"), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + except OSError as e: + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + self.assertEqual(nbytes, len(MSG)) + + # Tests for proper handling of truncated ancillary data + + def checkHopLimitTruncatedHeader(self, ancbufsize, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space, which should be too small to contain the ancillary + # data header (if ancbufsize is None, pass no second argument + # to recvmsg()). Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and no ancillary data is + # returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + args = () if ancbufsize is None else (ancbufsize,) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), *args) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no ancillary + # buffer size is provided. + self.checkHopLimitTruncatedHeader(ancbufsize=None, + # BSD seems to set + # MSG_CTRUNC only if an item + # has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + @testCmsgTruncNoBufSize.client_skip + def _testCmsgTruncNoBufSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc0(self): + # Check that no ancillary data is received when ancillary + # buffer size is zero. + self.checkHopLimitTruncatedHeader(ancbufsize=0, + ignoreflags=socket.MSG_CTRUNC) + + @testSingleCmsgTrunc0.client_skip + def _testSingleCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=1) + + @testSingleCmsgTrunc1.client_skip + def _testSingleCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc2Int(self): + self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) + + @testSingleCmsgTrunc2Int.client_skip + def _testSingleCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncLen0Minus1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) + + @testSingleCmsgTruncLen0Minus1.client_skip + def _testSingleCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncInData(self): + # Test truncation of a control message inside its associated + # data. The message may be returned with its data truncated, + # or not returned at all. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + self.assertLessEqual(len(ancdata), 1) + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + @testSingleCmsgTruncInData.client_skip + def _testSingleCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + def checkTruncatedSecondHeader(self, ancbufsize, ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space, which should be large enough to + # contain the first item, but too small to contain the header + # of the second. Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and only one ancillary + # data item is returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertIn(cmsg_type, {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT}) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + # Try the above test with various buffer sizes. + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc0(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT), + ignoreflags=socket.MSG_CTRUNC) + + @testSecondCmsgTrunc0.client_skip + def _testSecondCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) + + @testSecondCmsgTrunc1.client_skip + def _testSecondCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc2Int(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + 2 * SIZEOF_INT) + + @testSecondCmsgTrunc2Int.client_skip + def _testSecondCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTruncLen0Minus1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(0) - 1) + + @testSecondCmsgTruncLen0Minus1.client_skip + def _testSecondCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecomdCmsgTruncInData(self): + # Test truncation of the second of two control messages inside + # its associated data. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + cmsg_types = {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT} + + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + self.assertEqual(ancdata, []) + + @testSecomdCmsgTruncInData.client_skip + def _testSecomdCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + +# Derive concrete test classes for different socket types. + +class SendrecvmsgUDPTestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDPTest(SendmsgConnectionlessTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDPTest(RecvmsgTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDPTest(RecvmsgIntoTests, SendrecvmsgUDPTestBase): + pass + + +class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDP6TestBase): + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer, ignoring scope ID + self.assertEqual(addr1[:-1], addr2[:-1]) + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDP6Test(SendmsgConnectionlessTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDP6Test(RecvmsgTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDP6Test(RecvmsgIntoTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgRFC3542AncillaryUDP6Test(RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoRFC3542AncillaryUDP6Test(RecvmsgIntoMixin, + RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + + +class SendrecvmsgTCPTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, TCPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgTCPTest(SendmsgStreamTests, SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgTCPTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoTCPTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + + +class SendrecvmsgSCTPStreamTestBase(SendrecvmsgSCTPFlagsBase, + SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, SCTPStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgSCTPStreamTest(SendmsgStreamTests, SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCTPStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + +@requireAttrs(socket.socket, "recvmsg_into") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCTPStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgIntoSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + + +class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, UnixStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUnixStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUnixStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCMRightsStreamTest(SCMRightsTest, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg_into") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCMRightsStreamTest(RecvmsgIntoMixin, SCMRightsTest, + SendrecvmsgUnixStreamTestBase): + pass + + +# Test interrupting the interruptible send/receive methods with a +# signal when a timeout is set. These tests avoid having multiple +# threads alive during the test so that the OS cannot deliver the +# signal to the wrong one. + +class InterruptedTimeoutBase(unittest.TestCase): + # Base class for interrupted send/receive tests. Installs an + # empty handler for SIGALRM and removes it on teardown, along with + # any scheduled alarms. + + def setUp(self): + super().setUp() + orig_alrm_handler = signal.signal(signal.SIGALRM, + lambda signum, frame: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(self.setAlarm, 0) + + # Timeout for socket operations + timeout = 4.0 + + # Provide setAlarm() method to schedule delivery of SIGALRM after + # given number of seconds, or cancel it if zero, and an + # appropriate time value to use. Use setitimer() if available. + if hasattr(signal, "setitimer"): + alarm_time = 0.05 + + def setAlarm(self, seconds): + signal.setitimer(signal.ITIMER_REAL, seconds) + else: + # Old systems may deliver the alarm up to one second early + alarm_time = 2 + + def setAlarm(self, seconds): + signal.alarm(seconds) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase): + # Test interrupting the recv*() methods with signals when a + # timeout is set. + + def setUp(self): + super().setUp() + self.serv.settimeout(self.timeout) + + def checkInterruptedRecv(self, func, *args, **kwargs): + # Check that func(*args, **kwargs) raises OSError with an + # errno of EINTR when interrupted by a signal. + self.setAlarm(self.alarm_time) + with self.assertRaises(OSError) as cm: + func(*args, **kwargs) + self.assertNotIsInstance(cm.exception, socket.timeout) + self.assertEqual(cm.exception.errno, errno.EINTR) + + def testInterruptedRecvTimeout(self): + self.checkInterruptedRecv(self.serv.recv, 1024) + + def testInterruptedRecvIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recv_into, bytearray(1024)) + + def testInterruptedRecvfromTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom, 1024) + + def testInterruptedRecvfromIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom_into, bytearray(1024)) + + @requireAttrs(socket.socket, "recvmsg") + def testInterruptedRecvmsgTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg, 1024) + + @requireAttrs(socket.socket, "recvmsg_into") + def testInterruptedRecvmsgIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg_into, [bytearray(1024)]) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +@unittest.skipUnless(thread, 'Threading required for this test.') +class InterruptedSendTimeoutTest(InterruptedTimeoutBase, + ThreadSafeCleanupTestCase, + SocketListeningTestMixin, TCPTestBase): + # Test interrupting the interruptible send*() methods with signals + # when a timeout is set. + + def setUp(self): + super().setUp() + self.serv_conn = self.newSocket() + self.addCleanup(self.serv_conn.close) + # Use a thread to complete the connection, but wait for it to + # terminate before running the test, so that there is only one + # thread to accept the signal. + cli_thread = threading.Thread(target=self.doConnect) + cli_thread.start() + self.cli_conn, addr = self.serv.accept() + self.addCleanup(self.cli_conn.close) + cli_thread.join() + self.serv_conn.settimeout(self.timeout) + + def doConnect(self): + self.serv_conn.connect(self.serv_addr) + + def checkInterruptedSend(self, func, *args, **kwargs): + # Check that func(*args, **kwargs), run in a loop, raises + # OSError with an errno of EINTR when interrupted by a + # signal. + with self.assertRaises(OSError) as cm: + while True: + self.setAlarm(self.alarm_time) + func(*args, **kwargs) + self.assertNotIsInstance(cm.exception, socket.timeout) + self.assertEqual(cm.exception.errno, errno.EINTR) + + # Issue #12958: The following tests have problems on OS X prior to 10.7 + @support.requires_mac_ver(10, 7) + def testInterruptedSendTimeout(self): + self.checkInterruptedSend(self.serv_conn.send, b"a"*512) + + @support.requires_mac_ver(10, 7) + def testInterruptedSendtoTimeout(self): + # Passing an actual address here as Python's wrapper for + # sendto() doesn't allow passing a zero-length one; POSIX + # requires that the address is ignored since the socket is + # connection-mode, however. + self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512, + self.serv_addr) + + @support.requires_mac_ver(10, 7) + @requireAttrs(socket.socket, "sendmsg") + def testInterruptedSendmsgTimeout(self): + self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512]) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class TCPCloserTest(ThreadedTCPSocketTest): + + def testClose(self): + conn, addr = self.serv.accept() + conn.close() + + sd = self.cli + read, write, err = select.select([sd], [], [], 1.0) + self.assertEqual(read, [sd]) + self.assertEqual(sd.recv(1), b'') + + # Calling close() many times should be safe. + conn.close() + conn.close() + + def _testClose(self): + self.cli.connect((HOST, self.port)) + time.sleep(1.0) + +@unittest.skipUnless(hasattr(socket, 'socketpair'), + 'test needs socket.socketpair()') +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicSocketPairTest(SocketPairTest): + + def __init__(self, methodName='runTest'): + SocketPairTest.__init__(self, methodName=methodName) + + def _check_defaults(self, sock): + self.assertIsInstance(sock, socket.socket) + if hasattr(socket, 'AF_UNIX'): + self.assertEqual(sock.family, socket.AF_UNIX) + else: + self.assertEqual(sock.family, socket.AF_INET) + self.assertEqual(sock.type, socket.SOCK_STREAM) + self.assertEqual(sock.proto, 0) + + def _testDefaults(self): + self._check_defaults(self.cli) + + def testDefaults(self): + self._check_defaults(self.serv) + + def testRecv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.cli.send(MSG) + + def testSend(self): + self.serv.send(MSG) + + def _testSend(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NonBlockingTCPTests(ThreadedTCPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def testSetBlocking(self): + # Testing whether set blocking works + self.serv.setblocking(True) + self.assertIsNone(self.serv.gettimeout()) + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error setting non-blocking mode.") + + def _testSetBlocking(self): + pass + + @support.cpython_only + def testSetBlocking_overflow(self): + # Issue 15989 + import _testcapi + if _testcapi.UINT_MAX >= _testcapi.ULONG_MAX: + self.skipTest('needs UINT_MAX < ULONG_MAX') + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.serv.setblocking(_testcapi.UINT_MAX + 1) + self.assertIsNone(self.serv.gettimeout()) + + _testSetBlocking_overflow = support.cpython_only(_testSetBlocking) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'test needs socket.SOCK_NONBLOCK') + @support.requires_linux_version(2, 6, 28) + def testInitNonBlocking(self): + # reinit server socket + self.serv.close() + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM | + socket.SOCK_NONBLOCK) + self.port = support.bind_port(self.serv) + self.serv.listen(1) + # actual testing + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error creating with non-blocking mode.") + + def _testInitNonBlocking(self): + pass + + def testInheritFlags(self): + # Issue #7995: when calling accept() on a listening socket with a + # timeout, the resulting socket should not be non-blocking. + self.serv.settimeout(10) + try: + conn, addr = self.serv.accept() + message = conn.recv(len(MSG)) + finally: + conn.close() + self.serv.settimeout(None) + + def _testInheritFlags(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + time.sleep(0.5) + self.cli.send(MSG) + + def testAccept(self): + # Testing non-blocking accept + self.serv.setblocking(0) + try: + conn, addr = self.serv.accept() + except OSError: + pass + else: + self.fail("Error trying to do non-blocking accept.") + read, write, err = select.select([self.serv], [], []) + if self.serv in read: + conn, addr = self.serv.accept() + self.assertIsNone(conn.gettimeout()) + conn.close() + else: + self.fail("Error trying to do accept after select.") + + def _testAccept(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + + def testConnect(self): + # Testing non-blocking connect + conn, addr = self.serv.accept() + conn.close() + + def _testConnect(self): + self.cli.settimeout(10) + self.cli.connect((HOST, self.port)) + + def testRecv(self): + # Testing non-blocking recv + conn, addr = self.serv.accept() + conn.setblocking(0) + try: + msg = conn.recv(len(MSG)) + except OSError: + pass + else: + self.fail("Error trying to do non-blocking recv.") + read, write, err = select.select([conn], [], []) + if conn in read: + msg = conn.recv(len(MSG)) + conn.close() + self.assertEqual(msg, MSG) + else: + self.fail("Error during select call to non-blocking socket.") + + def _testRecv(self): + self.cli.connect((HOST, self.port)) + time.sleep(0.1) + self.cli.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class FileObjectClassTestCase(SocketConnectedTest): + """Unit tests for the object returned by socket.makefile() + + self.read_file is the io object returned by makefile() on + the client connection. You can read from this file to + get output from the server. + + self.write_file is the io object returned by makefile() on the + server connection. You can write to this file to send output + to the client. + """ + + bufsize = -1 # Use default buffer size + encoding = 'utf-8' + errors = 'strict' + newline = None + + read_mode = 'rb' + read_msg = MSG + write_mode = 'wb' + write_msg = MSG + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def setUp(self): + self.evt1, self.evt2, self.serv_finished, self.cli_finished = [ + threading.Event() for i in range(4)] + SocketConnectedTest.setUp(self) + self.read_file = self.cli_conn.makefile( + self.read_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def tearDown(self): + self.serv_finished.set() + self.read_file.close() + self.assertTrue(self.read_file.closed) + self.read_file = None + SocketConnectedTest.tearDown(self) + + def clientSetUp(self): + SocketConnectedTest.clientSetUp(self) + self.write_file = self.serv_conn.makefile( + self.write_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def clientTearDown(self): + self.cli_finished.set() + self.write_file.close() + self.assertTrue(self.write_file.closed) + self.write_file = None + SocketConnectedTest.clientTearDown(self) + + def testReadAfterTimeout(self): + # Issue #7322: A file object must disallow further reads + # after a timeout has occurred. + self.cli_conn.settimeout(1) + self.read_file.read(3) + # First read raises a timeout + self.assertRaises(socket.timeout, self.read_file.read, 1) + # Second read is disallowed + with self.assertRaises(OSError) as ctx: + self.read_file.read(1) + self.assertIn("cannot read from timed out object", str(ctx.exception)) + + def _testReadAfterTimeout(self): + self.write_file.write(self.write_msg[0:3]) + self.write_file.flush() + self.serv_finished.wait() + + def testSmallRead(self): + # Performing small file read test + first_seg = self.read_file.read(len(self.read_msg)-3) + second_seg = self.read_file.read(3) + msg = first_seg + second_seg + self.assertEqual(msg, self.read_msg) + + def _testSmallRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testFullRead(self): + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testFullRead(self): + self.write_file.write(self.write_msg) + self.write_file.close() + + def testUnbufferedRead(self): + # Performing unbuffered file read test + buf = type(self.read_msg)() + while 1: + char = self.read_file.read(1) + if not char: + break + buf += char + self.assertEqual(buf, self.read_msg) + + def _testUnbufferedRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testReadline(self): + # Performing file readline test + line = self.read_file.readline() + self.assertEqual(line, self.read_msg) + + def _testReadline(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testCloseAfterMakefile(self): + # The file returned by makefile should keep the socket open. + self.cli_conn.close() + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testCloseAfterMakefile(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileAfterMakefileClose(self): + self.read_file.close() + msg = self.cli_conn.recv(len(MSG)) + if isinstance(self.read_msg, str): + msg = msg.decode() + self.assertEqual(msg, self.read_msg) + + def _testMakefileAfterMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testClosedAttr(self): + self.assertTrue(not self.read_file.closed) + + def _testClosedAttr(self): + self.assertTrue(not self.write_file.closed) + + def testAttributes(self): + self.assertEqual(self.read_file.mode, self.read_mode) + self.assertEqual(self.read_file.name, self.cli_conn.fileno()) + + def _testAttributes(self): + self.assertEqual(self.write_file.mode, self.write_mode) + self.assertEqual(self.write_file.name, self.serv_conn.fileno()) + + def testRealClose(self): + self.read_file.close() + self.assertRaises(ValueError, self.read_file.fileno) + self.cli_conn.close() + self.assertRaises(OSError, self.cli_conn.getsockname) + + def _testRealClose(self): + pass + + +class FileObjectInterruptedTestCase(unittest.TestCase): + """Test that the file object correctly handles EINTR internally.""" + + class MockSocket(object): + def __init__(self, recv_funcs=()): + # A generator that returns callables that we'll call for each + # call to recv(). + self._recv_step = iter(recv_funcs) + + def recv_into(self, buffer): + data = next(self._recv_step)() + assert len(buffer) >= len(data) + buffer[:len(data)] = data + return len(data) + + def _decref_socketios(self): + pass + + def _textiowrap_for_test(self, buffering=-1): + raw = socket.SocketIO(self, "r") + if buffering < 0: + buffering = io.DEFAULT_BUFFER_SIZE + if buffering == 0: + return raw + buffer = io.BufferedReader(raw, buffering) + text = io.TextIOWrapper(buffer, None, None) + text.mode = "rb" + return text + + @staticmethod + def _raise_eintr(): + raise OSError(errno.EINTR, "interrupted") + + def _textiowrap_mock_socket(self, mock, buffering=-1): + raw = socket.SocketIO(mock, "r") + if buffering < 0: + buffering = io.DEFAULT_BUFFER_SIZE + if buffering == 0: + return raw + buffer = io.BufferedReader(raw, buffering) + text = io.TextIOWrapper(buffer, None, None) + text.mode = "rb" + return text + + def _test_readline(self, size=-1, buffering=-1): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : b"This is the first line\nAnd the sec", + self._raise_eintr, + lambda : b"ond line is here\n", + lambda : b"", + lambda : b"", # XXX(gps): io library does an extra EOF read + ]) + fo = mock_sock._textiowrap_for_test(buffering=buffering) + self.assertEqual(fo.readline(size), "This is the first line\n") + self.assertEqual(fo.readline(size), "And the second line is here\n") + + def _test_read(self, size=-1, buffering=-1): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : b"This is the first line\nAnd the sec", + self._raise_eintr, + lambda : b"ond line is here\n", + lambda : b"", + lambda : b"", # XXX(gps): io library does an extra EOF read + ]) + expecting = (b"This is the first line\n" + b"And the second line is here\n") + fo = mock_sock._textiowrap_for_test(buffering=buffering) + if buffering == 0: + data = b'' + else: + data = '' + expecting = expecting.decode('utf-8') + while len(data) != len(expecting): + part = fo.read(size) + if not part: + break + data += part + self.assertEqual(data, expecting) + + def test_default(self): + self._test_readline() + self._test_readline(size=100) + self._test_read() + self._test_read(size=100) + + def test_with_1k_buffer(self): + self._test_readline(buffering=1024) + self._test_readline(size=100, buffering=1024) + self._test_read(buffering=1024) + self._test_read(size=100, buffering=1024) + + def _test_readline_no_buffer(self, size=-1): + mock_sock = self.MockSocket(recv_funcs=[ + lambda : b"a", + lambda : b"\n", + lambda : b"B", + self._raise_eintr, + lambda : b"b", + lambda : b"", + ]) + fo = mock_sock._textiowrap_for_test(buffering=0) + self.assertEqual(fo.readline(size), b"a\n") + self.assertEqual(fo.readline(size), b"Bb") + + def test_no_buffer(self): + self._test_readline_no_buffer() + self._test_readline_no_buffer(size=4) + self._test_read(buffering=0) + self._test_read(size=100, buffering=0) + + +class UnbufferedFileObjectClassTestCase(FileObjectClassTestCase): + + """Repeat the tests from FileObjectClassTestCase with bufsize==0. + + In this case (and in this case only), it should be possible to + create a file object, read a line from it, create another file + object, read another line from it, without loss of data in the + first file object's buffer. Note that http.client relies on this + when reading multiple requests from the same socket.""" + + bufsize = 0 # Use unbuffered mode + + def testUnbufferedReadline(self): + # Read a line, create a new file object, read another line with it + line = self.read_file.readline() # first line + self.assertEqual(line, b"A. " + self.write_msg) # first line + self.read_file = self.cli_conn.makefile('rb', 0) + line = self.read_file.readline() # second line + self.assertEqual(line, b"B. " + self.write_msg) # second line + + def _testUnbufferedReadline(self): + self.write_file.write(b"A. " + self.write_msg) + self.write_file.write(b"B. " + self.write_msg) + self.write_file.flush() + + def testMakefileClose(self): + # The file returned by makefile should keep the socket open... + self.cli_conn.close() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, self.read_msg) + # ...until the file is itself closed + self.read_file.close() + self.assertRaises(OSError, self.cli_conn.recv, 1024) + + def _testMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileCloseSocketDestroy(self): + refcount_before = sys.getrefcount(self.cli_conn) + self.read_file.close() + refcount_after = sys.getrefcount(self.cli_conn) + self.assertEqual(refcount_before - 1, refcount_after) + + def _testMakefileCloseSocketDestroy(self): + pass + + # Non-blocking ops + # NOTE: to set `read_file` as non-blocking, we must call + # `cli_conn.setblocking` and vice-versa (see setUp / clientSetUp). + + def testSmallReadNonBlocking(self): + self.cli_conn.setblocking(False) + self.assertEqual(self.read_file.readinto(bytearray(10)), None) + self.assertEqual(self.read_file.read(len(self.read_msg) - 3), None) + self.evt1.set() + self.evt2.wait(1.0) + first_seg = self.read_file.read(len(self.read_msg) - 3) + if first_seg is None: + # Data not arrived (can happen under Windows), wait a bit + time.sleep(0.5) + first_seg = self.read_file.read(len(self.read_msg) - 3) + buf = bytearray(10) + n = self.read_file.readinto(buf) + self.assertEqual(n, 3) + msg = first_seg + buf[:n] + self.assertEqual(msg, self.read_msg) + self.assertEqual(self.read_file.readinto(bytearray(16)), None) + self.assertEqual(self.read_file.read(1), None) + + def _testSmallReadNonBlocking(self): + self.evt1.wait(1.0) + self.write_file.write(self.write_msg) + self.write_file.flush() + self.evt2.set() + # Avoid cloding the socket before the server test has finished, + # otherwise system recv() will return 0 instead of EWOULDBLOCK. + self.serv_finished.wait(5.0) + + def testWriteNonBlocking(self): + self.cli_finished.wait(5.0) + # The client thread can't skip directly - the SkipTest exception + # would appear as a failure. + if self.serv_skipped: + self.skipTest(self.serv_skipped) + + def _testWriteNonBlocking(self): + self.serv_skipped = None + self.serv_conn.setblocking(False) + # Try to saturate the socket buffer pipe with repeated large writes. + BIG = b"x" * support.SOCK_MAX_SIZE + LIMIT = 10 + # The first write() succeeds since a chunk of data can be buffered + n = self.write_file.write(BIG) + self.assertGreater(n, 0) + for i in range(LIMIT): + n = self.write_file.write(BIG) + if n is None: + # Succeeded + break + self.assertGreater(n, 0) + else: + # Let us know that this test didn't manage to establish + # the expected conditions. This is not a failure in itself but, + # if it happens repeatedly, the test should be fixed. + self.serv_skipped = "failed to saturate the socket buffer" + + +class LineBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 1 # Default-buffered for reading; line-buffered for writing + + +class SmallBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 2 # Exercise the buffering code + + +class UnicodeReadFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'wb' + write_msg = MSG + newline = '' + + +class UnicodeWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'rb' + read_msg = MSG + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class UnicodeReadWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class NetworkConnectionTest(object): + """Prove network connection.""" + + def clientSetUp(self): + # We're inherited below by BasicTCPTest2, which also inherits + # BasicTCPTest, which defines self.port referenced below. + self.cli = socket.create_connection((HOST, self.port)) + self.serv_conn = self.cli + +class BasicTCPTest2(NetworkConnectionTest, BasicTCPTest): + """Tests that NetworkConnection does not break existing TCP functionality. + """ + +class NetworkConnectionNoServer(unittest.TestCase): + + class MockSocket(socket.socket): + def connect(self, *args): + raise socket.timeout('timed out') + + @contextlib.contextmanager + def mocked_socket_module(self): + """Return a socket which times out on connect""" + old_socket = socket.socket + socket.socket = self.MockSocket + try: + yield + finally: + socket.socket = old_socket + + def test_connect(self): + port = support.find_unused_port() + cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(cli.close) + with self.assertRaises(OSError) as cm: + cli.connect((HOST, port)) + self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + + def test_create_connection(self): + # Issue #9792: errors raised by create_connection() should have + # a proper errno attribute. + port = support.find_unused_port() + with self.assertRaises(OSError) as cm: + socket.create_connection((HOST, port)) + + # Issue #16257: create_connection() calls getaddrinfo() against + # 'localhost'. This may result in an IPV6 addr being returned + # as well as an IPV4 one: + # >>> socket.getaddrinfo('localhost', port, 0, SOCK_STREAM) + # >>> [(2, 2, 0, '', ('127.0.0.1', 41230)), + # (26, 2, 0, '', ('::1', 41230, 0, 0))] + # + # create_connection() enumerates through all the addresses returned + # and if it doesn't successfully bind to any of them, it propagates + # the last exception it encountered. + # + # On Solaris, ENETUNREACH is returned in this circumstance instead + # of ECONNREFUSED. So, if that errno exists, add it to our list of + # expected errnos. + expected_errnos = [ errno.ECONNREFUSED, ] + if hasattr(errno, 'ENETUNREACH'): + expected_errnos.append(errno.ENETUNREACH) + + self.assertIn(cm.exception.errno, expected_errnos) + + def test_create_connection_timeout(self): + # Issue #9792: create_connection() should not recast timeout errors + # as generic socket errors. + with self.mocked_socket_module(): + with self.assertRaises(socket.timeout): + socket.create_connection((HOST, 1234)) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.source_port = support.find_unused_port() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def _justAccept(self): + conn, addr = self.serv.accept() + conn.close() + + testFamily = _justAccept + def _testFamily(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.family, 2) + + testSourceAddress = _justAccept + def _testSourceAddress(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30, + source_address=('', self.source_port)) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.getsockname()[1], self.source_port) + # The port number being used is sufficient to show that the bind() + # call happened. + + testTimeoutDefault = _justAccept + def _testTimeoutDefault(self): + # passing no explicit timeout uses socket's global default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(42) + try: + self.cli = socket.create_connection((HOST, self.port)) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), 42) + + testTimeoutNone = _justAccept + def _testTimeoutNone(self): + # None timeout means the same as sock.settimeout(None) + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + self.cli = socket.create_connection((HOST, self.port), timeout=None) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), None) + + testTimeoutValueNamed = _justAccept + def _testTimeoutValueNamed(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.assertEqual(self.cli.gettimeout(), 30) + + testTimeoutValueNonamed = _justAccept + def _testTimeoutValueNonamed(self): + self.cli = socket.create_connection((HOST, self.port), 30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.gettimeout(), 30) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionBehaviourTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def testInsideTimeout(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + time.sleep(3) + conn.send(b"done!") + testOutsideTimeout = testInsideTimeout + + def _testInsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port)) + data = sock.recv(5) + self.assertEqual(data, b"done!") + + def _testOutsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port), timeout=1) + self.assertRaises(socket.timeout, lambda: sock.recv(5)) + + +class TCPTimeoutTest(SocketTCPTest): + + def testTCPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.accept() + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (TCP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of error (TCP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (TCP)") + if not ok: + self.fail("accept() returned success when we did not expect it") + + @unittest.skipUnless(hasattr(signal, 'alarm'), + 'test needs signal.alarm()') + def testInterruptedTimeout(self): + # XXX I don't know how to do this test on MSWindows or any other + # plaform that doesn't support signal.alarm() or os.kill(), though + # the bug should have existed on all platforms. + self.serv.settimeout(5.0) # must be longer than alarm + class Alarm(Exception): + pass + def alarm_handler(signal, frame): + raise Alarm + old_alarm = signal.signal(signal.SIGALRM, alarm_handler) + try: + signal.alarm(2) # POSIX allows alarm to be up to 1 second early + try: + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of Alarm") + except Alarm: + pass + except: + self.fail("caught other exception instead of Alarm:" + " %s(%s):\n%s" % + (sys.exc_info()[:2] + (traceback.format_exc(),))) + else: + self.fail("nothing caught") + finally: + signal.alarm(0) # shut off alarm + except Alarm: + self.fail("got Alarm in wrong place") + finally: + # no alarm can be pending. Safe to restore old handler. + signal.signal(signal.SIGALRM, old_alarm) + +class UDPTimeoutTest(SocketUDPTest): + + def testUDPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.recv(1024) + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (UDP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.recv(1024) + except socket.timeout: + self.fail("caught timeout instead of error (UDP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (UDP)") + if not ok: + self.fail("recv() returned success when we did not expect it") + +class TestExceptions(unittest.TestCase): + + def testExceptionTree(self): + self.assertTrue(issubclass(OSError, Exception)) + self.assertTrue(issubclass(socket.herror, OSError)) + self.assertTrue(issubclass(socket.gaierror, OSError)) + self.assertTrue(issubclass(socket.timeout, OSError)) + +@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +class TestLinuxAbstractNamespace(unittest.TestCase): + + UNIX_PATH_MAX = 108 + + def testLinuxAbstractNamespace(self): + address = b"\x00python-test-hello\x00\xff" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s1: + s1.bind(address) + s1.listen(1) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s2: + s2.connect(s1.getsockname()) + with s1.accept()[0] as s3: + self.assertEqual(s1.getsockname(), address) + self.assertEqual(s2.getpeername(), address) + + def testMaxName(self): + address = b"\x00" + b"h" * (self.UNIX_PATH_MAX - 1) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(address) + self.assertEqual(s.getsockname(), address) + + def testNameOverflow(self): + address = "\x00" + "h" * self.UNIX_PATH_MAX + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + self.assertRaises(OSError, s.bind, address) + + def testStrName(self): + # Check that an abstract name can be passed as a string. + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s.bind("\x00python\x00test\x00") + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + finally: + s.close() + +@unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'test needs socket.AF_UNIX') +class TestUnixDomain(unittest.TestCase): + + def setUp(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + def tearDown(self): + self.sock.close() + + def encoded(self, path): + # Return the given path encoded in the file system encoding, + # or skip the test if this is not possible. + try: + return os.fsencode(path) + except UnicodeEncodeError: + self.skipTest( + "Pathname {0!a} cannot be represented in file " + "system encoding {1!r}".format( + path, sys.getfilesystemencoding())) + + def bind(self, sock, path): + # Bind the socket + try: + sock.bind(path) + except OSError as e: + if str(e) == "AF_UNIX path too long": + self.skipTest( + "Pathname {0!a} is too long to serve as an AF_UNIX path" + .format(path)) + else: + raise + + def testStrAddr(self): + # Test binding to and retrieving a normal string pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testBytesAddr(self): + # Test binding to a bytes pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, self.encoded(path)) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testSurrogateescapeBind(self): + # Test binding to a valid non-ASCII pathname, with the + # non-ASCII bytes supplied using surrogateescape encoding. + path = os.path.abspath(support.TESTFN_UNICODE) + b = self.encoded(path) + self.bind(self.sock, b.decode("ascii", "surrogateescape")) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testUnencodableAddr(self): + # Test binding to a pathname that cannot be encoded in the + # file system encoding. + if support.TESTFN_UNENCODABLE is None: + self.skipTest("No unencodable filename available") + path = os.path.abspath(support.TESTFN_UNENCODABLE) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BufferIOTest(SocketConnectedTest): + """ + Test the buffer versions of socket.recv() and socket.send(). + """ + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecvIntoArray(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvIntoBytearray(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoBytearray = _testRecvIntoArray + + def testRecvIntoMemoryview(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoMemoryview = _testRecvIntoArray + + def testRecvFromIntoArray(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvFromIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoBytearray(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoBytearray = _testRecvFromIntoArray + + def testRecvFromIntoMemoryview(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoMemoryview = _testRecvFromIntoArray + + def testRecvFromIntoSmallBuffer(self): + # See issue #20246. + buf = bytearray(8) + self.assertRaises(ValueError, self.cli_conn.recvfrom_into, buf, 1024) + + def _testRecvFromIntoSmallBuffer(self): + self.serv_conn.send(MSG) + + def testRecvFromIntoEmptyBuffer(self): + buf = bytearray() + self.cli_conn.recvfrom_into(buf) + self.cli_conn.recvfrom_into(buf, 0) + + _testRecvFromIntoEmptyBuffer = _testRecvFromIntoArray + + +TIPC_STYPE = 2000 +TIPC_LOWER = 200 +TIPC_UPPER = 210 + +def isTipcAvailable(): + """Check if the TIPC module is loaded + + The TIPC module is not loaded automatically on Ubuntu and probably + other Linux distros. + """ + if not hasattr(socket, "AF_TIPC"): + return False + if not os.path.isfile("/proc/modules"): + return False + with open("/proc/modules") as f: + for line in f: + if line.startswith("tipc "): + return True + return False + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCTest(unittest.TestCase): + def testRDM(self): + srv = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + cli = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + self.addCleanup(srv.close) + self.addCleanup(cli.close) + + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + srv.bind(srvaddr) + + sendaddr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + cli.sendto(MSG, sendaddr) + + msg, recvaddr = srv.recvfrom(1024) + + self.assertEqual(cli.getsockname(), recvaddr) + self.assertEqual(msg, MSG) + + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCThreadableTest(unittest.TestCase, ThreadableTest): + def __init__(self, methodName = 'runTest'): + unittest.TestCase.__init__(self, methodName = methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.srv = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.srv.close) + self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + self.srv.bind(srvaddr) + self.srv.listen(5) + self.serverExplicitReady() + self.conn, self.connaddr = self.srv.accept() + self.addCleanup(self.conn.close) + + def clientSetUp(self): + # The is a hittable race between serverExplicitReady() and the + # accept() call; sleep a little while to avoid it, otherwise + # we could get an exception + time.sleep(0.1) + self.cli = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.cli.close) + addr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + self.cli.connect(addr) + self.cliaddr = self.cli.getsockname() + + def testStream(self): + msg = self.conn.recv(1024) + self.assertEqual(msg, MSG) + self.assertEqual(self.cliaddr, self.connaddr) + + def _testStream(self): + self.cli.send(MSG) + self.cli.close() + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class ContextManagersTest(ThreadedTCPSocketTest): + + def _testSocketClass(self): + # base test + with socket.socket() as sock: + self.assertFalse(sock._closed) + self.assertTrue(sock._closed) + # close inside with block + with socket.socket() as sock: + sock.close() + self.assertTrue(sock._closed) + # exception inside with block + with socket.socket() as sock: + self.assertRaises(OSError, sock.sendall, b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionBase(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionBase(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + self.assertFalse(sock._closed) + sock.sendall(b'foo') + self.assertEqual(sock.recv(1024), b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionClose(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionClose(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + sock.close() + self.assertTrue(sock._closed) + self.assertRaises(OSError, sock.sendall, b'foo') + + +class InheritanceTest(unittest.TestCase): + @unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), + "SOCK_CLOEXEC not defined") + @support.requires_linux_version(2, 6, 28) + def test_SOCK_CLOEXEC(self): + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: + self.assertTrue(s.type & socket.SOCK_CLOEXEC) + self.assertFalse(s.get_inheritable()) + + def test_default_inheritable(self): + sock = socket.socket() + with sock: + self.assertEqual(sock.get_inheritable(), False) + + def test_dup(self): + sock = socket.socket() + with sock: + newsock = sock.dup() + sock.close() + with newsock: + self.assertEqual(newsock.get_inheritable(), False) + + def test_set_inheritable(self): + sock = socket.socket() + with sock: + sock.set_inheritable(True) + self.assertEqual(sock.get_inheritable(), True) + + sock.set_inheritable(False) + self.assertEqual(sock.get_inheritable(), False) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + + @unittest.skipUnless(hasattr(socket, "socketpair"), + "need socket.socketpair()") + def test_socketpair(self): + s1, s2 = socket.socketpair() + self.addCleanup(s1.close) + self.addCleanup(s2.close) + self.assertEqual(s1.get_inheritable(), False) + self.assertEqual(s2.get_inheritable(), False) + + +@unittest.skipUnless(hasattr(socket, "SOCK_NONBLOCK"), + "SOCK_NONBLOCK not defined") +class NonblockConstantTest(unittest.TestCase): + def checkNonblock(self, s, nonblock=True, timeout=0.0): + if nonblock: + self.assertTrue(s.type & socket.SOCK_NONBLOCK) + self.assertEqual(s.gettimeout(), timeout) + else: + self.assertFalse(s.type & socket.SOCK_NONBLOCK) + self.assertEqual(s.gettimeout(), None) + + @support.requires_linux_version(2, 6, 28) + def test_SOCK_NONBLOCK(self): + # a lot of it seems silly and redundant, but I wanted to test that + # changing back and forth worked ok + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) as s: + self.checkNonblock(s) + s.setblocking(1) + self.checkNonblock(s, False) + s.setblocking(0) + self.checkNonblock(s) + s.settimeout(None) + self.checkNonblock(s, False) + s.settimeout(2.0) + self.checkNonblock(s, timeout=2.0) + s.setblocking(1) + self.checkNonblock(s, False) + # defaulttimeout + t = socket.getdefaulttimeout() + socket.setdefaulttimeout(0.0) + with socket.socket() as s: + self.checkNonblock(s) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(2.0) + with socket.socket() as s: + self.checkNonblock(s, timeout=2.0) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(t) + + +@unittest.skipUnless(os.name == "nt", "Windows specific") +@unittest.skipUnless(multiprocessing, "need multiprocessing") +class TestSocketSharing(SocketTCPTest): + # This must be classmethod and not staticmethod or multiprocessing + # won't be able to bootstrap it. + @classmethod + def remoteProcessServer(cls, q): + # Recreate socket from shared data + sdata = q.get() + message = q.get() + + s = socket.fromshare(sdata) + s2, c = s.accept() + + # Send the message + s2.sendall(message) + s2.close() + s.close() + + def testShare(self): + # Transfer the listening server socket to another process + # and service it from there. + + # Create process: + q = multiprocessing.Queue() + p = multiprocessing.Process(target=self.remoteProcessServer, args=(q,)) + p.start() + + # Get the shared socket data + data = self.serv.share(p.pid) + + # Pass the shared socket to the other process + addr = self.serv.getsockname() + self.serv.close() + q.put(data) + + # The data that the server will send us + message = b"slapmahfro" + q.put(message) + + # Connect + s = socket.create_connection(addr) + # listen for the data + m = [] + while True: + data = s.recv(100) + if not data: + break + m.append(data) + s.close() + received = b"".join(m) + self.assertEqual(received, message) + p.join() + + def testShareLength(self): + data = self.serv.share(os.getpid()) + self.assertRaises(ValueError, socket.fromshare, data[:-1]) + self.assertRaises(ValueError, socket.fromshare, data+b"foo") + + def compareSockets(self, org, other): + # socket sharing is expected to work only for blocking socket + # since the internal python timout value isn't transfered. + self.assertEqual(org.gettimeout(), None) + self.assertEqual(org.gettimeout(), other.gettimeout()) + + self.assertEqual(org.family, other.family) + self.assertEqual(org.type, other.type) + # If the user specified "0" for proto, then + # internally windows will have picked the correct value. + # Python introspection on the socket however will still return + # 0. For the shared socket, the python value is recreated + # from the actual value, so it may not compare correctly. + if org.proto != 0: + self.assertEqual(org.proto, other.proto) + + def testShareLocal(self): + data = self.serv.share(os.getpid()) + s = socket.fromshare(data) + try: + self.compareSockets(self.serv, s) + finally: + s.close() + + def testTypes(self): + families = [socket.AF_INET, socket.AF_INET6] + types = [socket.SOCK_STREAM, socket.SOCK_DGRAM] + for f in families: + for t in types: + try: + source = socket.socket(f, t) + except OSError: + continue # This combination is not supported + try: + data = source.share(os.getpid()) + shared = socket.fromshare(data) + try: + self.compareSockets(source, shared) + finally: + shared.close() + finally: + source.close() + + +def test_main(): + tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest, + TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ] + + tests.extend([ + NonBlockingTCPTests, + FileObjectClassTestCase, + FileObjectInterruptedTestCase, + UnbufferedFileObjectClassTestCase, + LineBufferedFileObjectClassTestCase, + SmallBufferedFileObjectClassTestCase, + UnicodeReadFileObjectClassTestCase, + UnicodeWriteFileObjectClassTestCase, + UnicodeReadWriteFileObjectClassTestCase, + NetworkConnectionNoServer, + NetworkConnectionAttributesTest, + NetworkConnectionBehaviourTest, + ContextManagersTest, + InheritanceTest, + NonblockConstantTest + ]) + tests.append(BasicSocketPairTest) + tests.append(TestUnixDomain) + tests.append(TestLinuxAbstractNamespace) + tests.extend([TIPCTest, TIPCThreadableTest]) + tests.extend([BasicCANTest, CANTest]) + tests.extend([BasicRDSTest, RDSTest]) + tests.extend([ + CmsgMacroTests, + SendmsgUDPTest, + RecvmsgUDPTest, + RecvmsgIntoUDPTest, + SendmsgUDP6Test, + RecvmsgUDP6Test, + RecvmsgRFC3542AncillaryUDP6Test, + RecvmsgIntoRFC3542AncillaryUDP6Test, + RecvmsgIntoUDP6Test, + SendmsgTCPTest, + RecvmsgTCPTest, + RecvmsgIntoTCPTest, + SendmsgSCTPStreamTest, + RecvmsgSCTPStreamTest, + RecvmsgIntoSCTPStreamTest, + SendmsgUnixStreamTest, + RecvmsgUnixStreamTest, + RecvmsgIntoUnixStreamTest, + RecvmsgSCMRightsStreamTest, + RecvmsgIntoSCMRightsStreamTest, + # These are slow when setitimer() is not available + InterruptedRecvTimeoutTest, + InterruptedSendTimeoutTest, + TestSocketSharing, + ]) + + thread_info = support.threading_setup() + support.run_unittest(*tests) + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.4/test_ssl.py b/src/greentest/3.4/test_ssl.py new file mode 100644 index 0000000..19ef354 --- /dev/null +++ b/src/greentest/3.4/test_ssl.py @@ -0,0 +1,2946 @@ +# Test the support for SSL and sockets + +import sys +import unittest +from test import support +import socket +import select +import time +import datetime +import gc +import os +import errno +import pprint +import tempfile +import urllib.request +import traceback +import asyncore +import weakref +import platform +import functools + +ssl = support.import_module("ssl") + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = support.HOST + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/make_ssl_certs.py. +# Other certificates are simply fetched from the Internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = os.fsencode(CERTFILE) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = os.fsencode(ONLYCERT) +BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = os.fsencode(CAPATH) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") + + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNING_CA = data_file("pycacert.pem") + +REMOTE_HOST = "self-signed.pythontest.net" +REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +WRONGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") + +DHFILE = data_file("dh1024.pem") +BYTES_DHFILE = os.fsencode(DHFILE) + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + +def can_clear_options(): + # 0.9.8m or higher + return ssl._OPENSSL_API_VERSION >= (0, 9, 8, 13, 15) + +def no_sslv2_implies_sslv3_hello(): + # 0.9.7h or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) + +def have_verify_flags(): + # 0.9.8 or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 8, 0, 15) + +def asn1time(cert_time): + # Some versions of OpenSSL ignore seconds, see #18207 + # 0.9.8.i + if ssl._OPENSSL_API_VERSION == (0, 9, 8, 9, 15): + fmt = "%b %d %H:%M:%S %Y GMT" + dt = datetime.datetime.strptime(cert_time, fmt) + dt = dt.replace(second=0) + cert_time = dt.strftime(fmt) + # %d adds leading zero but ASN1_TIME_print() uses leading space + if cert_time[4] == "0": + cert_time = cert_time[:4] + " " + cert_time[5:] + + return cert_time + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): + if hasattr(ssl, 'PROTOCOL_SSLv2'): + @functools.wraps(func) + def f(*args, **kwargs): + try: + ssl.SSLContext(ssl.PROTOCOL_SSLv2) + except ssl.SSLError: + if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and + platform.linux_distribution() == ('debian', 'squeeze/sid', '')): + raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") + return func(*args, **kwargs) + return f + else: + return func + +needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + if ssl.HAS_ECDH: + ssl.OP_SINGLE_ECDH_USE + if ssl.OPENSSL_VERSION_INFO >= (1, 0): + ssl.OP_NO_COMPRESSION + self.assertIn(ssl.HAS_SNI, {True, False}) + self.assertIn(ssl.HAS_ECDH, {True, False}) + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + + data, is_cryptographic = ssl.RAND_pseudo_bytes(16) + self.assertEqual(len(data), 16) + self.assertEqual(is_cryptographic, v == 1) + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + + # negative num is invalid + self.assertRaises(ValueError, ssl.RAND_bytes, -5) + self.assertRaises(ValueError, ssl.RAND_pseudo_bytes, -5) + + if hasattr(ssl, 'RAND_egd'): + self.assertRaises(TypeError, ssl.RAND_egd, 1) + self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1) + ssl.RAND_add("this is a random string", 75.0) + + @unittest.skipUnless(os.name == 'posix', 'requires posix') + def test_random_fork(self): + status = ssl.RAND_status() + if not status: + self.fail("OpenSSL's PRNG has insufficient randomness") + + rfd, wfd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(rfd) + child_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(child_random), 16) + os.write(wfd, child_random) + os.close(wfd) + except BaseException: + os._exit(1) + else: + os._exit(0) + else: + os.close(wfd) + self.addCleanup(os.close, rfd) + _, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + child_random = os.read(rfd, 16) + self.assertEqual(len(child_random), 16) + parent_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(parent_random), 16) + + self.assertNotEqual(child_random, parent_random) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + p = ssl._ssl._test_decode_cert(CERTFILE) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['issuer'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + # Note the next three asserts will fail if the keys are regenerated + self.assertEqual(p['notAfter'], asn1time('Oct 5 23:01:56 2020 GMT')) + self.assertEqual(p['notBefore'], asn1time('Oct 8 23:01:56 2010 GMT')) + self.assertEqual(p['serialNumber'], 'D7C7381919AFC24E') + self.assertEqual(p['subject'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, int) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 0.9 + self.assertGreaterEqual(n, 0x900000) + # < 3.0 + self.assertLess(n, 0x30000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 0) + self.assertLess(major, 3) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + # Version string as returned by {Open,Libre}SSL, the format might change + if "LibreSSL" in s: + self.assertTrue(s.startswith("LibreSSL {:d}.{:d}".format(major, minor)), + (s, t)) + else: + self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)), + (s, t)) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + wr = weakref.ref(ss) + with support.check_warnings(("", ResourceWarning)): + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # OSError raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s) as ss: + self.assertRaises(OSError, ss.recv, 1) + self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) + self.assertRaises(OSError, ss.recvfrom, 1) + self.assertRaises(OSError, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(OSError, ss.send, b'x') + self.assertRaises(OSError, ss.sendto, b'x', ('0.0.0.0', 0)) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with ssl.wrap_socket(s) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + def test_errors(self): + sock = socket.socket() + self.assertRaisesRegex(ValueError, + "certfile must be specified", + ssl.wrap_socket, sock, keyfile=CERTFILE) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True, certfile="") + with ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) as s: + self.assertRaisesRegex(ValueError, "can't connect in server-side mode", + s.connect, (HOST, 8080)) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=CERTFILE, keyfile=WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=WRONGCERT, keyfile=WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def test_match_hostname(self): + def ok(cert, hostname): + ssl.match_hostname(cert, hostname) + def fail(cert, hostname): + self.assertRaises(ssl.CertificateError, + ssl.match_hostname, cert, hostname) + + cert = {'subject': ((('commonName', 'example.com'),),)} + ok(cert, 'example.com') + ok(cert, 'ExAmple.cOm') + fail(cert, 'www.example.com') + fail(cert, '.example.com') + fail(cert, 'example.org') + fail(cert, 'exampleXcom') + + cert = {'subject': ((('commonName', '*.a.com'),),)} + ok(cert, 'foo.a.com') + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + # only match one left-most wildcard + cert = {'subject': ((('commonName', 'f*.com'),),)} + ok(cert, 'foo.com') + ok(cert, 'f.com') + fail(cert, 'bar.com') + fail(cert, 'foo.a.com') + fail(cert, 'bar.foo.com') + + # NULL bytes are bad, CVE-2013-4073 + cert = {'subject': ((('commonName', + 'null.python.org\x00example.org'),),)} + ok(cert, 'null.python.org\x00example.org') # or raise an error? + fail(cert, 'example.org') + fail(cert, 'null.python.org') + + # error cases with wildcards + cert = {'subject': ((('commonName', '*.*.a.com'),),)} + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + cert = {'subject': ((('commonName', 'a.*.com'),),)} + fail(cert, 'a.foo.com') + fail(cert, 'a..com') + fail(cert, 'a.com') + + # wildcard doesn't match IDNA prefix 'xn--' + idna = 'püthon.python.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, idna) + cert = {'subject': ((('commonName', 'x*.python.org'),),)} + fail(cert, idna) + cert = {'subject': ((('commonName', 'xn--p*.python.org'),),)} + fail(cert, idna) + + # wildcard in first fragment and IDNA A-labels in sequent fragments + # are supported. + idna = 'www*.pythön.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, 'www.pythön.org'.encode("idna").decode("ascii")) + ok(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'ftp.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'pythön.org'.encode("idna").decode("ascii")) + + # Slightly fake real-world example + cert = {'notAfter': 'Jun 26 21:41:46 2011 GMT', + 'subject': ((('commonName', 'linuxfrz.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('DNS', 'linuxfr.com'), + ('othername', ''))} + ok(cert, 'linuxfr.org') + ok(cert, 'linuxfr.com') + # Not a "DNS" entry + fail(cert, '') + # When there is a subjectAltName, commonName isn't used + fail(cert, 'linuxfrz.org') + + # A pristine real-world example + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),), + (('commonName', 'mail.google.com'),))} + ok(cert, 'mail.google.com') + fail(cert, 'gmail.com') + # Only commonName is considered + fail(cert, 'California') + + # Neither commonName nor subjectAltName + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),))} + fail(cert, 'mail.google.com') + + # No DNS entry in subjectAltName but a commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('commonName', 'mail.google.com'),)), + 'subjectAltName': (('othername', 'blabla'), )} + ok(cert, 'mail.google.com') + + # No DNS entry subjectAltName and no commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),)), + 'subjectAltName': (('othername', 'blabla'),)} + fail(cert, 'google.com') + + # Empty cert / no cert + self.assertRaises(ValueError, ssl.match_hostname, None, 'example.com') + self.assertRaises(ValueError, ssl.match_hostname, {}, 'example.com') + + # Issue #17980: avoid denials of service by refusing more than one + # wildcard per fragment. + cert = {'subject': ((('commonName', 'a*b.com'),),)} + ok(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b.co*'),),)} + fail(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b*.com'),),)} + with self.assertRaises(ssl.CertificateError) as cm: + ssl.match_hostname(cert, 'axxbxxc.com') + self.assertIn("too many wildcards", str(cm.exception)) + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with socket.socket() as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + def test_dealloc_warn(self): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + r = repr(ss) + with self.assertWarns(ResourceWarning) as cm: + ss = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (set, bool)) + if isinstance(trust, set): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegex(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegex(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + +class ContextTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_constructor(self): + for protocol in PROTOCOLS: + ssl.SSLContext(protocol) + self.assertRaises(TypeError, ssl.SSLContext) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + @skip_if_broken_ubuntu_ssl + def test_protocol(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.protocol, proto) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @skip_if_broken_ubuntu_ssl + def test_options(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3, + ctx.options) + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1, + ctx.options) + if can_clear_options(): + ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1 + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3, + ctx.options) + ctx.options = 0 + # Ubuntu has OP_NO_SSLv3 forced on by default + self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + else: + with self.assertRaises(ValueError): + ctx.options = 0 + + def test_verify_mode(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read() + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read() + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegex(ssl.SSLError, "no start line"): + ctx.load_verify_locations(cadata="broken") + with self.assertRaisesRegex(ssl.SSLError, "not enough data"): + ctx.load_verify_locations(cadata=b"broken") + + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_dh_params(DHFILE) + if os.name != 'nt': + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(FileNotFoundError) as cm: + ctx.load_dh_params(WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + @skip_if_broken_ubuntu_ssl + def test_session_stats(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + @needs_sni + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @needs_sni + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': asn1time('Mar 29 12:29:49 2033 GMT'), + 'notBefore': asn1time('Mar 30 12:29:49 2003 GMT'), + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + with open(SIGNING_CA) as f: + cadata = f.read() + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_DH_USE", 0), + getattr(ssl, "OP_SINGLE_DH_USE", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + ) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + def test_check_hostname(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertFalse(ctx.check_hostname) + + # Requires CERT_REQUIRED or CERT_OPTIONAL + with self.assertRaises(ValueError): + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + self.assertEqual(cm.exception.library, 'PEM') + self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) + self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen(5) + c = socket.socket() + c.connect(s.getsockname()) + c.setblocking(False) + with ctx.wrap_socket(c, False, do_handshake_on_connect=False) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertTrue(s.startswith("The operation did not complete (read)"), s) + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + +class NetworkedTests(unittest.TestCase): + + def test_connect(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) + try: + s.connect((REMOTE_HOST, 443)) + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + + # this should fail because we have no verification certs + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + + # this should succeed because we specify the root cert + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + s.connect((REMOTE_HOST, 443)) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.setblocking(False) + rc = s.connect_ex((REMOTE_HOST, 443)) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + s.close() + + def test_connect_ex_error(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + rc = s.connect_ex((REMOTE_HOST, 444)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + finally: + s.close() + + def test_connect_with_context(self): + with support.transient_internet(REMOTE_HOST): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + # Same with a server hostname + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=REMOTE_HOST) + s.connect((REMOTE_HOST, 443)) + s.close() + # This should fail because we have no verification certs + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + # This should succeed because we specify the root cert + ctx.load_verify_locations(REMOTE_ROOT_CERT) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_cadata(self): + with open(REMOTE_ROOT_CERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + with support.transient_internet(REMOTE_HOST): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss.connect((REMOTE_HOST, 443)) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + with support.transient_internet(REMOTE_HOST): + s = socket.socket(socket.AF_INET) + s.connect((REMOTE_HOST, 443)) + s.setblocking(False) + s = ssl.wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + s.close() + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + def _test_get_server_certificate(host, port, cert=None): + with support.transient_internet(host): + pem = ssl.get_server_certificate((host, port), + ssl.PROTOCOL_SSLv23) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + + try: + pem = ssl.get_server_certificate((host, port), + ssl.PROTOCOL_SSLv23, + ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + pem = ssl.get_server_certificate((host, port), + ssl.PROTOCOL_SSLv23, + ca_certs=cert) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + + _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) + if support.IPV6_ENABLED: + _test_get_server_certificate('ipv6.google.com', 443) + + def test_ciphers(self): + remote = (REMOTE_HOST, 443) + with support.transient_internet(remote[0]): + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(remote) + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(remote) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(remote) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + # sha256.tbs-internet.com needs SNI to use the correct certificate + if not ssl.HAS_SNI: + self.skipTest("SNI needed for this test") + # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with support.transient_internet("sha256.tbs-internet.com"): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(sha256_cert) + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="sha256.tbs-internet.com") + try: + s.connect(remote) + if support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + self.assertEqual(len(ctx.get_ca_certs()), 1) + + @needs_sni + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + with support.transient_internet(REMOTE_HOST): + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s) as ss: + ss.connect((REMOTE_HOST, 443)) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + + from test.ssl_servers import make_https_server + + class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(1) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_protocols.append(self.sslconn.selected_npn_protocol()) + except (ssl.SSLError, ConnectionResetError) as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + self.server.conn_errors.append(e) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.server.stop() + self.close() + return False + else: + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + sys.stdout.write(" cert binary is " + str(len(cert_binary)) + " bytes\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + sys.stdout.write(" server: selected protocol is now " + + str(self.sslconn.selected_npn_protocol()) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except OSError: + if self.server.chatty: + handle_error("Test server failure:\n") + self.close() + self.running = False + # normally, we'd just stop here, but for the test + # harness, we want to stop the server + self.server.stop() + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + npn_protocols=None, ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLSv1) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if npn_protocols: + self.context.set_npn_protocols(npn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = support.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_protocols = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.sock.settimeout(0.05) + self.sock.listen(5) + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except socket.timeout: + pass + except KeyboardInterrupt: + self.stop() + self.sock.close() + + def stop(self): + self.active = False + + class AsyncoreEchoServer(threading.Thread): + + # this one's based on asyncore.dispatcher + + class EchoServer (asyncore.dispatcher): + + class ConnectionHandler (asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = ssl.wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accepted(self, sock_obj, addr): + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + + def start (self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + + def bad_cert_test(certfile): + """ + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with the given client certificate fails. + """ + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_REQUIRED, + cacerts=CERTFILE, chatty=False, + connectionchatty=False) + with server: + try: + with socket.socket() as sock: + s = ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + except ssl.SSLError as x: + if support.verbose: + sys.stdout.write("\nSSLError is %s\n" % x.args[1]) + except OSError as x: + if support.verbose: + sys.stdout.write("\nOSError is %s\n" % x.args[1]) + except OSError as x: + if x.errno != errno.ENOENT: + raise + if support.verbose: + sys.stdout.write("\OSError is %s\n" % str(x)) + else: + raise AssertionError("Use of invalid cert should have failed!") + + def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=sni_name) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_npn_protocol': s.selected_npn_protocol() + }) + s.close() + stats['server_npn_protocols'] = server.selected_protocols + return stats + + def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_SSLv23: + client_context.set_ciphers("ALL") + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(CERTFILE) + ctx.load_verify_locations(CERTFILE) + try: + server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except OSError as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + + + class ThreadedTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + for protocol in PROTOCOLS: + with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): + context = ssl.SSLContext(protocol) + context.load_cert_chain(CERTFILE) + server_params_test(context, context, + chatty=True, connectionchatty=True) + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + s = context.wrap_socket(socket.socket(), + do_handshake_on_connect=False) + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + s.close() + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(SIGNING_CA) + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + with self.assertRaisesRegex(ssl.SSLError, + "certificate verify failed"): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="localhost") as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: + with self.assertRaisesRegex(ssl.CertificateError, + "hostname 'invalid' doesn't match 'localhost'"): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with socket.socket() as s: + with self.assertRaisesRegex(ValueError, + "check_hostname requires server_hostname"): + context.wrap_socket(s) + + def test_empty_cert(self): + """Connecting with an empty cert file""" + bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir, + "nullcert.pem")) + def test_malformed_cert(self): + """Connecting with a badly formatted certificate (syntax error)""" + bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir, + "badcert.pem")) + def test_nonexisting_cert(self): + """Connecting with a non-existing cert file""" + bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir, + "wrongcert.pem")) + def test_malformed_key(self): + """Connecting with a badly formatted key (syntax error)""" + bad_cert_test(os.path.join(os.path.dirname(__file__) or os.curdir, + "badkey.pem")) + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = support.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen(5) + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with socket.socket() as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = ssl.wrap_socket(c) + except OSError: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") + def test_protocol_sslv2(self): + """Connecting to an SSLv2 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) + # SSLv23 client with specific SSL options + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv2) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv23(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) + except OSError as x: + # this fails on some older versions of OpenSSL (0.9.7l, for instance) + if support.verbose: + sys.stdout.write( + " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" + % str(x)) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True) + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL) + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED) + + # Server with specific SSL options + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), + "OpenSSL is compiled without SSLv3 support") + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, + False, client_options=ssl.OP_NO_SSLv2) + + @skip_if_broken_ubuntu_ssl + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), + "TLS version 1.1 not supported.") + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, True) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, True) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), + "TLS version 1.2 not supported.") + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, True, + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, True) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(1) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using a SocketServer to create and manage SSL connections.""" + server = make_https_server(self, certfile=CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + with open(CERTFILE, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = 'https://localhost:%d/%s' % ( + server.port, os.path.split(CERTFILE)[1]) + context = ssl.create_default_context(cafile=CERTFILE) + f = urllib.request.urlopen(url, context=context) + try: + dlen = f.info().get("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + indata = "TEST MESSAGE of mixed case\n" + + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = ssl.wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, whether to expect success, *args) + send_methods = [ + ('send', s.send, True, []), + ('sendto', s.sendto, False, ["some.address"]), + ('sendall', s.sendall, True, []), + ] + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = "PREFIX_" + + for meth_name, send_meth, expect_success, args in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + send_meth(indata, *args) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, bytearray(100)) + + s.write(b"over\n") + s.close() + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen(5) + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + ssl.wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = ssl.wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + server = context.wrap_socket(server, server_side=True) + + evt = threading.Event() + remote = None + peer = None + def serve(): + nonlocal remote, peer + server.listen(5) + # Block on the accept and wait on the connection to close. + evt.set() + remote, peer = server.accept() + remote.recv(1) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = context.wrap_socket(socket.socket()) + client.connect((host, port)) + client_addr = client.getsockname() + client.close() + t.join() + remote.close() + server.close() + # Sanity checks. + self.assertIsInstance(remote, ssl.SSLSocket) + self.assertEqual(peer, client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_default_ciphers(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + try: + # Force a set of weak ciphers on our client context + context.set_ciphers("DES") + except ssl.SSLError: + self.skipTest("no DES cipher available") + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_SSLv23, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) + self.assertIn("no shared cipher", str(server.conn_errors[0])) + + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain(CERTFILE) + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + if ssl.OPENSSL_VERSION_INFO < (1, 0, 0): + context.set_ciphers("ECCdraft:ECDH") + with ThreadedEchoServer(context=context) as server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got channel binding data: {0!r}\n" + .format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + s.close() + + # now, again + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got another channel binding data: {0!r}\n" + .format(new_cb_data)) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + s.close() + + def test_compression(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['compression'], None) + + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.load_dh_params(DHFILE) + context.set_ciphers("kEDH") + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: + self.fail("Non-DH cipher: " + cipher[0]) + + def test_selected_npn_protocol(self): + # selected_npn_protocol() is None unless NPN is used + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_npn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test") + def test_npn_protocols(self): + server_protocols = ['http/1.1', 'spdy/2'] + protocol_tests = [ + (['http/1.1', 'spdy/2'], 'http/1.1'), + (['spdy/2', 'http/1.1'], 'http/1.1'), + (['spdy/2', 'test'], 'spdy/2'), + (['abc', 'def'], 'abc') + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_npn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_cert_chain(CERTFILE) + client_context.set_npn_protocols(client_protocols) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_npn_protocol'] + self.assertEqual(client_result, expected, msg % (client_result, "client")) + server_result = stats['server_npn_protocols'][-1] \ + if len(stats['server_npn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, msg % (server_result, "server")) + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + @needs_sni + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, 'localhost') + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, 'localhost') + self.assertEqual(calls, []) + + @needs_sni + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @needs_sni + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1/0 + server_context.set_servername_callback(cb_raising) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertIn("ZeroDivisionError", stderr.getvalue()) + + @needs_sni + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertIn("TypeError", stderr.getvalue()) + + def test_read_write_after_close_raises_valuerror(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + + with server: + s = context.wrap_socket(socket.socket()) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + +def test_main(verbose=False): + if support.verbose: + plats = { + 'Linux': platform.linux_distribution, + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + tests = [ContextTests, BasicSocketTests, SSLErrorTests] + + if support.is_resource_enabled('network'): + tests.append(NetworkedTests) + + if _have_threads: + thread_info = support.threading_setup() + if thread_info: + tests.append(ThreadedTests) + + try: + support.run_unittest(*tests) + finally: + if _have_threads: + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.4/test_subprocess.py b/src/greentest/3.4/test_subprocess.py new file mode 100644 index 0000000..34602fe --- /dev/null +++ b/src/greentest/3.4/test_subprocess.py @@ -0,0 +1,2556 @@ +import unittest +from test import script_helper +from test import support +import subprocess +import sys +import signal +import io +import locale +import os +import errno +import tempfile +import time +import re +import selectors +import sysconfig +import warnings +import select +import shutil +import gc +import textwrap + +try: + import threading +except ImportError: + threading = None + +mswindows = (sys.platform == "win32") + +# +# Depends on the following external programs: Python +# + +if mswindows: + SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), ' + 'os.O_BINARY);') +else: + SETBINARY = '' + + +try: + mkstemp = tempfile.mkstemp +except AttributeError: + # tempfile.mkstemp is not available + def mkstemp(): + """Replacement for mkstemp, calling mktemp.""" + fname = tempfile.mktemp() + return os.open(fname, os.O_RDWR|os.O_CREAT), fname + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + # Try to minimize the number of children we have so this test + # doesn't crash on some buildbots (Alphas in particular). + support.reap_children() + + def tearDown(self): + for inst in subprocess._active: + inst.wait() + subprocess._cleanup() + self.assertFalse(subprocess._active, "subprocess._active not empty") + + def assertStderrEqual(self, stderr, expected, msg=None): + # In a debug build, stuff like "[6580 refs]" is printed to stderr at + # shutdown time. That frustrates tests trying to check stderr produced + # from a spawned Python process. + actual = support.strip_python_stderr(stderr) + # strip_python_stderr also strips whitespace, so we do too. + expected = expected.strip() + self.assertEqual(actual, expected, msg) + + +class PopenTestException(Exception): + pass + + +class PopenExecuteChildRaises(subprocess.Popen): + """Popen subclass for testing cleanup of subprocess.PIPE filehandles when + _execute_child fails. + """ + def _execute_child(self, *args, **kwargs): + raise PopenTestException("Forced Exception for Test") + + +class ProcessTestCase(BaseTestCase): + + def test_io_buffered_by_default(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + self.assertIsInstance(p.stdin, io.BufferedIOBase) + self.assertIsInstance(p.stdout, io.BufferedIOBase) + self.assertIsInstance(p.stderr, io.BufferedIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_io_unbuffered_works(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0) + try: + self.assertIsInstance(p.stdin, io.RawIOBase) + self.assertIsInstance(p.stdout, io.RawIOBase) + self.assertIsInstance(p.stderr, io.RawIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_call_seq(self): + # call() function with sequence argument + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(rc, 47) + + def test_call_timeout(self): + # call() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.call waits for the + # child. + self.assertRaises(subprocess.TimeoutExpired, subprocess.call, + [sys.executable, "-c", "while True: pass"], + timeout=0.1) + + def test_check_call_zero(self): + # check_call() function with zero return code + rc = subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(0)"]) + self.assertEqual(rc, 0) + + def test_check_call_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(c.exception.returncode, 47) + + def test_check_output(self): + # check_output() function with zero return code + output = subprocess.check_output( + [sys.executable, "-c", "print('BDFL')"]) + self.assertIn(b'BDFL', output) + + def test_check_output_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_output( + [sys.executable, "-c", "import sys; sys.exit(5)"]) + self.assertEqual(c.exception.returncode, 5) + + def test_check_output_stderr(self): + # check_output() function stderr redirected to stdout + output = subprocess.check_output( + [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], + stderr=subprocess.STDOUT) + self.assertIn(b'BDFL', output) + + def test_check_output_stdin_arg(self): + # check_output() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + stdin=tf) + self.assertIn(b'PEAR', output) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + input=b'pear') + self.assertIn(b'PEAR', output) + + def test_check_output_stdout_arg(self): + # check_output() refuses to accept 'stdout' argument + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdout=sys.stdout) + self.fail("Expected ValueError when stdout arg supplied.") + self.assertIn('stdout', c.exception.args[0]) + + def test_check_output_stdin_with_input_arg(self): + # check_output() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdin=tf, input=b'hare') + self.fail("Expected ValueError when stdin and input args supplied.") + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + # check_output() function with timeout arg + with self.assertRaises(subprocess.TimeoutExpired) as c: + output = subprocess.check_output( + [sys.executable, "-c", + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"], + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3) + self.fail("Expected TimeoutExpired.") + self.assertEqual(c.exception.output, b'BDFL') + + def test_call_kwargs(self): + # call() function with keyword args + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + rc = subprocess.call([sys.executable, "-c", + 'import sys, os;' + 'sys.exit(os.getenv("FRUIT")=="banana")'], + env=newenv) + self.assertEqual(rc, 1) + + def test_invalid_args(self): + # Popen() called with invalid arguments should raise TypeError + # but Popen.__del__ should not complain (issue #12085) + with support.captured_stderr() as s: + self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) + argcount = subprocess.Popen.__init__.__code__.co_argcount + too_many_args = [0] * (argcount + 1) + self.assertRaises(TypeError, subprocess.Popen, *too_many_args) + self.assertEqual(s.getvalue(), '') + + def test_stdin_none(self): + # .stdin is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + p.wait() + self.assertEqual(p.stdin, None) + + def test_stdout_none(self): + # .stdout is None when not redirected, and the child's stdout will + # be inherited from the parent. In order to test this we run a + # subprocess in a subprocess: + # this_test + # \-- subprocess created by this test (parent) + # \-- subprocess created by the parent subprocess (child) + # The parent doesn't specify stdout, so the child will use the + # parent's stdout. This test checks that the message printed by the + # child goes to the parent stdout. The parent also checks that the + # child's stdout is None. See #11963. + code = ('import sys; from subprocess import Popen, PIPE;' + 'p = Popen([sys.executable, "-c", "print(\'test_stdout_none\')"],' + ' stdin=PIPE, stderr=PIPE);' + 'p.wait(); assert p.stdout is None;') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test_stdout_none') + + def test_stderr_none(self): + # .stderr is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + p.wait() + self.assertEqual(p.stderr, None) + + def _assert_python(self, pre_args, **kwargs): + # We include sys.exit() to prevent the test runner from hanging + # whenever python is found. + args = pre_args + ["import sys; sys.exit(47)"] + p = subprocess.Popen(args, **kwargs) + p.wait() + self.assertEqual(47, p.returncode) + + def test_executable(self): + # Check that the executable argument works. + # + # On Unix (non-Mac and non-Windows), Python looks at args[0] to + # determine where its standard library is, so we need the directory + # of args[0] to be valid for the Popen() call to Python to succeed. + # See also issue #16170 and issue #7774. + doesnotexist = os.path.join(os.path.dirname(sys.executable), + "doesnotexist") + self._assert_python([doesnotexist, "-c"], executable=sys.executable) + + def test_executable_takes_precedence(self): + # Check that the executable argument takes precedence over args[0]. + # + # Verify first that the call succeeds without the executable arg. + pre_args = [sys.executable, "-c"] + self._assert_python(pre_args) + self.assertRaises(FileNotFoundError, self._assert_python, pre_args, + executable="doesnotexist") + + @unittest.skipIf(mswindows, "executable argument replaces shell") + def test_executable_replaces_shell(self): + # Check that the executable argument replaces the default shell + # when shell=True. + self._assert_python([], executable=sys.executable, shell=True) + + # For use in the test_cwd* tests below. + def _normalize_cwd(self, cwd): + # Normalize an expected cwd (for Tru64 support). + # We can't use os.path.realpath since it doesn't expand Tru64 {memb} + # strings. See bug #1063571. + original_cwd = os.getcwd() + os.chdir(cwd) + cwd = os.getcwd() + os.chdir(original_cwd) + return cwd + + # For use in the test_cwd* tests below. + def _split_python_path(self): + # Return normalized (python_dir, python_base). + python_path = os.path.realpath(sys.executable) + return os.path.split(python_path) + + # For use in the test_cwd* tests below. + def _assert_cwd(self, expected_cwd, python_arg, **kwargs): + # Invoke Python via Popen, and assert that (1) the call succeeds, + # and that (2) the current working directory of the child process + # matches *expected_cwd*. + p = subprocess.Popen([python_arg, "-c", + "import os, sys; " + "sys.stdout.write(os.getcwd()); " + "sys.exit(47)"], + stdout=subprocess.PIPE, + **kwargs) + self.addCleanup(p.stdout.close) + p.wait() + self.assertEqual(47, p.returncode) + normcase = os.path.normcase + self.assertEqual(normcase(expected_cwd), + normcase(p.stdout.read().decode("utf-8"))) + + def test_cwd(self): + # Check that cwd changes the cwd for the child process. + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_arg(self): + # Check that Popen looks for args[0] relative to cwd if args[0] + # is relative. + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + with support.temp_cwd('test_cwd_with_relative_arg', quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python]) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, rel_python, cwd=python_dir) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_executable(self): + # Check that Popen looks for executable relative to cwd if executable + # is relative (and that executable takes precedence over args[0]). + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + doesntexist = "somethingyoudonthave" + with support.temp_cwd('test_cwd_with_relative_executable', quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python, + cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, doesntexist, executable=rel_python, + cwd=python_dir) + + def test_cwd_with_absolute_arg(self): + # Check that Popen can find the executable when the cwd is wrong + # if args[0] is an absolute path. + python_dir, python_base = self._split_python_path() + abs_python = os.path.join(python_dir, python_base) + rel_python = os.path.join(os.curdir, python_base) + with script_helper.temp_dir() as wrong_dir: + # Before calling with an absolute path, confirm that using a + # relative path fails. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + wrong_dir = self._normalize_cwd(wrong_dir) + self._assert_cwd(wrong_dir, abs_python, cwd=wrong_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + def test_executable_with_cwd(self): + python_dir, python_base = self._split_python_path() + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, "somethingyoudonthave", + executable=sys.executable, cwd=python_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + @unittest.skipIf(sysconfig.is_python_build(), + "need an installed Python. See #7774") + def test_executable_without_cwd(self): + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + self._assert_cwd(os.getcwd(), "somethingyoudonthave", + executable=sys.executable) + + def test_stdin_pipe(self): + # stdin redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.stdin.write(b"pear") + p.stdin.close() + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_filedes(self): + # stdin is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + os.write(d, b"pear") + os.lseek(d, 0, 0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=d) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_fileobj(self): + # stdin is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b"pear") + tf.seek(0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=tf) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdout_pipe(self): + # stdout redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), b"orange") + + def test_stdout_filedes(self): + # stdout is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=d) + p.wait() + os.lseek(d, 0, 0) + self.assertEqual(os.read(d, 1024), b"orange") + + def test_stdout_fileobj(self): + # stdout is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=tf) + p.wait() + tf.seek(0) + self.assertEqual(tf.read(), b"orange") + + def test_stderr_pipe(self): + # stderr redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=subprocess.PIPE) + self.addCleanup(p.stderr.close) + self.assertStderrEqual(p.stderr.read(), b"strawberry") + + def test_stderr_filedes(self): + # stderr is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=d) + p.wait() + os.lseek(d, 0, 0) + self.assertStderrEqual(os.read(d, 1024), b"strawberry") + + def test_stderr_fileobj(self): + # stderr is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"strawberry") + + def test_stdout_stderr_pipe(self): + # capture stdout and stderr to the same pipe + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + self.addCleanup(p.stdout.close) + self.assertStderrEqual(p.stdout.read(), b"appleorange") + + def test_stdout_stderr_file(self): + # capture stdout and stderr to the same open file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=tf, + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"appleorange") + + def test_stdout_filedes_of_stdout(self): + # stdout is set to 1 (#1531862). + # To avoid printing the text on stdout, we do something similar to + # test_stdout_none (see above). The parent subprocess calls the child + # subprocess passing stdout=1, and this test uses stdout=PIPE in + # order to capture and check the output of the parent. See #11963. + code = ('import sys, subprocess; ' + 'rc = subprocess.call([sys.executable, "-c", ' + ' "import os, sys; sys.exit(os.write(sys.stdout.fileno(), ' + 'b\'test with stdout=1\'))"], stdout=1); ' + 'assert rc == 18') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test with stdout=1') + + def test_stdout_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'for i in range(10240):' + 'print("x" * 1024)'], + stdout=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdout, None) + + def test_stderr_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys\n' + 'for i in range(10240):' + 'sys.stderr.write("x" * 1024)'], + stderr=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stderr, None) + + def test_stdin_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdin.read(1)'], + stdin=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdin, None) + + def test_env(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "orange" + with subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange") + + # Windows requires at least the SYSTEMROOT environment variable to start + # Python + @unittest.skipIf(sys.platform == 'win32', + 'cannot test an empty env on Windows') + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') is not None, + 'the python library cannot be loaded ' + 'with an empty environment') + def test_empty_env(self): + with subprocess.Popen([sys.executable, "-c", + 'import os; ' + 'print(list(os.environ.keys()))'], + stdout=subprocess.PIPE, + env={}) as p: + stdout, stderr = p.communicate() + self.assertIn(stdout.strip(), + (b"[]", + # Mac OS X adds __CF_USER_TEXT_ENCODING variable to an empty + # environment + b"['__CF_USER_TEXT_ENCODING']")) + + def test_communicate_stdin(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.communicate(b"pear") + self.assertEqual(p.returncode, 1) + + def test_communicate_stdout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("pineapple")'], + stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, b"pineapple") + self.assertEqual(stderr, None) + + def test_communicate_stderr(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("pineapple")'], + stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple");' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(b"banana") + self.assertEqual(stdout, b"banana") + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate_timeout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stderr.write("pineapple\\n");' + 'time.sleep(1);' + 'sys.stderr.write("pear\\n");' + 'sys.stdout.write(sys.stdin.read())'], + universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, "banana", + timeout=0.3) + # Make sure we can keep waiting for it, and that we get the whole output + # after it completes. + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr.encode(), b"pineapple\npear\n") + + def test_communicate_timeout_large_ouput(self): + # Test an expiring timeout while the child is outputting lots of data. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));'], + stdout=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, timeout=0.4) + (stdout, _) = p.communicate() + self.assertEqual(len(stdout), 4 * 64 * 1024) + + # Test for the fd leak reported in http://bugs.python.org/issue2791. + def test_communicate_pipe_fd_leak(self): + for stdin_pipe in (False, True): + for stdout_pipe in (False, True): + for stderr_pipe in (False, True): + options = {} + if stdin_pipe: + options['stdin'] = subprocess.PIPE + if stdout_pipe: + options['stdout'] = subprocess.PIPE + if stderr_pipe: + options['stderr'] = subprocess.PIPE + if not options: + continue + p = subprocess.Popen((sys.executable, "-c", "pass"), **options) + p.communicate() + if p.stdin is not None: + self.assertTrue(p.stdin.closed) + if p.stdout is not None: + self.assertTrue(p.stdout.closed) + if p.stderr is not None: + self.assertTrue(p.stderr.closed) + + def test_communicate_returns(self): + # communicate() should return None if no redirection is active + p = subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(47)"]) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertEqual(stderr, None) + + def test_communicate_pipe_buf(self): + # communicate() with writes larger than pipe_buf + # This test will probably deadlock rather than fail, if + # communicate() does not work properly. + x, y = os.pipe() + os.close(x) + os.close(y) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read(47));' + 'sys.stderr.write("x" * %d);' + 'sys.stdout.write(sys.stdin.read())' % + support.PIPE_MAX_SIZE], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + string_to_write = b"a" * support.PIPE_MAX_SIZE + (stdout, stderr) = p.communicate(string_to_write) + self.assertEqual(stdout, string_to_write) + + def test_writes_before_communicate(self): + # stdin.write before communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.stdin.write(b"banana") + (stdout, stderr) = p.communicate(b"split") + self.assertEqual(stdout, b"bananasplit") + self.assertStderrEqual(stderr, b"") + + def test_universal_newlines(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(sys.stdin.readline().encode());' + 'buf.flush();' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(sys.stdin.read().encode());' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + p.stdin.write("line1\n") + p.stdin.flush() + self.assertEqual(p.stdout.readline(), "line1\n") + p.stdin.write("line3\n") + p.stdin.close() + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.readline(), + "line2\n") + self.assertEqual(p.stdout.read(6), + "line3\n") + self.assertEqual(p.stdout.read(), + "line4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate(self): + # universal newlines through communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, + "line2\nline4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate_stdin(self): + # universal newlines through communicate(), with only stdin + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.readline() + assert s == "line1\\n", repr(s) + s = sys.stdin.read() + assert s == "line3\\n", repr(s) + ''')], + stdin=subprocess.PIPE, + universal_newlines=1) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_input_none(self): + # Test communicate(input=None) with universal newlines. + # + # We set stdout to PIPE because, as of this writing, a different + # code path is tested when the number of pipes is zero or one. + p = subprocess.Popen([sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + p.communicate() + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_stdin_stdout_stderr(self): + # universal newlines through communicate(), with stdin, stdout, stderr + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.buffer.readline() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line2\\r") + sys.stderr.buffer.write(b"eline2\\n") + s = sys.stdin.buffer.read() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line4\\n") + sys.stdout.buffer.write(b"line5\\r\\n") + sys.stderr.buffer.write(b"eline6\\r") + sys.stderr.buffer.write(b"eline7\\r\\nz") + ''')], + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) + # Python debug build push something like "[42442 refs]\n" + # to stderr at exit of subprocess. + # Don't use assertStderrEqual because it strips CR and LF from output. + self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + + def test_universal_newlines_communicate_encodings(self): + # Check that universal newlines mode works for various encodings, + # in particular for encodings in the UTF-16 and UTF-32 families. + # See issue #15595. + # + # UTF-16 and UTF-32-BE are sufficient to check both with BOM and + # without, and UTF-16 and UTF-32. + import _bootlocale + for encoding in ['utf-16', 'utf-32-be']: + old_getpreferredencoding = _bootlocale.getpreferredencoding + # Indirectly via io.TextIOWrapper, Popen() defaults to + # locale.getpreferredencoding(False) and earlier in Python 3.2 to + # locale.getpreferredencoding(). + def getpreferredencoding(do_setlocale=True): + return encoding + code = ("import sys; " + r"sys.stdout.buffer.write('1\r\n2\r3\n4'.encode('%s'))" % + encoding) + args = [sys.executable, '-c', code] + try: + _bootlocale.getpreferredencoding = getpreferredencoding + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout, stderr = popen.communicate(input='') + finally: + _bootlocale.getpreferredencoding = old_getpreferredencoding + self.assertEqual(stdout, '1\n2\n3\n4') + + def test_no_leaking(self): + # Make sure we leak no resources + if not mswindows: + max_handles = 1026 # too much for most UNIX systems + else: + max_handles = 2050 # too much for (at least some) Windows setups + handles = [] + tmpdir = tempfile.mkdtemp() + try: + for i in range(max_handles): + try: + tmpfile = os.path.join(tmpdir, support.TESTFN) + handles.append(os.open(tmpfile, os.O_WRONLY|os.O_CREAT)) + except OSError as e: + if e.errno != errno.EMFILE: + raise + break + else: + self.skipTest("failed to reach the file descriptor limit " + "(tried %d)" % max_handles) + # Close a couple of them (should be enough for a subprocess) + for i in range(10): + os.close(handles.pop()) + # Loop creating some subprocesses. If one of them leaks some fds, + # the next loop iteration will fail by reaching the max fd limit. + for i in range(15): + p = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write(sys.stdin.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + data = p.communicate(b"lime")[0] + self.assertEqual(data, b"lime") + finally: + for h in handles: + os.close(h) + shutil.rmtree(tmpdir) + + def test_list2cmdline(self): + self.assertEqual(subprocess.list2cmdline(['a b c', 'd', 'e']), + '"a b c" d e') + self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']), + 'ab\\"c \\ d') + self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']), + 'ab\\"c " \\\\" d') + self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']), + 'a\\\\\\b "de fg" h') + self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']), + 'a\\\\\\"b c d') + self.assertEqual(subprocess.list2cmdline(['a\\\\b c', 'd', 'e']), + '"a\\\\b c" d e') + self.assertEqual(subprocess.list2cmdline(['a\\\\b\\ c', 'd', 'e']), + '"a\\\\b\\ c" d e') + self.assertEqual(subprocess.list2cmdline(['ab', '']), + 'ab ""') + + def test_poll(self): + p = subprocess.Popen([sys.executable, "-c", + "import os; os.read(0, 1)"], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + self.assertIsNone(p.poll()) + os.write(p.stdin.fileno(), b'A') + p.wait() + # Subsequent invocations should just return the returncode + self.assertEqual(p.poll(), 0) + + def test_wait(self): + p = subprocess.Popen([sys.executable, "-c", "pass"]) + self.assertEqual(p.wait(), 0) + # Subsequent invocations should just return the returncode + self.assertEqual(p.wait(), 0) + + def test_wait_timeout(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(0.3)"]) + with self.assertRaises(subprocess.TimeoutExpired) as c: + p.wait(timeout=0.0001) + self.assertIn("0.0001", str(c.exception)) # For coverage of __str__. + # Some heavily loaded buildbots (sparc Debian 3.x) require this much + # time to start. + self.assertEqual(p.wait(timeout=3), 0) + + def test_invalid_bufsize(self): + # an invalid type of the bufsize argument should raise + # TypeError. + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], "orange") + + def test_bufsize_is_none(self): + # bufsize=None should be the same as bufsize=0. + p = subprocess.Popen([sys.executable, "-c", "pass"], None) + self.assertEqual(p.wait(), 0) + # Again with keyword arg + p = subprocess.Popen([sys.executable, "-c", "pass"], bufsize=None) + self.assertEqual(p.wait(), 0) + + def _test_bufsize_equal_one(self, line, expected, universal_newlines): + # subprocess may deadlock with bufsize=1, see issue #21332 + with subprocess.Popen([sys.executable, "-c", "import sys;" + "sys.stdout.write(sys.stdin.readline());" + "sys.stdout.flush()"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + bufsize=1, + universal_newlines=universal_newlines) as p: + p.stdin.write(line) # expect that it flushes the line in text mode + os.close(p.stdin.fileno()) # close it without flushing the buffer + read_line = p.stdout.readline() + try: + p.stdin.close() + except OSError: + pass + p.stdin = None + self.assertEqual(p.returncode, 0) + self.assertEqual(read_line, expected) + + def test_bufsize_equal_one_text_mode(self): + # line is flushed in text mode with bufsize=1. + # we should get the full line in return + line = "line\n" + self._test_bufsize_equal_one(line, line, universal_newlines=True) + + def test_bufsize_equal_one_binary_mode(self): + # line is not flushed in binary mode with bufsize=1. + # we should get empty response + line = b'line' + os.linesep.encode() # assume ascii-based locale + self._test_bufsize_equal_one(line, b'', universal_newlines=False) + + def test_leaking_fds_on_error(self): + # see bug #5179: Popen leaks file descriptors to PIPEs if + # the child fails to execute; this will eventually exhaust + # the maximum number of open fds. 1024 seems a very common + # value for that limit, but Windows has 2048, so we loop + # 1024 times (each call leaked two fds). + for i in range(1024): + with self.assertRaises(OSError) as c: + subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # ignore errors that indicate the command was not found + if c.exception.errno not in (errno.ENOENT, errno.EACCES): + raise c.exception + + @unittest.skipIf(threading is None, "threading required") + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(['nonexisting_i_hope'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + + @unittest.skipIf(threading is None, "threading required") + def test_threadsafe_wait(self): + """Issue21291: Popen.wait() needs to be threadsafe for returncode.""" + proc = subprocess.Popen([sys.executable, '-c', + 'import time; time.sleep(12)']) + self.assertEqual(proc.returncode, None) + results = [] + + def kill_proc_timer_thread(): + results.append(('thread-start-poll-result', proc.poll())) + # terminate it from the thread and wait for the result. + proc.kill() + proc.wait() + results.append(('thread-after-kill-and-wait', proc.returncode)) + # this wait should be a no-op given the above. + proc.wait() + results.append(('thread-after-second-wait', proc.returncode)) + + # This is a timing sensitive test, the failure mode is + # triggered when both the main thread and this thread are in + # the wait() call at once. The delay here is to allow the + # main thread to most likely be blocked in its wait() call. + t = threading.Timer(0.2, kill_proc_timer_thread) + t.start() + + if mswindows: + expected_errorcode = 1 + else: + # Should be -9 because of the proc.kill() from the thread. + expected_errorcode = -9 + + # Wait for the process to finish; the thread should kill it + # long before it finishes on its own. Supplying a timeout + # triggers a different code path for better coverage. + proc.wait(timeout=20) + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in wait from main thread") + + # This should be a no-op with no change in returncode. + proc.wait() + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in second main wait.") + + t.join() + # Ensure that all of the thread results are as expected. + # When a race condition occurs in wait(), the returncode could + # be set by the wrong thread that doesn't actually have it + # leading to an incorrect value. + self.assertEqual([('thread-start-poll-result', None), + ('thread-after-kill-and-wait', expected_errorcode), + ('thread-after-second-wait', expected_errorcode)], + results) + + def test_issue8780(self): + # Ensure that stdout is inherited from the parent + # if stdout=PIPE is not used + code = ';'.join(( + 'import subprocess, sys', + 'retcode = subprocess.call(' + "[sys.executable, '-c', 'print(\"Hello World!\")'])", + 'assert retcode == 0')) + output = subprocess.check_output([sys.executable, '-c', code]) + self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + + def test_handles_closed_on_exception(self): + # If CreateProcess exits with an error, ensure the + # duplicate output handles are released + ifhandle, ifname = mkstemp() + ofhandle, ofname = mkstemp() + efhandle, efname = mkstemp() + try: + subprocess.Popen (["*"], stdin=ifhandle, stdout=ofhandle, + stderr=efhandle) + except OSError: + os.close(ifhandle) + os.remove(ifname) + os.close(ofhandle) + os.remove(ofname) + os.close(efhandle) + os.remove(efname) + self.assertFalse(os.path.exists(ifname)) + self.assertFalse(os.path.exists(ofname)) + self.assertFalse(os.path.exists(efname)) + + def test_communicate_epipe(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.communicate(b"x" * 2**20) + + def test_communicate_epipe_only_stdin(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + p.wait() + p.communicate(b"x" * 2**20) + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), + "Requires signal.SIGUSR1") + @unittest.skipUnless(hasattr(os, 'kill'), + "Requires os.kill") + @unittest.skipUnless(hasattr(os, 'getppid'), + "Requires os.getppid") + def test_communicate_eintr(self): + # Issue #12493: communicate() should handle EINTR + def handler(signum, frame): + pass + old_handler = signal.signal(signal.SIGUSR1, handler) + self.addCleanup(signal.signal, signal.SIGUSR1, old_handler) + + args = [sys.executable, "-c", + 'import os, signal;' + 'os.kill(os.getppid(), signal.SIGUSR1)'] + for stream in ('stdout', 'stderr'): + kw = {stream: subprocess.PIPE} + with subprocess.Popen(args, **kw) as process: + # communicate() will be interrupted by SIGUSR1 + process.communicate() + + + # This test is Linux-ish specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + def test_failed_child_execute_fd_leak(self): + """Test for the fork() failure fd leak reported in issue16327.""" + fd_directory = '/proc/%d/fd' % os.getpid() + fds_before_popen = os.listdir(fd_directory) + with self.assertRaises(PopenTestException): + PopenExecuteChildRaises( + [sys.executable, '-c', 'pass'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # NOTE: This test doesn't verify that the real _execute_child + # does not close the file descriptors itself on the way out + # during an exception. Code inspection has confirmed that. + + fds_after_exception = os.listdir(fd_directory) + self.assertEqual(fds_before_popen, fds_after_exception) + +@unittest.skipIf(mswindows, "POSIX specific tests") +class POSIXProcessTestCase(BaseTestCase): + + def setUp(self): + super().setUp() + self._nonexistent_dir = "/_this/pa.th/does/not/exist" + + def _get_chdir_exception(self): + try: + os.chdir(self._nonexistent_dir) + except OSError as e: + # This avoids hard coding the errno value or the OS perror() + # string and instead capture the exception that we want to see + # below for comparison. + desired_exception = e + desired_exception.strerror += ': ' + repr(self._nonexistent_dir) + else: + self.fail("chdir to nonexistant directory %s succeeded." % + self._nonexistent_dir) + return desired_exception + + def test_exception_cwd(self): + """Test error in the child raised in the parent for a bad cwd.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + cwd=self._nonexistent_dir) + except OSError as e: + # Test that the child process chdir failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_executable(self): + """Test error in the child raised in the parent for a bad executable.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + executable=self._nonexistent_dir) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_args_0(self): + """Test error in the child raised in the parent for a bad args[0].""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([self._nonexistent_dir, "-c", ""]) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_restore_signals(self): + # Code coverage for both values of restore_signals to make sure it + # at least does not blow up. + # A test for behavior would be complex. Contributions welcome. + subprocess.call([sys.executable, "-c", ""], restore_signals=True) + subprocess.call([sys.executable, "-c", ""], restore_signals=False) + + def test_start_new_session(self): + # For code coverage of calling setsid(). We don't care if we get an + # EPERM error from it depending on the test execution environment, that + # still indicates that it was called. + try: + output = subprocess.check_output( + [sys.executable, "-c", + "import os; print(os.getpgid(os.getpid()))"], + start_new_session=True) + except OSError as e: + if e.errno != errno.EPERM: + raise + else: + parent_pgid = os.getpgid(os.getpid()) + child_pgid = int(output) + self.assertNotEqual(parent_pgid, child_pgid) + + def test_run_abort(self): + # returncode handles signal termination + with support.SuppressCrashReport(): + p = subprocess.Popen([sys.executable, "-c", + 'import os; os.abort()']) + p.wait() + self.assertEqual(-p.returncode, signal.SIGABRT) + + def test_preexec(self): + # DISCLAIMER: Setting environment variables is *not* a good use + # of a preexec_fn. This is merely a test. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + preexec_fn=lambda: os.putenv("FRUIT", "apple")) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), b"apple") + + def test_preexec_exception(self): + def raise_it(): + raise ValueError("What if two swallows carried a coconut?") + try: + p = subprocess.Popen([sys.executable, "-c", ""], + preexec_fn=raise_it) + except subprocess.SubprocessError as e: + self.assertTrue( + subprocess._posixsubprocess, + "Expected a ValueError from the preexec_fn") + except ValueError as e: + self.assertIn("coconut", e.args[0]) + else: + self.fail("Exception raised by preexec_fn did not make it " + "to the parent process.") + + class _TestExecuteChildPopen(subprocess.Popen): + """Used to test behavior at the end of _execute_child.""" + def __init__(self, testcase, *args, **kwargs): + self._testcase = testcase + subprocess.Popen.__init__(self, *args, **kwargs) + + def _execute_child(self, *args, **kwargs): + try: + subprocess.Popen._execute_child(self, *args, **kwargs) + finally: + # Open a bunch of file descriptors and verify that + # none of them are the same as the ones the Popen + # instance is using for stdin/stdout/stderr. + devzero_fds = [os.open("/dev/zero", os.O_RDONLY) + for _ in range(8)] + try: + for fd in devzero_fds: + self._testcase.assertNotIn( + fd, (self.stdin.fileno(), self.stdout.fileno(), + self.stderr.fileno()), + msg="At least one fd was closed early.") + finally: + for fd in devzero_fds: + os.close(fd) + + @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") + def test_preexec_errpipe_does_not_double_close_pipes(self): + """Issue16140: Don't double close pipes on preexec error.""" + + def raise_it(): + raise subprocess.SubprocessError( + "force the _execute_child() errpipe_data path.") + + with self.assertRaises(subprocess.SubprocessError): + self._TestExecuteChildPopen( + self, [sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=raise_it) + + def test_preexec_gc_module_failure(self): + # This tests the code that disables garbage collection if the child + # process will execute any Python. + def raise_runtime_error(): + raise RuntimeError("this shouldn't escape") + enabled = gc.isenabled() + orig_gc_disable = gc.disable + orig_gc_isenabled = gc.isenabled + try: + gc.disable() + self.assertFalse(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertFalse(gc.isenabled(), + "Popen enabled gc when it shouldn't.") + + gc.enable() + self.assertTrue(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertTrue(gc.isenabled(), "Popen left gc disabled.") + + gc.disable = raise_runtime_error + self.assertRaises(RuntimeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + + del gc.isenabled # force an AttributeError + self.assertRaises(AttributeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + finally: + gc.disable = orig_gc_disable + gc.isenabled = orig_gc_isenabled + if not enabled: + gc.disable() + + def test_args_string(self): + # args is a string + fd, fname = mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!/bin/sh\n") + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + p = subprocess.Popen(fname) + p.wait() + os.remove(fname) + self.assertEqual(p.returncode, 47) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + startupinfo=47) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + creationflags=47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen(["echo $FRUIT"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen("echo $FRUIT", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_call_string(self): + # call() function with string argument on UNIX + fd, fname = mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!/bin/sh\n") + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + rc = subprocess.call(fname) + os.remove(fname) + self.assertEqual(rc, 47) + + def test_specific_shell(self): + # Issue #9265: Incorrect name passed as arg[0]. + shells = [] + for prefix in ['/bin', '/usr/bin/', '/usr/local/bin']: + for name in ['bash', 'ksh']: + sh = os.path.join(prefix, name) + if os.path.isfile(sh): + shells.append(sh) + if not shells: # Will probably work for any shell but csh. + self.skipTest("bash or ksh required for this test") + sh = '/bin/sh' + if os.path.isfile(sh) and not os.path.islink(sh): + # Test will fail if /bin/sh is a symlink to csh. + shells.append(sh) + for sh in shells: + p = subprocess.Popen("echo $0", executable=sh, shell=True, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) + + def _kill_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + # Also set the SIGINT handler to the default to make sure it's not + # being ignored (some tests rely on that.) + old_handler = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + signal.signal(signal.SIGINT, old_handler) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + return p + + @unittest.skipIf(sys.platform.startswith(('netbsd', 'openbsd')), + "Due to known OS bug (issue #16762)") + def _kill_dead_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + p.communicate() + + def test_send_signal(self): + p = self._kill_process('send_signal', signal.SIGINT) + _, stderr = p.communicate() + self.assertIn(b'KeyboardInterrupt', stderr) + self.assertNotEqual(p.wait(), 0) + + def test_kill(self): + p = self._kill_process('kill') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGKILL) + + def test_terminate(self): + p = self._kill_process('terminate') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGTERM) + + def test_send_signal_dead(self): + # Sending a signal to a dead process + self._kill_dead_process('send_signal', signal.SIGINT) + + def test_kill_dead(self): + # Killing a dead process + self._kill_dead_process('kill') + + def test_terminate_dead(self): + # Terminating a dead process + self._kill_dead_process('terminate') + + def _save_fds(self, save_fds): + fds = [] + for fd in save_fds: + inheritable = os.get_inheritable(fd) + saved = os.dup(fd) + fds.append((fd, saved, inheritable)) + return fds + + def _restore_fds(self, fds): + for fd, saved, inheritable in fds: + os.dup2(saved, fd, inheritable=inheritable) + os.close(saved) + + def check_close_std_fds(self, fds): + # Issue #9905: test that subprocess pipes still work properly with + # some standard fds closed + stdin = 0 + saved_fds = self._save_fds(fds) + for fd, saved, inheritable in saved_fds: + if fd == 0: + stdin = saved + break + try: + for fd in fds: + os.close(fd) + out, err = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + err = support.strip_python_stderr(err) + self.assertEqual((out, err), (b'apple', b'orange')) + finally: + self._restore_fds(saved_fds) + + def test_close_fd_0(self): + self.check_close_std_fds([0]) + + def test_close_fd_1(self): + self.check_close_std_fds([1]) + + def test_close_fd_2(self): + self.check_close_std_fds([2]) + + def test_close_fds_0_1(self): + self.check_close_std_fds([0, 1]) + + def test_close_fds_0_2(self): + self.check_close_std_fds([0, 2]) + + def test_close_fds_1_2(self): + self.check_close_std_fds([1, 2]) + + def test_close_fds_0_1_2(self): + # Issue #10806: test that subprocess pipes still work properly with + # all standard fds closed. + self.check_close_std_fds([0, 1, 2]) + + def test_small_errpipe_write_fd(self): + """Issue #15798: Popen should work when stdio fds are available.""" + new_stdin = os.dup(0) + new_stdout = os.dup(1) + try: + os.close(0) + os.close(1) + + # Side test: if errpipe_write fails to have its CLOEXEC + # flag set this should cause the parent to think the exec + # failed. Extremely unlikely: everyone supports CLOEXEC. + subprocess.Popen([ + sys.executable, "-c", + "print('AssertionError:0:CLOEXEC failure.')"]).wait() + finally: + # Restore original stdin and stdout + os.dup2(new_stdin, 0) + os.dup2(new_stdout, 1) + os.close(new_stdin) + os.close(new_stdout) + + def test_remapping_std_fds(self): + # open up some temporary files + temps = [mkstemp() for i in range(3)] + try: + temp_fds = [fd for fd, fname in temps] + + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # write some data to what will become stdin, and rewind + os.write(temp_fds[1], b"STDIN") + os.lseek(temp_fds[1], 0, 0) + + # move the standard file descriptors out of the way + saved_fds = self._save_fds(range(3)) + try: + # duplicate the file objects over the standard fd's + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # now use those files in the "wrong" order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=temp_fds[1], + stdout=temp_fds[2], + stderr=temp_fds[0]) + p.wait() + finally: + self._restore_fds(saved_fds) + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(temp_fds[2], 1024) + err = support.strip_python_stderr(os.read(temp_fds[0], 1024)) + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + def check_swap_fds(self, stdin_no, stdout_no, stderr_no): + # open up some temporary files + temps = [mkstemp() for i in range(3)] + temp_fds = [fd for fd, fname in temps] + try: + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # save a copy of the standard file descriptors + saved_fds = self._save_fds(range(3)) + try: + # duplicate the temp files over the standard fd's 0, 1, 2 + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # write some data to what will become stdin, and rewind + os.write(stdin_no, b"STDIN") + os.lseek(stdin_no, 0, 0) + + # now use those files in the given order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=stdin_no, + stdout=stdout_no, + stderr=stderr_no) + p.wait() + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(stdout_no, 1024) + err = support.strip_python_stderr(os.read(stderr_no, 1024)) + finally: + self._restore_fds(saved_fds) + + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + # When duping fds, if there arises a situation where one of the fds is + # either 0, 1 or 2, it is possible that it is overwritten (#12607). + # This tests all combinations of this. + def test_swap_fds(self): + self.check_swap_fds(0, 1, 2) + self.check_swap_fds(0, 2, 1) + self.check_swap_fds(1, 0, 2) + self.check_swap_fds(1, 2, 0) + self.check_swap_fds(2, 0, 1) + self.check_swap_fds(2, 1, 0) + + def test_surrogates_error_message(self): + def prepare(): + raise ValueError("surrogate:\uDCff") + + try: + subprocess.call( + [sys.executable, "-c", "pass"], + preexec_fn=prepare) + except ValueError as err: + # Pure Python implementations keeps the message + self.assertIsNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "surrogate:\uDCff") + except subprocess.SubprocessError as err: + # _posixsubprocess uses a default message + self.assertIsNotNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "Exception occurred in preexec_fn.") + else: + self.fail("Expected ValueError or subprocess.SubprocessError") + + def test_undecodable_env(self): + for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): + encoded_value = value.encode("ascii", "surrogateescape") + + # test str with surrogates + script = "import os; print(ascii(os.getenv(%s)))" % repr(key) + env = os.environ.copy() + env[key] = value + # Use C locale to get ASCII for the locale encoding to force + # surrogate-escaping of \xFF in the child process; otherwise it can + # be decoded as-is if the default locale is latin-1. + env['LC_ALL'] = 'C' + if sys.platform.startswith("aix"): + # On AIX, the C locale uses the Latin1 encoding + decoded_value = encoded_value.decode("latin1", "surrogateescape") + else: + # On other UNIXes, the C locale uses the ASCII encoding + decoded_value = value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(decoded_value)) + + # test bytes + key = key.encode("ascii", "surrogateescape") + script = "import os; print(ascii(os.getenvb(%s)))" % repr(key) + env = os.environ.copy() + env[key] = encoded_value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(encoded_value)) + + def test_bytes_program(self): + abs_program = os.fsencode(sys.executable) + path, program = os.path.split(sys.executable) + program = os.fsencode(program) + + # absolute bytes path + exitcode = subprocess.call([abs_program, "-c", "pass"]) + self.assertEqual(exitcode, 0) + + # absolute bytes path as a string + cmd = b"'" + abs_program + b"' -c pass" + exitcode = subprocess.call(cmd, shell=True) + self.assertEqual(exitcode, 0) + + # bytes program, unicode PATH + env = os.environ.copy() + env["PATH"] = path + exitcode = subprocess.call([program, "-c", "pass"], env=env) + self.assertEqual(exitcode, 0) + + # bytes program, bytes PATH + envb = os.environb.copy() + envb[b"PATH"] = os.fsencode(path) + exitcode = subprocess.call([program, "-c", "pass"], env=envb) + self.assertEqual(exitcode, 0) + + def test_pipe_cloexec(self): + sleeper = support.findfile("input_reader.py", subdir="subprocessdata") + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + p1 = subprocess.Popen([sys.executable, sleeper], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=False) + + self.addCleanup(p1.communicate, b'') + + p2 = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + + output, error = p2.communicate() + result_fds = set(map(int, output.split(b','))) + unwanted_fds = set([p1.stdin.fileno(), p1.stdout.fileno(), + p1.stderr.fileno()]) + + self.assertFalse(result_fds & unwanted_fds, + "Expected no fds from %r to be open in child, " + "found %r" % + (unwanted_fds, result_fds & unwanted_fds)) + + def test_pipe_cloexec_real_tools(self): + qcat = support.findfile("qcat.py", subdir="subprocessdata") + qgrep = support.findfile("qgrep.py", subdir="subprocessdata") + + subdata = b'zxcvbn' + data = subdata * 4 + b'\n' + + p1 = subprocess.Popen([sys.executable, qcat], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + close_fds=False) + + p2 = subprocess.Popen([sys.executable, qgrep, subdata], + stdin=p1.stdout, stdout=subprocess.PIPE, + close_fds=False) + + self.addCleanup(p1.wait) + self.addCleanup(p2.wait) + def kill_p1(): + try: + p1.terminate() + except ProcessLookupError: + pass + def kill_p2(): + try: + p2.terminate() + except ProcessLookupError: + pass + self.addCleanup(kill_p1) + self.addCleanup(kill_p2) + + p1.stdin.write(data) + p1.stdin.close() + + readfiles, ignored1, ignored2 = select.select([p2.stdout], [], [], 10) + + self.assertTrue(readfiles, "The child hung") + self.assertEqual(p2.stdout.read(), data) + + p1.stdout.close() + p2.stdout.close() + + def test_close_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + + open_fds = set(fds) + # add a bunch more fds + for _ in range(9): + fd = os.open(os.devnull, os.O_RDONLY) + self.addCleanup(os.close, fd) + open_fds.add(fd) + + for fd in open_fds: + os.set_inheritable(fd, True) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertEqual(remaining_fds & open_fds, open_fds, + "Some fds were closed") + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & open_fds, + "Some fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + # Keep some of the fd's we opened open in the subprocess. + # This tests _posixsubprocess.c's proper handling of fds_to_keep. + fds_to_keep = set(open_fds.pop() for _ in range(8)) + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=()) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & fds_to_keep & open_fds, + "Some fds not in pass_fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD.") + def test_close_fds_when_max_fd_is_lowered(self): + """Confirm that issue21618 is fixed (may fail under valgrind).""" + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # This launches the meat of the test in a child process to + # avoid messing with the larger unittest processes maximum + # number of file descriptors. + # This process launches: + # +--> Process that lowers its RLIMIT_NOFILE aftr setting up + # a bunch of high open fds above the new lower rlimit. + # Those are reported via stdout before launching a new + # process with close_fds=False to run the actual test: + # +--> The TEST: This one launches a fd_status.py + # subprocess with close_fds=True so we can find out if + # any of the fds above the lowered rlimit are still open. + p = subprocess.Popen([sys.executable, '-c', textwrap.dedent( + ''' + import os, resource, subprocess, sys, textwrap + open_fds = set() + # Add a bunch more fds to pass down. + for _ in range(40): + fd = os.open(os.devnull, os.O_RDONLY) + open_fds.add(fd) + + # Leave a two pairs of low ones available for use by the + # internal child error pipe and the stdout pipe. + # We also leave 10 more open as some Python buildbots run into + # "too many open files" errors during the test if we do not. + for fd in sorted(open_fds)[:14]: + os.close(fd) + open_fds.remove(fd) + + for fd in open_fds: + #self.addCleanup(os.close, fd) + os.set_inheritable(fd, True) + + max_fd_open = max(open_fds) + + # Communicate the open_fds to the parent unittest.TestCase process. + print(','.join(map(str, sorted(open_fds)))) + sys.stdout.flush() + + rlim_cur, rlim_max = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + # 29 is lower than the highest fds we are leaving open. + resource.setrlimit(resource.RLIMIT_NOFILE, (29, rlim_max)) + # Launch a new Python interpreter with our low fd rlim_cur that + # inherits open fds above that limit. It then uses subprocess + # with close_fds=True to get a report of open fds in the child. + # An explicit list of fds to check is passed to fd_status.py as + # letting fd_status rely on its default logic would miss the + # fds above rlim_cur as it normally only checks up to that limit. + subprocess.Popen( + [sys.executable, '-c', + textwrap.dedent(""" + import subprocess, sys + subprocess.Popen([sys.executable, %r] + + [str(x) for x in range({max_fd})], + close_fds=True).wait() + """.format(max_fd=max_fd_open+1))], + close_fds=False).wait() + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, (rlim_cur, rlim_max)) + ''' % fd_status)], stdout=subprocess.PIPE) + + output, unused_stderr = p.communicate() + output_lines = output.splitlines() + self.assertEqual(len(output_lines), 2, + msg="expected exactly two lines of output:\n%r" % output) + opened_fds = set(map(int, output_lines[0].strip().split(b','))) + remaining_fds = set(map(int, output_lines[1].strip().split(b','))) + + self.assertFalse(remaining_fds & opened_fds, + msg="Some fds were left open.") + + + # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file + # descriptor of a pipe closed in the parent process is valid in the + # child process according to fstat(), but the mode of the file + # descriptor is invalid, and read or write raise an error. + @support.requires_mac_ver(10, 5) + def test_pass_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + open_fds = set() + + for x in range(5): + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + os.set_inheritable(fds[0], True) + os.set_inheritable(fds[1], True) + open_fds.update(fds) + + for fd in open_fds: + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=(fd, )) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + to_be_closed = open_fds - {fd} + + self.assertIn(fd, remaining_fds, "fd to be passed not passed") + self.assertFalse(remaining_fds & to_be_closed, + "fd to be closed passed") + + # pass_fds overrides close_fds with a warning. + with self.assertWarns(RuntimeWarning) as context: + self.assertFalse(subprocess.call( + [sys.executable, "-c", "import sys; sys.exit(0)"], + close_fds=False, pass_fds=(fd, ))) + self.assertIn('overriding close_fds', str(context.warning)) + + def test_pass_fds_inheritable(self): + script = support.findfile("fd_status.py", subdir="subprocessdata") + + inheritable, non_inheritable = os.pipe() + self.addCleanup(os.close, inheritable) + self.addCleanup(os.close, non_inheritable) + os.set_inheritable(inheritable, True) + os.set_inheritable(non_inheritable, False) + pass_fds = (inheritable, non_inheritable) + args = [sys.executable, script] + args += list(map(str, pass_fds)) + + p = subprocess.Popen(args, + stdout=subprocess.PIPE, close_fds=True, + pass_fds=pass_fds) + output, ignored = p.communicate() + fds = set(map(int, output.split(b','))) + + # the inheritable file descriptor must be inherited, so its inheritable + # flag must be set in the child process after fork() and before exec() + self.assertEqual(fds, set(pass_fds), "output=%a" % output) + + # inheritable flag must not be changed in the parent process + self.assertEqual(os.get_inheritable(inheritable), True) + self.assertEqual(os.get_inheritable(non_inheritable), False) + + def test_stdout_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stdin=inout) + p.wait() + + def test_stdout_stderr_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stderr=inout) + p.wait() + + def test_stderr_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stderr=inout, stdin=inout) + p.wait() + + def test_wait_when_sigchild_ignored(self): + # NOTE: sigchild_ignore.py may not be an effective test on all OSes. + sigchild_ignore = support.findfile("sigchild_ignore.py", + subdir="subprocessdata") + p = subprocess.Popen([sys.executable, sigchild_ignore], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" + " non-zero with this error:\n%s" % + stderr.decode('utf-8')) + + def test_select_unbuffered(self): + # Issue #11459: bufsize=0 should really set the pipes as + # unbuffered (and therefore let select() work properly). + select = support.import_module("select") + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple")'], + stdout=subprocess.PIPE, + bufsize=0) + f = p.stdout + self.addCleanup(f.close) + try: + self.assertEqual(f.read(4), b"appl") + self.assertIn(f, select.select([f], [], [], 0.0)[0]) + finally: + p.wait() + + def test_zombie_fast_process_del(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, it wouldn't be added to subprocess._active, and would + # remain a zombie. + # spawn a Popen, and delete its reference before it exits + p = subprocess.Popen([sys.executable, "-c", + 'import sys, time;' + 'time.sleep(0.2)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + def test_leak_fast_process_del_killed(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, and the process got killed by a signal, it would never + # be removed from subprocess._active, which triggered a FD and memory + # leak. + # spawn a Popen, delete its reference and kill it + p = subprocess.Popen([sys.executable, "-c", + 'import time;' + 'time.sleep(3)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + os.kill(pid, signal.SIGKILL) + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + # let some time for the process to exit, and create a new Popen: this + # should trigger the wait() of p + time.sleep(0.2) + with self.assertRaises(OSError) as c: + with subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + # p should have been wait()ed on, and removed from the _active list + self.assertRaises(OSError, os.waitpid, pid, 0) + self.assertNotIn(ident, [id(o) for o in subprocess._active]) + + def test_close_fds_after_preexec(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # this FD is used as dup2() target by preexec_fn, and should be closed + # in the child process + fd = os.dup(1) + self.addCleanup(os.close, fd) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + preexec_fn=lambda: os.dup2(1, fd)) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + + self.assertNotIn(fd, remaining_fds) + + @support.cpython_only + def test_fork_exec(self): + # Issue #22290: fork_exec() must not crash on memory allocation failure + # or other errors + import _posixsubprocess + gc_enabled = gc.isenabled() + try: + # Use a preexec function and enable the garbage collector + # to force fork_exec() to re-enable the garbage collector + # on error. + func = lambda: None + gc.enable() + + executable_list = "exec" # error: must be a sequence + + for args, exe_list, cwd, env_list in ( + (123, [b"exe"], None, [b"env"]), + ([b"arg"], 123, None, [b"env"]), + ([b"arg"], [b"exe"], 123, [b"env"]), + ([b"arg"], [b"exe"], None, 123), + ): + with self.assertRaises(TypeError): + _posixsubprocess.fork_exec( + args, exe_list, + True, [], cwd, env_list, + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, func) + finally: + if not gc_enabled: + gc.disable() + + + +@unittest.skipUnless(mswindows, "Windows specific tests") +class Win32ProcessTestCase(BaseTestCase): + + def test_startupinfo(self): + # startupinfo argument + # We uses hardcoded constants, because we do not want to + # depend on win32all. + STARTF_USESHOWWINDOW = 1 + SW_MAXIMIZE = 3 + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = STARTF_USESHOWWINDOW + startupinfo.wShowWindow = SW_MAXIMIZE + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_creationflags(self): + # creationflags argument + CREATE_NEW_CONSOLE = 16 + sys.stderr.write(" a DOS box should flash briefly ...\n") + subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + preexec_fn=lambda: 1) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + stdout=subprocess.PIPE, + close_fds=True) + + def test_close_fds(self): + # close file descriptors + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"], + close_fds=True) + self.assertEqual(rc, 47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen(["set"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn(b"physalis", p.stdout.read()) + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn(b"physalis", p.stdout.read()) + + def test_call_string(self): + # call() function with string argument on Windows + rc = subprocess.call(sys.executable + + ' -c "import sys; sys.exit(47)"') + self.assertEqual(rc, 47) + + def _kill_process(self, method, *args): + # Some win32 buildbot raises EOFError if stdin is inherited + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + returncode = p.wait() + self.assertNotEqual(returncode, 0) + + def _kill_dead_process(self, method, *args): + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + sys.exit(42) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() + self.assertEqual(rc, 42) + + def test_send_signal(self): + self._kill_process('send_signal', signal.SIGTERM) + + def test_kill(self): + self._kill_process('kill') + + def test_terminate(self): + self._kill_process('terminate') + + def test_send_signal_dead(self): + self._kill_dead_process('send_signal', signal.SIGTERM) + + def test_kill_dead(self): + self._kill_dead_process('kill') + + def test_terminate_dead(self): + self._kill_dead_process('terminate') + +class CommandTests(unittest.TestCase): + def test_getoutput(self): + self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') + self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), + (0, 'xyzzy')) + + # we use mkdtemp in the next line to create an empty directory + # under our exclusive control; from that, we can invent a pathname + # that we _know_ won't exist. This is guaranteed to fail. + dir = None + try: + dir = tempfile.mkdtemp() + name = os.path.join(dir, "foo") + status, output = subprocess.getstatusoutput( + ("type " if mswindows else "cat ") + name) + self.assertNotEqual(status, 0) + finally: + if dir is not None: + os.rmdir(dir) + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class ProcessTestCaseNoPoll(ProcessTestCase): + def setUp(self): + self.orig_selector = subprocess._PopenSelector + subprocess._PopenSelector = selectors.SelectSelector + ProcessTestCase.setUp(self) + + def tearDown(self): + subprocess._PopenSelector = self.orig_selector + ProcessTestCase.tearDown(self) + + +class HelperFunctionTests(unittest.TestCase): + @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows") + def test_eintr_retry_call(self): + record_calls = [] + def fake_os_func(*args): + record_calls.append(args) + if len(record_calls) == 2: + raise OSError(errno.EINTR, "fake interrupted system call") + return tuple(reversed(args)) + + self.assertEqual((999, 256), + subprocess._eintr_retry_call(fake_os_func, 256, 999)) + self.assertEqual([(256, 999)], record_calls) + # This time there will be an EINTR so it will loop once. + self.assertEqual((666,), + subprocess._eintr_retry_call(fake_os_func, 666)) + self.assertEqual([(256, 999), (666,), (666,)], record_calls) + + +@unittest.skipUnless(mswindows, "Windows-specific tests") +class CommandsWithSpaces (BaseTestCase): + + def setUp(self): + super().setUp() + f, fname = mkstemp(".py", "te st") + self.fname = fname.lower () + os.write(f, b"import sys;" + b"sys.stdout.write('%d %s' % (len(sys.argv), [a.lower () for a in sys.argv]))" + ) + os.close(f) + + def tearDown(self): + os.remove(self.fname) + super().tearDown() + + def with_spaces(self, *args, **kwargs): + kwargs['stdout'] = subprocess.PIPE + p = subprocess.Popen(*args, **kwargs) + self.addCleanup(p.stdout.close) + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) + + def test_shell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd"), shell=1) + + def test_shell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) + + def test_noshell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd")) + + def test_noshell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"]) + + +class ContextManagerTests(BaseTestCase): + + def test_pipe(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write('stdout');" + "sys.stderr.write('stderr');"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + self.assertEqual(proc.stdout.read(), b"stdout") + self.assertStderrEqual(proc.stderr.read(), b"stderr") + + self.assertTrue(proc.stdout.closed) + self.assertTrue(proc.stderr.closed) + + def test_returncode(self): + with subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(100)"]) as proc: + pass + # __exit__ calls wait(), so the returncode should be set + self.assertEqual(proc.returncode, 100) + + def test_communicate_stdin(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.exit(sys.stdin.read() == 'context')"], + stdin=subprocess.PIPE) as proc: + proc.communicate(b"context") + self.assertEqual(proc.returncode, 1) + + def test_invalid_args(self): + with self.assertRaises(FileNotFoundError) as c: + with subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + + def test_broken_pipe_cleanup(self): + """Broken pipe error should not prevent wait() (Issue 21619)""" + proc = subprocess.Popen([sys.executable, '-c', 'pass'], + stdin=subprocess.PIPE, + bufsize=support.PIPE_MAX_SIZE*2) + proc = proc.__enter__() + # Prepare to send enough data to overflow any OS pipe buffering and + # guarantee a broken pipe error. Data is held in BufferedWriter + # buffer until closed. + proc.stdin.write(b'x' * support.PIPE_MAX_SIZE) + self.assertIsNone(proc.returncode) + # EPIPE expected under POSIX; EINVAL under Windows + self.assertRaises(OSError, proc.__exit__, None, None, None) + self.assertEqual(proc.returncode, 0) + self.assertTrue(proc.stdin.closed) + + +def test_main(): + unit_tests = (ProcessTestCase, + POSIXProcessTestCase, + Win32ProcessTestCase, + CommandTests, + ProcessTestCaseNoPoll, + HelperFunctionTests, + CommandsWithSpaces, + ContextManagerTests, + ) + + support.run_unittest(*unit_tests) + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.4/test_threading.py b/src/greentest/3.4/test_threading.py new file mode 100644 index 0000000..3a71e06 --- /dev/null +++ b/src/greentest/3.4/test_threading.py @@ -0,0 +1,1106 @@ +""" +Tests for the threading module. +""" + +import test.support +from test.support import verbose, strip_python_stderr, import_module, cpython_only +from test.script_helper import assert_python_ok, assert_python_failure + +import random +import re +import sys +_thread = import_module('_thread') +threading = import_module('threading') +import time +import unittest +import weakref +import os +import subprocess + +from test import lock_tests + + +# Between fork() and exec(), only async-safe functions are allowed (issues +# #12316 and #11870), and fork() from a worker thread is known to trigger +# problems with some operating systems (issue #3863): skip problematic tests +# on platforms known to behave badly. +platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', + 'hp-ux11') + + +# A trivial mutable counter. +class Counter(object): + def __init__(self): + self.value = 0 + def inc(self): + self.value += 1 + def dec(self): + self.value -= 1 + def get(self): + return self.value + +class TestThread(threading.Thread): + def __init__(self, name, testcase, sema, mutex, nrunning): + threading.Thread.__init__(self, name=name) + self.testcase = testcase + self.sema = sema + self.mutex = mutex + self.nrunning = nrunning + + def run(self): + delay = random.random() / 10000.0 + if verbose: + print('task %s will run for %.1f usec' % + (self.name, delay * 1e6)) + + with self.sema: + with self.mutex: + self.nrunning.inc() + if verbose: + print(self.nrunning.get(), 'tasks are running') + self.testcase.assertTrue(self.nrunning.get() <= 3) + + time.sleep(delay) + if verbose: + print('task', self.name, 'done') + + with self.mutex: + self.nrunning.dec() + self.testcase.assertTrue(self.nrunning.get() >= 0) + if verbose: + print('%s is finished. %d tasks are running' % + (self.name, self.nrunning.get())) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test.support.threading_setup() + + def tearDown(self): + test.support.threading_cleanup(*self._threads) + test.support.reap_children() + + +class ThreadTests(BaseTestCase): + + # Create a bunch of threads, let each do some work, wait until all are + # done. + def test_various_ops(self): + # This takes about n/3 seconds to run (about n/3 clumps of tasks, + # times about 1 second per clump). + NUMTASKS = 10 + + # no more than 3 of the 10 can run at once + sema = threading.BoundedSemaphore(value=3) + mutex = threading.RLock() + numrunning = Counter() + + threads = [] + + for i in range(NUMTASKS): + t = TestThread(""%i, self, sema, mutex, numrunning) + threads.append(t) + self.assertEqual(t.ident, None) + self.assertTrue(re.match('', repr(t))) + t.start() + + if verbose: + print('waiting for all tasks to complete') + for t in threads: + t.join() + self.assertTrue(not t.is_alive()) + self.assertNotEqual(t.ident, 0) + self.assertFalse(t.ident is None) + self.assertTrue(re.match('', + repr(t))) + if verbose: + print('all tasks done') + self.assertEqual(numrunning.get(), 0) + + def test_ident_of_no_threading_threads(self): + # The ident still must work for the main thread and dummy threads. + self.assertFalse(threading.currentThread().ident is None) + def f(): + ident.append(threading.currentThread().ident) + done.set() + done = threading.Event() + ident = [] + _thread.start_new_thread(f, ()) + done.wait() + self.assertFalse(ident[0] is None) + # Kill the "immortal" _DummyThread + del threading._active[ident[0]] + + # run with a small(ish) thread stack size (256kB) + def test_various_ops_small_stack(self): + if verbose: + print('with 256kB thread stack size...') + try: + threading.stack_size(262144) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1MB) + def test_various_ops_large_stack(self): + if verbose: + print('with 1MB thread stack size...') + try: + threading.stack_size(0x100000) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + def test_foreign_thread(self): + # Check that a "foreign" thread can use the threading module. + def f(mutex): + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + threading.current_thread() + mutex.release() + + mutex = threading.Lock() + mutex.acquire() + tid = _thread.start_new_thread(f, (mutex,)) + # Wait for the thread to finish. + mutex.acquire() + self.assertIn(tid, threading._active) + self.assertIsInstance(threading._active[tid], threading._DummyThread) + del threading._active[tid] + + # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) + # exposed at the Python level. This test relies on ctypes to get at it. + def test_PyThreadState_SetAsyncExc(self): + ctypes = import_module("ctypes") + + set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc + + class AsyncExc(Exception): + pass + + exception = ctypes.py_object(AsyncExc) + + # First check it works when setting the exception from the same thread. + tid = threading.get_ident() + + try: + result = set_async_exc(ctypes.c_long(tid), exception) + # The exception is async, so we might have to keep the VM busy until + # it notices. + while True: + pass + except AsyncExc: + pass + else: + # This code is unreachable but it reflects the intent. If we wanted + # to be smarter the above loop wouldn't be infinite. + self.fail("AsyncExc not raised") + try: + self.assertEqual(result, 1) # one thread state modified + except UnboundLocalError: + # The exception was raised too quickly for us to get the result. + pass + + # `worker_started` is set by the thread when it's inside a try/except + # block waiting to catch the asynchronously set AsyncExc exception. + # `worker_saw_exception` is set by the thread upon catching that + # exception. + worker_started = threading.Event() + worker_saw_exception = threading.Event() + + class Worker(threading.Thread): + def run(self): + self.id = threading.get_ident() + self.finished = False + + try: + while True: + worker_started.set() + time.sleep(0.1) + except AsyncExc: + self.finished = True + worker_saw_exception.set() + + t = Worker() + t.daemon = True # so if this fails, we don't hang Python at shutdown + t.start() + if verbose: + print(" started worker thread") + + # Try a thread id that doesn't make sense. + if verbose: + print(" trying nonsensical thread id") + result = set_async_exc(ctypes.c_long(-1), exception) + self.assertEqual(result, 0) # no thread states modified + + # Now raise an exception in the worker thread. + if verbose: + print(" waiting for worker thread to get started") + ret = worker_started.wait() + self.assertTrue(ret) + if verbose: + print(" verifying worker hasn't exited") + self.assertTrue(not t.finished) + if verbose: + print(" attempting to raise asynch exception in worker") + result = set_async_exc(ctypes.c_long(t.id), exception) + self.assertEqual(result, 1) # one thread state modified + if verbose: + print(" waiting for worker to say it caught the exception") + worker_saw_exception.wait(timeout=10) + self.assertTrue(t.finished) + if verbose: + print(" all OK -- joining worker") + if t.finished: + t.join() + # else the thread is still running, and we have no way to kill it + + def test_limbo_cleanup(self): + # Issue 7481: Failure to start thread should cleanup the limbo map. + def fail_new_thread(*args): + raise threading.ThreadError() + _start_new_thread = threading._start_new_thread + threading._start_new_thread = fail_new_thread + try: + t = threading.Thread(target=lambda: None) + self.assertRaises(threading.ThreadError, t.start) + self.assertFalse( + t in threading._limbo, + "Failed to cleanup _limbo map on failure of Thread.start().") + finally: + threading._start_new_thread = _start_new_thread + + def test_finalize_runnning_thread(self): + # Issue 1402: the PyGILState_Ensure / _Release functions may be called + # very late on python exit: on deallocation of a running thread for + # example. + import_module("ctypes") + + rc, out, err = assert_python_failure("-c", """if 1: + import ctypes, sys, time, _thread + + # This lock is used as a simple event variable. + ready = _thread.allocate_lock() + ready.acquire() + + # Module globals are cleared before __del__ is run + # So we save the functions in class dict + class C: + ensure = ctypes.pythonapi.PyGILState_Ensure + release = ctypes.pythonapi.PyGILState_Release + def __del__(self): + state = self.ensure() + self.release(state) + + def waitingThread(): + x = C() + ready.release() + time.sleep(100) + + _thread.start_new_thread(waitingThread, ()) + ready.acquire() # Be sure the other thread is waiting. + sys.exit(42) + """) + self.assertEqual(rc, 42) + + def test_finalize_with_trace(self): + # Issue1733757 + # Avoid a deadlock when sys.settrace steps into threading._shutdown + assert_python_ok("-c", """if 1: + import sys, threading + + # A deadlock-killer, to prevent the + # testsuite to hang forever + def killer(): + import os, time + time.sleep(2) + print('program blocked; aborting') + os._exit(2) + t = threading.Thread(target=killer) + t.daemon = True + t.start() + + # This is the trace function + def func(frame, event, arg): + threading.current_thread() + return func + + sys.settrace(func) + """) + + def test_join_nondaemon_on_shutdown(self): + # Issue 1722344 + # Raising SystemExit skipped threading._shutdown + rc, out, err = assert_python_ok("-c", """if 1: + import threading + from time import sleep + + def child(): + sleep(1) + # As a non-daemon thread we SHOULD wake up and nothing + # should be torn down yet + print("Woke up, sleep function is:", sleep) + + threading.Thread(target=child).start() + raise SystemExit + """) + self.assertEqual(out.strip(), + b"Woke up, sleep function is: ") + self.assertEqual(err, b"") + + def test_enumerate_after_join(self): + # Try hard to trigger #1703448: a thread is still returned in + # threading.enumerate() after it has been join()ed. + enum = threading.enumerate + old_interval = sys.getswitchinterval() + try: + for i in range(1, 100): + sys.setswitchinterval(i * 0.0002) + t = threading.Thread(target=lambda: None) + t.start() + t.join() + l = enum() + self.assertNotIn(t, l, + "#1703448 triggered after %d trials: %s" % (i, l)) + finally: + sys.setswitchinterval(old_interval) + + def test_no_refcycle_through_target(self): + class RunSelfFunction(object): + def __init__(self, should_raise): + # The links in this refcycle from Thread back to self + # should be cleaned up when the thread completes. + self.should_raise = should_raise + self.thread = threading.Thread(target=self._run, + args=(self,), + kwargs={'yet_another':self}) + self.thread.start() + + def _run(self, other_ref, yet_another): + if self.should_raise: + raise SystemExit + + cyclic_object = RunSelfFunction(should_raise=False) + weak_cyclic_object = weakref.ref(cyclic_object) + cyclic_object.thread.join() + del cyclic_object + self.assertIsNone(weak_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_cyclic_object()))) + + raising_cyclic_object = RunSelfFunction(should_raise=True) + weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) + raising_cyclic_object.thread.join() + del raising_cyclic_object + self.assertIsNone(weak_raising_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_raising_cyclic_object()))) + + def test_old_threading_api(self): + # Just a quick sanity check to make sure the old method names are + # still present + t = threading.Thread() + t.isDaemon() + t.setDaemon(True) + t.getName() + t.setName("name") + t.isAlive() + e = threading.Event() + e.isSet() + threading.activeCount() + + def test_repr_daemon(self): + t = threading.Thread() + self.assertFalse('daemon' in repr(t)) + t.daemon = True + self.assertTrue('daemon' in repr(t)) + + def test_deamon_param(self): + t = threading.Thread() + self.assertFalse(t.daemon) + t = threading.Thread(daemon=False) + self.assertFalse(t.daemon) + t = threading.Thread(daemon=True) + self.assertTrue(t.daemon) + + @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + def test_dummy_thread_after_fork(self): + # Issue #14308: a dummy thread in the active list doesn't mess up + # the after-fork mechanism. + code = """if 1: + import _thread, threading, os, time + + def background_thread(evt): + # Creates and registers the _DummyThread instance + threading.current_thread() + evt.set() + time.sleep(10) + + evt = threading.Event() + _thread.start_new_thread(background_thread, (evt,)) + evt.wait() + assert threading.active_count() == 2, threading.active_count() + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + os.wait() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + old_interval = sys.getswitchinterval() + self.addCleanup(sys.setswitchinterval, old_interval) + + # Make the bug more likely to manifest. + sys.setswitchinterval(1e-6) + + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + self.addCleanup(t.join) + pid = os.fork() + if pid == 0: + os._exit(1 if t.is_alive() else 0) + else: + pid, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + def test_main_thread(self): + main = threading.main_thread() + self.assertEqual(main.name, 'MainThread') + self.assertEqual(main.ident, threading.current_thread().ident) + self.assertEqual(main.ident, threading.get_ident()) + + def f(): + self.assertNotEqual(threading.main_thread().ident, + threading.current_thread().ident) + th = threading.Thread(target=f) + th.start() + th.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork(self): + code = """if 1: + import os, threading + + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + else: + os.waitpid(pid, 0) + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "MainThread\nTrue\nTrue\n") + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_nonmain_thread(self): + code = """if 1: + import os, threading, sys + + def f(): + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + os.waitpid(pid, 0) + + th = threading.Thread(target=f) + th.start() + th.join() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertTrue(t._tstate_lock is None) + + def test_repr_stopped(self): + # Verify that "stopped" shows up in repr(Thread) appropriately. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + t = threading.Thread(target=f) + t.start() + started.acquire() + self.assertIn("started", repr(t)) + finish.release() + # "stopped" should appear in the repr in a reasonable amount of time. + # Implementation detail: as of this writing, that's trivially true + # if .join() is called, and almost trivially true if .is_alive() is + # called. The detail we're testing here is that "stopped" shows up + # "all on its own". + LOOKING_FOR = "stopped" + for i in range(500): + if LOOKING_FOR in repr(t): + break + time.sleep(0.01) + self.assertIn(LOOKING_FOR, repr(t)) # we waited at least 5 seconds + + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) + + @cpython_only + def test_frame_tstate_tracing(self): + # Issue #14432: Crash when a generator is created in a C thread that is + # destroyed while the generator is still used. The issue was that a + # generator contains a frame, and the frame kept a reference to the + # Python state of the destroyed C thread. The crash occurs when a trace + # function is setup. + + def noop_trace(frame, event, arg): + # no operation + return noop_trace + + def generator(): + while 1: + yield "generator" + + def callback(): + if callback.gen is None: + callback.gen = generator() + return next(callback.gen) + callback.gen = None + + old_trace = sys.gettrace() + sys.settrace(noop_trace) + try: + # Install a trace function + threading.settrace(noop_trace) + + # Create a generator in a C thread which exits after the call + import _testcapi + _testcapi.call_in_temporary_c_thread(callback) + + # Call the generator in a different Python thread, check that the + # generator didn't keep a reference to the destroyed thread state + for test in range(3): + # The trace function is still called here + callback() + finally: + sys.settrace(old_trace) + + +class ThreadJoinOnShutdown(BaseTestCase): + + def _run_and_join(self, script): + script = """if 1: + import sys, os, time, threading + + # a thread, which waits for the main program to terminate + def joiningfunc(mainthread): + mainthread.join() + print('end of thread') + # stdout is fully buffered because not a tty, we have to flush + # before exit. + sys.stdout.flush() + \n""" + script + + rc, out, err = assert_python_ok("-c", script) + data = out.decode().replace('\r', '') + self.assertEqual(data, "end of main\nend of thread\n") + + def test_1_join_on_shutdown(self): + # The usual case: on exit, wait for a non-daemon thread + script = """if 1: + import os + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + time.sleep(0.1) + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_2_join_in_forked_process(self): + # Like the test above, but from a forked interpreter + script = """if 1: + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_3_join_in_forked_from_thread(self): + # Like the test above, but fork() was called from a worker thread + # In the forked process, the main Thread object must be marked as stopped. + + script = """if 1: + main_thread = threading.current_thread() + def worker(): + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(main_thread,)) + print('end of main') + t.start() + t.join() # Should not block: main_thread is already stopped + + w = threading.Thread(target=worker) + w.start() + """ + self._run_and_join(script) + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_4_daemon_threads(self): + # Check that a daemon thread cannot crash the interpreter on shutdown + # by manipulating internal structures that are being disposed of in + # the main thread. + script = """if True: + import os + import random + import sys + import time + import threading + + thread_has_run = set() + + def random_io(): + '''Loop for a while sleeping random tiny amounts and doing some I/O.''' + while True: + in_f = open(os.__file__, 'rb') + stuff = in_f.read(200) + null_f = open(os.devnull, 'wb') + null_f.write(stuff) + time.sleep(random.random() / 1995) + null_f.close() + in_f.close() + thread_has_run.add(threading.current_thread()) + + def main(): + count = 0 + for _ in range(40): + new_thread = threading.Thread(target=random_io) + new_thread.daemon = True + new_thread.start() + count += 1 + while len(thread_has_run) < count: + time.sleep(0.001) + # Trigger process shutdown + sys.exit(0) + + main() + """ + rc, out, err = assert_python_ok('-c', script) + self.assertFalse(err) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_reinit_tls_after_fork(self): + # Issue #13817: fork() would deadlock in a multithreaded program with + # the ad-hoc TLS implementation. + + def do_fork_and_wait(): + # just fork a child process and wait it + pid = os.fork() + if pid > 0: + os.waitpid(pid, 0) + else: + os._exit(0) + + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() + + for t in threads: + t.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_clear_threads_states_after_fork(self): + # Issue #17094: check that threads states are cleared after fork() + + # start a bunch of threads + threads = [] + for i in range(16): + t = threading.Thread(target=lambda : time.sleep(0.3)) + threads.append(t) + t.start() + + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(0) + else: + os._exit(1) + else: + _, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + for t in threads: + t.join() + + +class SubinterpThreadingTests(BaseTestCase): + + def test_threads_join(self): + # Non-daemon threads should be joined at subinterpreter shutdown + # (issue #18808) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + @cpython_only + def test_daemon_threads_fatal_error(self): + subinterp_code = r"""if 1: + import os + import threading + import time + + def f(): + # Make sure the daemon thread is still running when + # Py_EndInterpreter is called. + time.sleep(10) + threading.Thread(target=f, daemon=True).start() + """ + script = r"""if 1: + import _testcapi + + _testcapi.run_in_subinterp(%r) + """ % (subinterp_code,) + with test.support.SuppressCrashReport(): + rc, out, err = assert_python_failure("-c", script) + self.assertIn("Fatal Python error: Py_EndInterpreter: " + "not the last thread", err.decode()) + + +class ThreadingExceptionTests(BaseTestCase): + # A RuntimeError should be raised if Thread.start() is called + # multiple times. + def test_start_thread_again(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, thread.start) + + def test_joining_current_thread(self): + current_thread = threading.current_thread() + self.assertRaises(RuntimeError, current_thread.join); + + def test_joining_inactive_thread(self): + thread = threading.Thread() + self.assertRaises(RuntimeError, thread.join) + + def test_daemonize_active_thread(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + + def test_releasing_unacquired_lock(self): + lock = threading.Lock() + self.assertRaises(RuntimeError, lock.release) + + @unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(), + 'test macosx problem') + def test_recursion_limit(self): + # Issue 9670 + # test that excessive recursion within a non-main thread causes + # an exception rather than crashing the interpreter on platforms + # like Mac OS X or FreeBSD which have small default stack sizes + # for threads + script = """if True: + import threading + + def recurse(): + return recurse() + + def outer(): + try: + recurse() + except RuntimeError: + pass + + w = threading.Thread(target=outer) + w.start() + w.join() + print('end of main thread') + """ + expected_output = "end of main thread\n" + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + data = stdout.decode().replace('\r', '') + self.assertEqual(p.returncode, 0, "Unexpected error: " + stderr.decode()) + self.assertEqual(data, expected_output) + + def test_print_exception(self): + script = r"""if True: + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_1(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + sys.stderr = None + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_2(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + sys.stderr = None + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + self.assertNotIn("Unhandled exception", err.decode()) + + +class TimerTests(BaseTestCase): + + def setUp(self): + BaseTestCase.setUp(self) + self.callback_args = [] + self.callback_event = threading.Event() + + def test_init_immutable_default_args(self): + # Issue 17435: constructor defaults were mutable objects, they could be + # mutated via the object attributes and affect other Timer objects. + timer1 = threading.Timer(0.01, self._callback_spy) + timer1.start() + self.callback_event.wait() + timer1.args.append("blah") + timer1.kwargs["foo"] = "bar" + self.callback_event.clear() + timer2 = threading.Timer(0.01, self._callback_spy) + timer2.start() + self.callback_event.wait() + self.assertEqual(len(self.callback_args), 2) + self.assertEqual(self.callback_args, [((), {}), ((), {})]) + + def _callback_spy(self, *args, **kwargs): + self.callback_args.append((args[:], kwargs.copy())) + self.callback_event.set() + +class LockTests(lock_tests.LockTests): + locktype = staticmethod(threading.Lock) + +class PyRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._PyRLock) + +@unittest.skipIf(threading._CRLock is None, 'RLock not implemented in C') +class CRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._CRLock) + +class EventTests(lock_tests.EventTests): + eventtype = staticmethod(threading.Event) + + @unittest.skip("not on gevent") + def test_reset_internal_locks(self): + pass + +class ConditionAsRLockTests(lock_tests.RLockTests): + # An Condition uses an RLock by default and exports its API. + locktype = staticmethod(threading.Condition) + +class ConditionTests(lock_tests.ConditionTests): + condtype = staticmethod(threading.Condition) + +class SemaphoreTests(lock_tests.SemaphoreTests): + semtype = staticmethod(threading.Semaphore) + +class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): + semtype = staticmethod(threading.BoundedSemaphore) + +class BarrierTests(lock_tests.BarrierTests): + barriertype = staticmethod(threading.Barrier) + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.4/version b/src/greentest/3.4/version new file mode 100644 index 0000000..6cb9d3d --- /dev/null +++ b/src/greentest/3.4/version @@ -0,0 +1 @@ +3.4.3 diff --git a/src/greentest/3.5/allsans.pem b/src/greentest/3.5/allsans.pem new file mode 100644 index 0000000..3ee4f59 --- /dev/null +++ b/src/greentest/3.5/allsans.pem @@ -0,0 +1,37 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOoy7/QOtTjQ0niE +6uDcTwtkC0R2Tvy1AjVnXohCntZfdzbTGDoYTgXSOLsP8A697jUiJ8VCePGH50xG +Z4DKnAF3a9O3a9nr2pLXb0iY3XOMv+YEBii7CfI+3oxFYgCl0sMgHzDD2ZTVYAsm +DWgLUVsE2gHEccRwrM2tPf2EgR+FAgMBAAECgYEA3qyfyYVSeTrTYxO93x6ZaVMu +A2IZp9zSxMQL9bKiI2GRj+cV2ebSCGbg2btFnD6qBor7FWsmYz+8g6FNN/9sY4az +61rMqMtQvLBe+7L8w70FeTze4qQ4Y1oQri0qD6tBWhDVlpnbI5Py9bkZKD67yVUk +elcEA/5x4PrYXkuqsAECQQD80NjT0mDvaY0JOOaQFSEpMv6QiUA8GGX8Xli7IoKb +tAolPG8rQBa+qSpcWfDMTrWw/aWHuMEEQoP/bVDH9W4FAkEA7SYQbBAKnojZ5A3G +kOHdV7aeivRQxQk/JN8Fb8oKB9Csvpv/BsuGxPKXHdhFa6CBTTsNRtHQw/szPo4l +xMIjgQJAPoMxqibR+0EBM6+TKzteSL6oPXsCnBl4Vk/J5vPgkbmR7KUl4+7j8N8J +b2554TrxKEN/w7CGYZRE6UrRd7ATNQJAWD7Yz41sli+wfPdPU2xo1BHljyl4wMk/ +EPZYbI/PCbdyAH/F935WyQTIjNeEhZc1Zkq6FwdOWw8ns3hrv3rKgQJAHXv1BqUa +czGPIFxX2TNoqtcl6/En4vrxVB1wzsfzkkDAg98kBl7qsF+S3qujSzKikjeaVbI2 +/CyWR2P3yLtOmA== +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDcjCCAtugAwIBAgIJAN5dc9TOWjB7MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTYwODA1 +MTAyMTExWhcNMjYwODAzMTAyMTExWjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO +Q2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0 +aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQDqMu/0DrU40NJ4hOrg3E8LZAtEdk78tQI1Z16IQp7WX3c20xg6GE4F0ji7D/AO +ve41IifFQnjxh+dMRmeAypwBd2vTt2vZ69qS129ImN1zjL/mBAYouwnyPt6MRWIA +pdLDIB8ww9mU1WALJg1oC1FbBNoBxHHEcKzNrT39hIEfhQIDAQABo4IBODCCATQw +ggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBp +ZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMC +AQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUu +b3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMw +IQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGly +bmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAA +AAAAAAAAAAAAAAAAAYgEKgMEBTANBgkqhkiG9w0BAQsFAAOBgQAy16h+F+nOmeiT +VWR0fc8F/j6FcadbLseAUaogcC15OGxCl4UYpLV88HBkABOoGCpP155qwWTwOrdG +iYPGJSusf1OnJEbvzFejZf6u078bPd9/ZL4VWLjv+FPGkjd+N+/OaqMvgj8Lu99f +3Y/C4S7YbHxxwff6C6l2Xli+q6gnuQ== +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/badcert.pem b/src/greentest/3.5/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/greentest/3.5/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/badkey.pem b/src/greentest/3.5/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/greentest/3.5/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/capath/0e4015b9.0 b/src/greentest/3.5/capath/0e4015b9.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.5/capath/0e4015b9.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/capath/4e1295a3.0 b/src/greentest/3.5/capath/4e1295a3.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.5/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/capath/5ed36f99.0 b/src/greentest/3.5/capath/5ed36f99.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.5/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/capath/6e88d7b8.0 b/src/greentest/3.5/capath/6e88d7b8.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.5/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/capath/99d0fa06.0 b/src/greentest/3.5/capath/99d0fa06.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.5/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/capath/ce7b8643.0 b/src/greentest/3.5/capath/ce7b8643.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.5/capath/ce7b8643.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/dh1024.pem b/src/greentest/3.5/dh1024.pem new file mode 100644 index 0000000..a391176 --- /dev/null +++ b/src/greentest/3.5/dh1024.pem @@ -0,0 +1,7 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAIbzw1s9CT8SV5yv6L7esdAdZYZjPi3qWFs61CYTFFQnf2s/d09NYaJt +rrvJhIzWavqnue71qXCf83/J3nz3FEwUU/L0mGyheVbsSHiI64wUo3u50wK5Igo0 +RNs/LD0irs7m0icZ//hijafTU+JOBiuA8zMI+oZfU7BGuc9XrUprAgEC +-----END DH PARAMETERS----- + +Generated with: openssl dhparam -out dh1024.pem 1024 diff --git a/src/greentest/3.5/keycert.passwd.pem b/src/greentest/3.5/keycert.passwd.pem new file mode 100644 index 0000000..e905748 --- /dev/null +++ b/src/greentest/3.5/keycert.passwd.pem @@ -0,0 +1,33 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/keycert.pem b/src/greentest/3.5/keycert.pem new file mode 100644 index 0000000..64318aa --- /dev/null +++ b/src/greentest/3.5/keycert.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/keycert2.pem b/src/greentest/3.5/keycert2.pem new file mode 100644 index 0000000..e8a9e08 --- /dev/null +++ b/src/greentest/3.5/keycert2.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAJnsJZVrppL+W5I9 +zGQrrawWwE5QJpBK9nWw17mXrZ03R1cD9BamLGivVISbPlRlAVnZBEyh1ATpsB7d +CUQ+WHEvALquvx4+Yw5l+fXeiYRjrLRBYZuVy8yNtXzU3iWcGObcYRkUdiXdOyP7 +sLF2YZHRvQZpzgDBKkrraeQ81w21AgMBAAECgYBEm7n07FMHWlE+0kT0sXNsLYfy +YE+QKZnJw9WkaDN+zFEEPELkhZVt5BjsMraJr6v2fIEqF0gGGJPkbenffVq2B5dC +lWUOxvJHufMK4sM3Cp6s/gOp3LP+QkzVnvJSfAyZU6l+4PGX5pLdUsXYjPxgzjzL +S36tF7/2Uv1WePyLUQJBAMsPhYzUXOPRgmbhcJiqi9A9c3GO8kvSDYTCKt3VMnqz +HBn6MQ4VQasCD1F+7jWTI0FU/3vdw8non/Fj8hhYqZcCQQDCDRdvmZqDiZnpMqDq +L6ZSrLTVtMvZXZbgwForaAD9uHj51TME7+eYT7EG2YCgJTXJ4YvRJEnPNyskwdKt +vTSTAkEAtaaN/vyemEJ82BIGStwONNw0ILsSr5cZ9tBHzqiA/tipY+e36HRFiXhP +QcU9zXlxyWkDH8iz9DSAmE2jbfoqwwJANlMJ65E543cjIlitGcKLMnvtCCLcKpb7 +xSG0XJB6Lo11OKPJ66jp0gcFTSCY1Lx2CXVd+gfJrfwI1Pp562+bhwJBAJ9IfDPU +R8OpO9v1SGd8x33Owm7uXOpB9d63/T70AD1QOXjKUC4eXYbt0WWfWuny/RNPRuyh +w7DXSfUF+kPKolU= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICXTCCAcagAwIBAgIJAIO3upAG445fMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTAeFw0x +MDEwMDkxNTAxMDBaFw0yMDEwMDYxNTAxMDBaMGIxCzAJBgNVBAYTAlhZMRcwFQYD +VQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZv +dW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEAmewllWumkv5bkj3MZCutrBbATlAmkEr2dbDXuZetnTdHVwP0 +FqYsaK9UhJs+VGUBWdkETKHUBOmwHt0JRD5YcS8Auq6/Hj5jDmX59d6JhGOstEFh +m5XLzI21fNTeJZwY5txhGRR2Jd07I/uwsXZhkdG9BmnOAMEqSutp5DzXDbUCAwEA +AaMbMBkwFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBBQUAA4GB +AH+iMClLLGSaKWgwXsmdVo4FhTZZHo8Uprrtg3N9FxEeE50btpDVQysgRt5ias3K +m+bME9zbKwvbVWD5zZdjus4pDgzwF/iHyccL8JyYhxOvS/9zmvAtFXj/APIIbZFp +IT75d9f88ScIGEtknZQejnrdhB64tYki/EqluiuKBqKD +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/keycert3.pem b/src/greentest/3.5/keycert3.pem new file mode 100644 index 0000000..5bfa62c --- /dev/null +++ b/src/greentest/3.5/keycert3.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP +jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM +9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ +aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe +yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j +y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ +AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW +5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL +9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 +1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT +DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh +1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m +JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 +RnJdHOMXWem7/w== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: + 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: + c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: + 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: + f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: + 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: + f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: + af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: + 21:82:a5:3c:88:e5:be:1b:b1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: + e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: + f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: + e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: + d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: + 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: + ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: + 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: + 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: + 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: + 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: + f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: + 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: + a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: + fc:a9:94:71 +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C +tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola +N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 +TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR +iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG +xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo +5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv +mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF +YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh +2EJ36/yplHE= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/keycert4.pem b/src/greentest/3.5/keycert4.pem new file mode 100644 index 0000000..53355c8 --- /dev/null +++ b/src/greentest/3.5/keycert4.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv +L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2 +NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1 +L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L +pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de +R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9 +myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT +drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS +Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx +i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK +Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu +JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN ++/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/ +e83Gq6ffLVfKNQ== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443282 (0xb09264b1f2da21d2) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14: + 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9: + cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a: + b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76: + 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7: + 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a: + d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79: + 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd: + 81:7e:bd:1b:ae:0b:5d:c6:39 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0: + 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f: + 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56: + 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15: + 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1: + 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb: + 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e: + e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18: + d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1: + af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53: + 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68: + 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da: + 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92: + 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa: + 49:12:1e:ce +-----BEGIN CERTIFICATE----- +MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z +dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU +aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0 +ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ +hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v +xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338 +Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP +XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0 +UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz +aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb +oF+6ufu6+kkSHs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/nokia.pem b/src/greentest/3.5/nokia.pem new file mode 100644 index 0000000..0d044df --- /dev/null +++ b/src/greentest/3.5/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/nullbytecert.pem b/src/greentest/3.5/nullbytecert.pem new file mode 100644 index 0000000..447186c --- /dev/null +++ b/src/greentest/3.5/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/nullcert.pem b/src/greentest/3.5/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/greentest/3.5/pycacert.pem b/src/greentest/3.5/pycacert.pem new file mode 100644 index 0000000..09b1f3e --- /dev/null +++ b/src/greentest/3.5/pycacert.pem @@ -0,0 +1,78 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Jan 2 19:47:07 2023 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: + 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: + e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: + e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: + 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: + 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: + a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: + e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: + 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: + 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: + e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: + c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: + cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: + 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: + 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: + 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: + e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: + c5:4d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + X509v3 Authority Key Identifier: + keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: + 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: + a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: + 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: + 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: + 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: + fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: + 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: + 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: + 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: + ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: + 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: + b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: + 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: + 5e:58:c8:9e +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/pycakey.pem b/src/greentest/3.5/pycakey.pem new file mode 100644 index 0000000..fc6effe --- /dev/null +++ b/src/greentest/3.5/pycakey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDn3unjDJ8AtqH9 +K1uW0m/M4L6GuSBe7AN6VavqpOn5SYXSZtXtx3rqVo4tj+dC4mIoqZ/WG47rtbSc +nxSr3+aUi3YdPm0kYe0MvwCKYQzfXMg2cxYAzUe6baSkdIiDIwoZ/AmnPEpL0+cd +LeTqTFQh8ybbiTcY1AK7QDJfpP8tHPfUu+yOz1yCrOZ8CGxIhWEHfyXgXOC8NF/g +uQRHdchHC4281shoXzODYtIgRDWxrYEais28NbBci0fWGOmcGJfMATwpzOge5OTB +uN7nwhEYh1qTNNimJfcUcevkIaLSDy4u1GIANdPW71xgS0ypFOLdFVhGNzMmt+cu +Xe1C5MVNAgMBAAECggEBAJPM7QuUrPn4cLN/Ysd15lwTWn9oHDFFgkYFvCs66gXE +ju/6Kx2BjWE4wTJby09AHM/MqB0DvguT7Mf1Q2j3tPQ1HZowg8OwRDleuwp6KIls +jBbhL0Jdl/5HC67ktWvZ9wNvO/wFG1rQfT6FVajf9LUbWEaSZbOG2SLhHfsHorzu +xjTJaI3bQ/0+79B1exwk5ruwhzFRd/XpY8hls7D/RfPIuHDlBghkW3N59KFWrf5h +6bNEh2THm0+IyGcGqs0FD+QCOXyvsjwSUswqrr2ctLREOeDcd5ReUjSxYgjcJRrm +J7ceIY/+uwDJxw/OlnmBvF6pQMkKwYW2gFztu+g2t4UCgYEA/9yo01Exz4crxXsy +tAlnDJM++nZcm07rtFjTKHUfKY/cCgNTa8udM0svnfwlid/dpgLsI38gx04HHC1i +EZ4acz+ToIWedLxM0nq73//xeRWEazOvCz1mMTZaMldahTWAyzN8qVK2B/625Yy4 +wNYWyweBBwEB8MzaCs73spksXOsCgYEA5/7wvhiofYGFAfMuANeJIwDL2OtBnoOv +mVNfCmi3GC38fzwyi5ZpskWDiS2woJ+LQfs9Qu4EcZbUFLd7gbeOvb5gmFUtYope +LitUUKunIR18MkQ+mQDBpQPQPhk4QJP5reCbWkrfTu7b5o/iS41s6fBTFmuzhLcT +C71vFdCyeKcCgYAiCCqYeOtELDmBOeLDmaCQRqGQ1N96dOPbCBmF/xYXBCCDYG/f +HaUaJnz96YTgstsbcrYP/p/Qgqtlbw/lQf9IpwMuzbcG1ejt8g89OyDWNyt2ytgU +iaUnFJCos3/Byh0Iah/BsdOueo2/OJl2ZMOBW80orlSgv86cs2y037TL4wKBgQDm +OOyW+MlbowhnIvfoBfwlLEkefnej4nKD6WRLZBcue5Qyf355X06Mhsc9foXlH+6G +D9h/bswiHNdhp6N82rdgPGiHQx/CxiUoE/+b/nvgNO5mw6qLE2EXbG1e8pAMJcyE +bHw+YkawggDfELI036fRj5gki8SeUz8nS1nNgElbyQKBgCRDX9Jh+MwSLu4QBWdt +/fi+lv3K6kun/fI7EOV1vCV/j871tICu7pu5BrOLxAHqoVfU9AUX299/2KjCb5pv +kjogiUK6qWCWBlfuqDNWGCoUGt1rhznUva0nNjSMy5rinBhhjpROZC2pw48lOluP +UuvXsaPph7GTqPuy4Kab12YC +-----END PRIVATE KEY----- diff --git a/src/greentest/3.5/revocation.crl b/src/greentest/3.5/revocation.crl new file mode 100644 index 0000000..6d89b08 --- /dev/null +++ b/src/greentest/3.5/revocation.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTEzMTEyMTE3MDg0N1oXDTIzMDkzMDE3MDg0N1qgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQCNJXC2mVKauEeN3LlQ3ZtM5gkH3ExH ++i4bmJjtJn497WwvvoIeUdrmVXgJQR93RtV37hZwN0SXMLlNmUZPH4rHhihayw4m +unCzVj/OhCCY7/TPjKuJ1O/0XhaLBpBVjQN7R/1ujoRKbSia/CD3vcn7Fqxzw7LK +fSRCKRGTj1CZiuxrphtFchwALXSiFDy9mr2ZKhImcyq1PydfgEzU78APpOkMQsIC +UNJ/cf3c9emzf+dUtcMEcejQ3mynBo4eIGg1EW42bz4q4hSjzQlKcBV0muw5qXhc +HOxH2iTFhQ7SrvVuK/dM14rYM4B5mSX3nRC1kNmXpS9j3wJDhuwmjHed +-----END X509 CRL----- diff --git a/src/greentest/3.5/selfsigned_pythontestdotnet.pem b/src/greentest/3.5/selfsigned_pythontestdotnet.pem new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.5/selfsigned_pythontestdotnet.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/sha256.pem b/src/greentest/3.5/sha256.pem new file mode 100644 index 0000000..d3db4b8 --- /dev/null +++ b/src/greentest/3.5/sha256.pem @@ -0,0 +1,128 @@ +# Certificate chain for https://sha256.tbs-internet.com + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC +-----BEGIN CERTIFICATE----- +MIIGXDCCBUSgAwIBAgIRAKpVmHgg9nfCodAVwcP4siwwDQYJKoZIhvcNAQELBQAw +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMB4XDTEyMDEwNDAwMDAwMFoXDTE0MDIxNzIzNTk1OVowgcsxCzAJBgNV +BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV +BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM +VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS +c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKQIX/zdJcyxty0m +PM1XQSoSSifueS3AVcgqMsaIKS/u+rYzsv4hQ/qA6vLn5m5/ewUcZDj7zdi6rBVf +PaVNXJ6YinLX0tkaW8TEjeVuZG5yksGZlhCt1CJ1Ho9XLiLaP4uJ7MCoNUntpJ+E +LfrOdgsIj91kPmwjDJeztVcQCvKzhjVJA/KxdInc0JvOATn7rpaSmQI5bvIjufgo +qVsTPwVFzuUYULXBk7KxRT7MiEqnd5HvviNh0285QC478zl3v0I0Fb5El4yD3p49 +IthcRnxzMKc0UhU5ogi0SbONyBfm/mzONVfSxpM+MlyvZmJqrbuuLoEDzJD+t8PU +xSuzgbcCAwEAAaOCAj4wggI6MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMB0GA1UdDgQWBBT/qTGYdaj+f61c2IRFL/B1eEsM8DAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG +CisGAQQBgjcKAwMGCWCGSAGG+EIEATBLBgNVHSAERDBCMEAGCisGAQQB5TcCBAEw +MjAwBggrBgEFBQcCARYkaHR0cHM6Ly93d3cudGJzLWludGVybmV0LmNvbS9DQS9D +UFM0MG0GA1UdHwRmMGQwMqAwoC6GLGh0dHA6Ly9jcmwudGJzLWludGVybmV0LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMC6gLKAqhihodHRwOi8vY3JsLnRicy14NTA5LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMIGmBggrBgEFBQcBAQSBmTCBljA4BggrBgEFBQcw +AoYsaHR0cDovL2NydC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQVNHQy5jcnQw +NAYIKwYBBQUHMAKGKGh0dHA6Ly9jcnQudGJzLXg1MDkuY29tL1RCU1g1MDlDQVNH +Qy5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLnRicy14NTA5LmNvbTA/BgNV +HREEODA2ghdzaGEyNTYudGJzLWludGVybmV0LmNvbYIbd3d3LnNoYTI1Ni50YnMt +aW50ZXJuZXQuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQA0pOuL8QvAa5yksTbGShzX +ABApagunUGoEydv4YJT1MXy9tTp7DrWaozZSlsqBxrYAXP1d9r2fuKbEniYHxaQ0 +UYaf1VSIlDo1yuC8wE7wxbHDIpQ/E5KAyxiaJ8obtDhFstWAPAH+UoGXq0kj2teN +21sFQ5dXgA95nldvVFsFhrRUNB6xXAcaj0VZFhttI0ZfQZmQwEI/P+N9Jr40OGun +aa+Dn0TMeUH4U20YntfLbu2nDcJcYfyurm+8/0Tr4HznLnedXu9pCPYj0TaddrgT +XO0oFiyy7qGaY6+qKh71yD64Y3ycCJ/HR9Wm39mjZYc9ezYwT4noP6r7Lk8YO7/q +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6 +rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0 +9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ +ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk +owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G +Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk +9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ +MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3 +AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk +ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k +by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw +cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV +VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B +ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN +AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232 +euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY +1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98 +RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz +8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV +v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E= +-----END CERTIFICATE----- + 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT +AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0 +ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB +IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05 +4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6 +2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh +alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv +u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW +xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p +XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd +tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB +BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX +BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov +L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN +AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO +rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd +FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM ++bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI +3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb ++M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g= +-----END CERTIFICATE----- + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG +EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD +VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu +dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 +E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ +D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK +4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq +lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW +bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB +o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT +MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js +LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr +BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB +AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft +Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj +j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH +KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv +2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 +mfnGV/TJVTl4uix5yaaIK/QI +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/ssl_cert.pem b/src/greentest/3.5/ssl_cert.pem new file mode 100644 index 0000000..47a7d7e --- /dev/null +++ b/src/greentest/3.5/ssl_cert.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.5/ssl_key.passwd.pem b/src/greentest/3.5/ssl_key.passwd.pem new file mode 100644 index 0000000..2524672 --- /dev/null +++ b/src/greentest/3.5/ssl_key.passwd.pem @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- diff --git a/src/greentest/3.5/ssl_key.pem b/src/greentest/3.5/ssl_key.pem new file mode 100644 index 0000000..3fd3bbd --- /dev/null +++ b/src/greentest/3.5/ssl_key.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- diff --git a/src/greentest/3.5/test_httplib.py b/src/greentest/3.5/test_httplib.py new file mode 100644 index 0000000..61ed6bb --- /dev/null +++ b/src/greentest/3.5/test_httplib.py @@ -0,0 +1,1753 @@ +import errno +from http import client +import io +import itertools +import os +import array +import socket + +import unittest +TestCase = unittest.TestCase + +from test import support + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') +# Self-signed cert file for self-signed.pythontest.net +CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') + +# constants for testing chunked encoding +chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd! \r\n' + '8\r\n' + 'and now \r\n' + '22\r\n' + 'for something completely different\r\n' +) +chunked_expected = b'hello world! and now for something completely different' +chunk_extension = ";foo=bar" +last_chunk = "0\r\n" +last_chunk_extended = "0" + chunk_extension + "\r\n" +trailers = "X-Dummy: foo\r\nX-Dumm2: bar\r\n" +chunked_end = "\r\n" + +HOST = support.HOST + +class FakeSocket: + def __init__(self, text, fileclass=io.BytesIO, host=None, port=None): + if isinstance(text, str): + text = text.encode("ascii") + self.text = text + self.fileclass = fileclass + self.data = b'' + self.sendall_calls = 0 + self.file_closed = False + self.host = host + self.port = port + + def sendall(self, data): + self.sendall_calls += 1 + self.data += data + + def makefile(self, mode, bufsize=None): + if mode != 'r' and mode != 'rb': + raise client.UnimplementedFileMode() + # keep the file around so we can check how much was read from it + self.file = self.fileclass(self.text) + self.file.close = self.file_close #nerf close () + return self.file + + def file_close(self): + self.file_closed = True + + def close(self): + pass + + def setsockopt(self, level, optname, value): + pass + +class EPipeSocket(FakeSocket): + + def __init__(self, text, pipe_trigger): + # When sendall() is called with pipe_trigger, raise EPIPE. + FakeSocket.__init__(self, text) + self.pipe_trigger = pipe_trigger + + def sendall(self, data): + if self.pipe_trigger in data: + raise OSError(errno.EPIPE, "gotcha") + self.data += data + + def close(self): + pass + +class NoEOFBytesIO(io.BytesIO): + """Like BytesIO, but raises AssertionError on EOF. + + This is used below to test that http.client doesn't try to read + more from the underlying file than it should. + """ + def read(self, n=-1): + data = io.BytesIO.read(self, n) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + + def readline(self, length=None): + data = io.BytesIO.readline(self, length) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + +class FakeSocketHTTPConnection(client.HTTPConnection): + """HTTPConnection subclass using FakeSocket; counts connect() calls""" + + def __init__(self, *args): + self.connections = 0 + super().__init__('example.com') + self.fake_socket_args = args + self._create_connection = self.create_connection + + def connect(self): + """Count the number of times connect() is invoked""" + self.connections += 1 + return super().connect() + + def create_connection(self, *pos, **kw): + return FakeSocket(*self.fake_socket_args) + +class HeaderTests(TestCase): + def test_auto_headers(self): + # Some headers are added automatically, but should not be added by + # .request() if they are explicitly set. + + class HeaderCountingBuffer(list): + def __init__(self): + self.count = {} + def append(self, item): + kv = item.split(b':') + if len(kv) > 1: + # item is a 'Key: Value' header string + lcKey = kv[0].decode('ascii').lower() + self.count.setdefault(lcKey, 0) + self.count[lcKey] += 1 + list.append(self, item) + + for explicit_header in True, False: + for header in 'Content-length', 'Host', 'Accept-encoding': + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('blahblahblah') + conn._buffer = HeaderCountingBuffer() + + body = 'spamspamspam' + headers = {} + if explicit_header: + headers[header] = str(len(body)) + conn.request('POST', '/', body, headers) + self.assertEqual(conn._buffer.count[header.lower()], 1) + + def test_content_length_0(self): + + class ContentLengthChecker(list): + def __init__(self): + list.__init__(self) + self.content_length = None + def append(self, item): + kv = item.split(b':', 1) + if len(kv) > 1 and kv[0].lower() == b'content-length': + self.content_length = kv[1].strip() + list.append(self, item) + + # Here, we're testing that methods expecting a body get a + # content-length set to zero if the body is empty (either None or '') + bodies = (None, '') + methods_with_body = ('PUT', 'POST', 'PATCH') + for method, body in itertools.product(methods_with_body, bodies): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', body) + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # For these methods, we make sure that content-length is not set when + # the body is None because it might cause unexpected behaviour on the + # server. + methods_without_body = ( + 'GET', 'CONNECT', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', + ) + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', None) + self.assertEqual( + conn._buffer.content_length, None, + 'Header Content-Length set for empty body on {}'.format(method) + ) + + # If the body is set to '', that's considered to be "present but + # empty" rather than "missing", so content length would be set, even + # for methods that don't expect a body. + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', '') + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # If the body is set, make sure Content-Length is set. + for method in itertools.chain(methods_without_body, methods_with_body): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', ' ') + self.assertEqual( + conn._buffer.content_length, b'1', + 'Header Content-Length incorrect on {}'.format(method) + ) + + def test_putheader(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.putrequest('GET','/') + conn.putheader('Content-length', 42) + self.assertIn(b'Content-length: 42', conn._buffer) + + conn.putheader('Foo', ' bar ') + self.assertIn(b'Foo: bar ', conn._buffer) + conn.putheader('Bar', '\tbaz\t') + self.assertIn(b'Bar: \tbaz\t', conn._buffer) + conn.putheader('Authorization', 'Bearer mytoken') + self.assertIn(b'Authorization: Bearer mytoken', conn._buffer) + conn.putheader('IterHeader', 'IterA', 'IterB') + self.assertIn(b'IterHeader: IterA\r\n\tIterB', conn._buffer) + conn.putheader('LatinHeader', b'\xFF') + self.assertIn(b'LatinHeader: \xFF', conn._buffer) + conn.putheader('Utf8Header', b'\xc3\x80') + self.assertIn(b'Utf8Header: \xc3\x80', conn._buffer) + conn.putheader('C1-Control', b'next\x85line') + self.assertIn(b'C1-Control: next\x85line', conn._buffer) + conn.putheader('Embedded-Fold-Space', 'is\r\n allowed') + self.assertIn(b'Embedded-Fold-Space: is\r\n allowed', conn._buffer) + conn.putheader('Embedded-Fold-Tab', 'is\r\n\tallowed') + self.assertIn(b'Embedded-Fold-Tab: is\r\n\tallowed', conn._buffer) + conn.putheader('Key Space', 'value') + self.assertIn(b'Key Space: value', conn._buffer) + conn.putheader('KeySpace ', 'value') + self.assertIn(b'KeySpace : value', conn._buffer) + conn.putheader(b'Nonbreak\xa0Space', 'value') + self.assertIn(b'Nonbreak\xa0Space: value', conn._buffer) + conn.putheader(b'\xa0NonbreakSpace', 'value') + self.assertIn(b'\xa0NonbreakSpace: value', conn._buffer) + + def test_ipv6host_header(self): + # Default host header on IPv6 transaction should be wrapped by [] if + # it is an IPv6 address + expected = b'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001::]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + expected = b'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001:102A::]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + def test_malformed_headers_coped_with(self): + # Issue 19996 + body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + + self.assertEqual(resp.getheader('First'), 'val') + self.assertEqual(resp.getheader('Second'), 'val') + + def test_parse_all_octets(self): + # Ensure no valid header field octet breaks the parser + body = ( + b'HTTP/1.1 200 OK\r\n' + b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters + b'VCHAR: ' + bytes(range(0x21, 0x7E + 1)) + b'\r\n' + b'obs-text: ' + bytes(range(0x80, 0xFF + 1)) + b'\r\n' + b'obs-fold: text\r\n' + b' folded with space\r\n' + b'\tfolded with tab\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.getheader('Content-Length'), '0') + self.assertEqual(resp.msg['Content-Length'], '0') + self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value') + self.assertEqual(resp.msg["!#$%&'*+-.^_`|~"], 'value') + vchar = ''.join(map(chr, range(0x21, 0x7E + 1))) + self.assertEqual(resp.getheader('VCHAR'), vchar) + self.assertEqual(resp.msg['VCHAR'], vchar) + self.assertIsNotNone(resp.getheader('obs-text')) + self.assertIn('obs-text', resp.msg) + for folded in (resp.getheader('obs-fold'), resp.msg['obs-fold']): + self.assertTrue(folded.startswith('text')) + self.assertIn(' folded with space', folded) + self.assertTrue(folded.endswith('folded with tab')) + + def test_invalid_headers(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/') + + # http://tools.ietf.org/html/rfc7230#section-3.2.4, whitespace is no + # longer allowed in header names + cases = ( + (b'Invalid\r\nName', b'ValidValue'), + (b'Invalid\rName', b'ValidValue'), + (b'Invalid\nName', b'ValidValue'), + (b'\r\nInvalidName', b'ValidValue'), + (b'\rInvalidName', b'ValidValue'), + (b'\nInvalidName', b'ValidValue'), + (b' InvalidName', b'ValidValue'), + (b'\tInvalidName', b'ValidValue'), + (b'Invalid:Name', b'ValidValue'), + (b':InvalidName', b'ValidValue'), + (b'ValidName', b'Invalid\r\nValue'), + (b'ValidName', b'Invalid\rValue'), + (b'ValidName', b'Invalid\nValue'), + (b'ValidName', b'InvalidValue\r\n'), + (b'ValidName', b'InvalidValue\r'), + (b'ValidName', b'InvalidValue\n'), + ) + for name, value in cases: + with self.subTest((name, value)): + with self.assertRaisesRegex(ValueError, 'Invalid header'): + conn.putheader(name, value) + + +class BasicTest(TestCase): + def test_status_lines(self): + # Test HTTP status lines + + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(0), b'') # Issue #20007 + self.assertFalse(resp.isclosed()) + self.assertFalse(resp.closed) + self.assertEqual(resp.read(), b"Text") + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + self.assertRaises(client.BadStatusLine, resp.begin) + + def test_bad_status_repr(self): + exc = client.BadStatusLine('') + self.assertEqual(repr(exc), '''BadStatusLine("\'\'",)''') + + def test_partial_reads(self): + # if we have Content-Length, HTTPResponse knows when to close itself, + # the same behaviour as when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_mixed_reads(self): + # readline() should update the remaining length, so that read() knows + # how much data is left and does not raise IncompleteRead + body = "HTTP/1.1 200 Ok\r\nContent-Length: 13\r\n\r\nText\r\nAnother" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.readline(), b'Text\r\n') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(), b'Another') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos(self): + # if we have Content-Length, HTTPResponse knows when to close itself, + # the same behaviour as when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_reads_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + + def test_partial_reads_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + + def test_partial_readintos_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:80", "www.python.org", 80), + ("www.python.org:", "www.python.org", 80), + ("www.python.org", "www.python.org", 80), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", 80)): + c = client.HTTPConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + def test_response_headers(self): + # test response with multiple message headers with the same field name. + text = ('HTTP/1.1 200 OK\r\n' + 'Set-Cookie: Customer="WILE_E_COYOTE"; ' + 'Version="1"; Path="/acme"\r\n' + 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' + ' Path="/acme"\r\n' + '\r\n' + 'No body\r\n') + hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"' + ', ' + 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"') + s = FakeSocket(text) + r = client.HTTPResponse(s) + r.begin() + cookies = r.getheader("Set-Cookie") + self.assertEqual(cookies, hdr) + + def test_read_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + if resp.read(): + self.fail("Did not expect response from HEAD request") + + def test_readinto_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + if resp.readinto(b) != 0: + self.fail("Did not expect response from HEAD request") + self.assertEqual(bytes(b), b'\x00'*5) + + def test_too_many_headers(self): + headers = '\r\n'.join('Header%d: foo' % i + for i in range(client._MAXHEADERS + 1)) + '\r\n' + text = ('HTTP/1.1 200 OK\r\n' + headers) + s = FakeSocket(text) + r = client.HTTPResponse(s) + self.assertRaisesRegex(client.HTTPException, + r"got more than \d+ headers", r.begin) + + def test_send_file(self): + expected = (b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' + b'Accept-Encoding: identity\r\nContent-Length:') + + with open(__file__, 'rb') as body: + conn = client.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertTrue(sock.data.startswith(expected), '%r != %r' % + (sock.data[:len(expected)], expected)) + + def test_send(self): + expected = b'this is a test this is only a test' + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + conn.send(expected) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(array.array('b', expected)) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(io.BytesIO(expected)) + self.assertEqual(expected, sock.data) + + def test_send_updating_file(self): + def data(): + yield 'data' + yield None + yield 'data_two' + + class UpdatingFile(): + mode = 'r' + d = data() + def read(self, blocksize=-1): + return self.d.__next__() + + expected = b'data' + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.send(UpdatingFile()) + self.assertEqual(sock.data, expected) + + + def test_send_iter(self): + expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \ + b'\r\nonetwothree' + + def body(): + yield b"one" + yield b"two" + yield b"three" + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.request('GET', '/foo', body(), {'Content-Length': '11'}) + self.assertEqual(sock.data, expected) + + def test_send_type_error(self): + # See: Issue #12676 + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + with self.assertRaises(TypeError): + conn.request('POST', 'test', conn) + + def test_chunked(self): + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(n) + resp.read(n) + resp.read(), expected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_readinto_chunked(self): + + expected = chunked_expected + nexpected = len(expected) + b = bytearray(128) + + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + n = resp.readinto(b) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(n, nexpected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + m = memoryview(b) + i = resp.readinto(m[0:n]) + i += resp.readinto(m[i:n + i]) + i += resp.readinto(m[i:]) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(i, nexpected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + n = resp.readinto(b) + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_readinto_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertEqual(bytes(b), b'\x00'*5) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_negative_content_length(self): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), b'Hello\r\n') + self.assertTrue(resp.isclosed()) + + def test_incomplete_read(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, b'Hello\r\n') + self.assertEqual(repr(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertEqual(str(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertTrue(resp.isclosed()) + else: + self.fail('IncompleteRead expected') + + def test_epipe(self): + sock = EPipeSocket( + "HTTP/1.0 401 Authorization Required\r\n" + "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"example\"\r\n", + b"Content-Length") + conn = client.HTTPConnection("example.com") + conn.sock = sock + self.assertRaises(OSError, + lambda: conn.request("PUT", "/url", "body")) + resp = conn.getresponse() + self.assertEqual(401, resp.status) + self.assertEqual("Basic realm=\"example\"", + resp.getheader("www-authenticate")) + + # Test lines overflowing the max line size (_MAXLINE in http.client) + + def test_overflowing_status_line(self): + body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n" + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises((client.LineTooLong, client.BadStatusLine), resp.begin) + + def test_overflowing_header_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises(client.LineTooLong, resp.begin) + + def test_overflowing_chunked_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + + '0' * 65536 + 'a\r\n' + 'hello world\r\n' + '0\r\n' + '\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertRaises(client.LineTooLong, resp.read) + + def test_early_eof(self): + # Test httpresponse with no \r\n termination, + body = "HTTP/1.1 200 Ok" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_error_leak(self): + # Test that the socket is not leaked if getresponse() fails + conn = client.HTTPConnection('example.com') + response = None + class Response(client.HTTPResponse): + def __init__(self, *pos, **kw): + nonlocal response + response = self # Avoid garbage collector closing the socket + client.HTTPResponse.__init__(self, *pos, **kw) + conn.response_class = Response + conn.sock = FakeSocket('Invalid status line') + conn.request('GET', '/') + self.assertRaises(client.BadStatusLine, conn.getresponse) + self.assertTrue(response.closed) + self.assertTrue(conn.sock.file_closed) + + def test_chunked_extension(self): + extra = '3;foo=bar\r\n' + 'abc\r\n' + expected = chunked_expected + b'abc' + + sock = FakeSocket(chunked_start + extra + last_chunk_extended + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + def test_chunked_missing_end(self): + """some servers may serve up a short chunked encoding stream""" + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk) #no terminating crlf + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + def test_chunked_trailers(self): + """See that trailers are read and ignored""" + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # we should have reached the end of the file + self.assertEqual(sock.file.read(), b"") #we read to the end + resp.close() + + def test_chunked_sync(self): + """Check that we don't read past the end of the chunked-encoding stream""" + expected = chunked_expected + extradata = "extradata" + sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata.encode("ascii")) #we read to the end + resp.close() + + def test_content_length_sync(self): + """Check that we don't read past the end of the Content-Length stream""" + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_readlines_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.readlines(2000), [expected]) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_read1_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read1(2000), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_readline_bound_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.readline(10), expected) + self.assertEqual(resp.readline(10), b"") + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_read1_bound_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 30\r\n\r\n' + expected*3 + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read1(20), expected*2) + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_response_fileno(self): + # Make sure fd returned by fileno is valid. + threading = support.import_module("threading") + + serv = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + self.addCleanup(serv.close) + serv.bind((HOST, 0)) + serv.listen() + + result = None + def run_server(): + [conn, address] = serv.accept() + with conn, conn.makefile("rb") as reader: + # Read the request header until a blank line + while True: + line = reader.readline() + if not line.rstrip(b"\r\n"): + break + conn.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n") + nonlocal result + result = reader.read() + + thread = threading.Thread(target=run_server) + thread.start() + conn = client.HTTPConnection(*serv.getsockname()) + conn.request("CONNECT", "dummy:1234") + response = conn.getresponse() + try: + self.assertEqual(response.status, client.OK) + s = socket.socket(fileno=response.fileno()) + try: + s.sendall(b"proxied data\n") + finally: + s.detach() + finally: + response.close() + conn.close() + thread.join() + self.assertEqual(result, b"proxied data\n") + +class ExtendedReadTest(TestCase): + """ + Test peek(), read1(), readline() + """ + lines = ( + 'HTTP/1.1 200 OK\r\n' + '\r\n' + 'hello world!\n' + 'and now \n' + 'for something completely different\n' + 'foo' + ) + lines_expected = lines[lines.find('hello'):].encode("ascii") + lines_chunked = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd!\n\r\n' + '9\r\n' + 'and now \n\r\n' + '23\r\n' + 'for something completely different\n\r\n' + '3\r\n' + 'foo\r\n' + '0\r\n' # terminating chunk + '\r\n' # end of trailers + ) + + def setUp(self): + sock = FakeSocket(self.lines) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + resp.fp = io.BufferedReader(resp.fp) + self.resp = resp + + + + def test_peek(self): + resp = self.resp + # patch up the buffered peek so that it returns not too much stuff + oldpeek = resp.fp.peek + def mypeek(n=-1): + p = oldpeek(n) + if n >= 0: + return p[:n] + return p[:10] + resp.fp.peek = mypeek + + all = [] + while True: + # try a short peek + p = resp.peek(3) + if p: + self.assertGreater(len(p), 0) + # then unbounded peek + p2 = resp.peek() + self.assertGreaterEqual(len(p2), len(p)) + self.assertTrue(p2.startswith(p)) + next = resp.read(len(p2)) + self.assertEqual(next, p2) + else: + next = resp.read() + self.assertFalse(next) + all.append(next) + if not next: + break + self.assertEqual(b"".join(all), self.lines_expected) + + def test_readline(self): + resp = self.resp + self._verify_readline(self.resp.readline, self.lines_expected) + + def _verify_readline(self, readline, expected): + all = [] + while True: + # short readlines + line = readline(5) + if line and line != b"foo": + if len(line) < 5: + self.assertTrue(line.endswith(b"\n")) + all.append(line) + if not line: + break + self.assertEqual(b"".join(all), expected) + + def test_read1(self): + resp = self.resp + def r(): + res = resp.read1(4) + self.assertLessEqual(len(res), 4) + return res + readliner = Readliner(r) + self._verify_readline(readliner.readline, self.lines_expected) + + def test_read1_unbounded(self): + resp = self.resp + all = [] + while True: + data = resp.read1() + if not data: + break + all.append(data) + self.assertEqual(b"".join(all), self.lines_expected) + + def test_read1_bounded(self): + resp = self.resp + all = [] + while True: + data = resp.read1(10) + if not data: + break + self.assertLessEqual(len(data), 10) + all.append(data) + self.assertEqual(b"".join(all), self.lines_expected) + + def test_read1_0(self): + self.assertEqual(self.resp.read1(0), b"") + + def test_peek_0(self): + p = self.resp.peek(0) + self.assertLessEqual(0, len(p)) + +class ExtendedReadTestChunked(ExtendedReadTest): + """ + Test peek(), read1(), readline() in chunked mode + """ + lines = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd!\n\r\n' + '9\r\n' + 'and now \n\r\n' + '23\r\n' + 'for something completely different\n\r\n' + '3\r\n' + 'foo\r\n' + '0\r\n' # terminating chunk + '\r\n' # end of trailers + ) + + +class Readliner: + """ + a simple readline class that uses an arbitrary read function and buffering + """ + def __init__(self, readfunc): + self.readfunc = readfunc + self.remainder = b"" + + def readline(self, limit): + data = [] + datalen = 0 + read = self.remainder + try: + while True: + idx = read.find(b'\n') + if idx != -1: + break + if datalen + len(read) >= limit: + idx = limit - datalen - 1 + # read more data + data.append(read) + read = self.readfunc() + if not read: + idx = 0 #eof condition + break + idx += 1 + data.append(read[:idx]) + self.remainder = read[idx:] + return b"".join(data) + except: + self.remainder = b"".join(data) + raise + + +class OfflineTest(TestCase): + def test_all(self): + # Documented objects defined in the module should be in __all__ + expected = {"responses"} # White-list documented dict() object + # HTTPMessage, parse_headers(), and the HTTP status code constants are + # intentionally omitted for simplicity + blacklist = {"HTTPMessage", "parse_headers"} + for name in dir(client): + if name.startswith("_") or name in blacklist: + continue + module_object = getattr(client, name) + if getattr(module_object, "__module__", None) == "http.client": + expected.add(name) + self.assertCountEqual(client.__all__, expected) + + def test_responses(self): + self.assertEqual(client.responses[client.NOT_FOUND], "Not Found") + + def test_client_constants(self): + # Make sure we don't break backward compatibility with 3.4 + expected = [ + 'CONTINUE', + 'SWITCHING_PROTOCOLS', + 'PROCESSING', + 'OK', + 'CREATED', + 'ACCEPTED', + 'NON_AUTHORITATIVE_INFORMATION', + 'NO_CONTENT', + 'RESET_CONTENT', + 'PARTIAL_CONTENT', + 'MULTI_STATUS', + 'IM_USED', + 'MULTIPLE_CHOICES', + 'MOVED_PERMANENTLY', + 'FOUND', + 'SEE_OTHER', + 'NOT_MODIFIED', + 'USE_PROXY', + 'TEMPORARY_REDIRECT', + 'BAD_REQUEST', + 'UNAUTHORIZED', + 'PAYMENT_REQUIRED', + 'FORBIDDEN', + 'NOT_FOUND', + 'METHOD_NOT_ALLOWED', + 'NOT_ACCEPTABLE', + 'PROXY_AUTHENTICATION_REQUIRED', + 'REQUEST_TIMEOUT', + 'CONFLICT', + 'GONE', + 'LENGTH_REQUIRED', + 'PRECONDITION_FAILED', + 'REQUEST_ENTITY_TOO_LARGE', + 'REQUEST_URI_TOO_LONG', + 'UNSUPPORTED_MEDIA_TYPE', + 'REQUESTED_RANGE_NOT_SATISFIABLE', + 'EXPECTATION_FAILED', + 'UNPROCESSABLE_ENTITY', + 'LOCKED', + 'FAILED_DEPENDENCY', + 'UPGRADE_REQUIRED', + 'PRECONDITION_REQUIRED', + 'TOO_MANY_REQUESTS', + 'REQUEST_HEADER_FIELDS_TOO_LARGE', + 'INTERNAL_SERVER_ERROR', + 'NOT_IMPLEMENTED', + 'BAD_GATEWAY', + 'SERVICE_UNAVAILABLE', + 'GATEWAY_TIMEOUT', + 'HTTP_VERSION_NOT_SUPPORTED', + 'INSUFFICIENT_STORAGE', + 'NOT_EXTENDED', + 'NETWORK_AUTHENTICATION_REQUIRED', + ] + for const in expected: + with self.subTest(constant=const): + self.assertTrue(hasattr(client, const)) + + +class SourceAddressTest(TestCase): + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.source_port = support.find_unused_port() + self.serv.listen() + self.conn = None + + def tearDown(self): + if self.conn: + self.conn.close() + self.conn = None + self.serv.close() + self.serv = None + + def testHTTPConnectionSourceAddress(self): + self.conn = client.HTTPConnection(HOST, self.port, + source_address=('', self.source_port)) + self.conn.connect() + self.assertEqual(self.conn.sock.getsockname()[1], self.source_port) + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not defined') + def testHTTPSConnectionSourceAddress(self): + self.conn = client.HTTPSConnection(HOST, self.port, + source_address=('', self.source_port)) + # We don't test anything here other than the constructor not barfing as + # this code doesn't deal with setting up an active running SSL server + # for an ssl_wrapped connect() to actually return from. + + +class TimeoutTest(TestCase): + PORT = None + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + TimeoutTest.PORT = support.bind_port(self.serv) + self.serv.listen() + + def tearDown(self): + self.serv.close() + self.serv = None + + def testTimeoutAttribute(self): + # This will prove that the timeout gets through HTTPConnection + # and into the socket. + + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + # no timeout -- do not use global socket default + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, + timeout=None) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), None) + httpConn.close() + + # a value + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30) + httpConn.connect() + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + +class PersistenceTest(TestCase): + + def test_reuse_reconnect(self): + # Should reuse or reconnect depending on header from server + tests = ( + ('1.0', '', False), + ('1.0', 'Connection: keep-alive\r\n', True), + ('1.1', '', True), + ('1.1', 'Connection: close\r\n', False), + ('1.0', 'Connection: keep-ALIVE\r\n', True), + ('1.1', 'Connection: cloSE\r\n', False), + ) + for version, header, reuse in tests: + with self.subTest(version=version, header=header): + msg = ( + 'HTTP/{} 200 OK\r\n' + '{}' + 'Content-Length: 12\r\n' + '\r\n' + 'Dummy body\r\n' + ).format(version, header) + conn = FakeSocketHTTPConnection(msg) + self.assertIsNone(conn.sock) + conn.request('GET', '/open-connection') + with conn.getresponse() as response: + self.assertEqual(conn.sock is None, not reuse) + response.read() + self.assertEqual(conn.sock is None, not reuse) + self.assertEqual(conn.connections, 1) + conn.request('GET', '/subsequent-request') + self.assertEqual(conn.connections, 1 if reuse else 2) + + def test_disconnected(self): + + def make_reset_reader(text): + """Return BufferedReader that raises ECONNRESET at EOF""" + stream = io.BytesIO(text) + def readinto(buffer): + size = io.BytesIO.readinto(stream, buffer) + if size == 0: + raise ConnectionResetError() + return size + stream.readinto = readinto + return io.BufferedReader(stream) + + tests = ( + (io.BytesIO, client.RemoteDisconnected), + (make_reset_reader, ConnectionResetError), + ) + for stream_factory, exception in tests: + with self.subTest(exception=exception): + conn = FakeSocketHTTPConnection(b'', stream_factory) + conn.request('GET', '/eof-response') + self.assertRaises(exception, conn.getresponse) + self.assertIsNone(conn.sock) + # HTTPConnection.connect() should be automatically invoked + conn.request('GET', '/reconnect') + self.assertEqual(conn.connections, 2) + + def test_100_close(self): + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + # Missing final response + ) + conn.request('GET', '/', headers={'Expect': '100-continue'}) + self.assertRaises(client.RemoteDisconnected, conn.getresponse) + self.assertIsNone(conn.sock) + conn.request('GET', '/reconnect') + self.assertEqual(conn.connections, 2) + + +class HTTPSTest(TestCase): + + def setUp(self): + if not hasattr(client, 'HTTPSConnection'): + self.skipTest('ssl support required') + + def make_server(self, certfile): + from test.ssl_servers import make_https_server + return make_https_server(self, certfile=certfile) + + def test_attributes(self): + # simple test to check it's storing the timeout + h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) + self.assertEqual(h.timeout, 30) + + def test_networked(self): + # Default settings: requires a valid cert from a trusted CA + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + h = client.HTTPSConnection('self-signed.pythontest.net', 443) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_networked_noverification(self): + # Switch off cert verification + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl._create_unverified_context() + h = client.HTTPSConnection('self-signed.pythontest.net', 443, + context=context) + h.request('GET', '/') + resp = h.getresponse() + h.close() + self.assertIn('nginx', resp.getheader('server')) + resp.close() + + @support.system_must_validate_cert + def test_networked_trusted_by_default_cert(self): + # Default settings: requires a valid cert from a trusted CA + support.requires('network') + with support.transient_internet('www.python.org'): + h = client.HTTPSConnection('www.python.org', 443) + h.request('GET', '/') + resp = h.getresponse() + content_type = resp.getheader('content-type') + resp.close() + h.close() + self.assertIn('text/html', content_type) + + def test_networked_good_cert(self): + # We feed the server's cert as a validating cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_selfsigned_pythontestdotnet) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h.request('GET', '/') + resp = h.getresponse() + server_string = resp.getheader('server') + resp.close() + h.close() + self.assertIn('nginx', server_string) + + def test_networked_bad_cert(self): + # We feed a "CA" cert that is unrelated to the server's cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_unknown_cert(self): + # The custom cert isn't known to the default trust bundle + import ssl + server = self.make_server(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_good_hostname(self): + # The (valid) cert validates the HTTP hostname + import ssl + server = self.make_server(CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port, context=context) + self.addCleanup(h.close) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.addCleanup(resp.close) + self.assertEqual(resp.status, 404) + + def test_local_bad_hostname(self): + # The (valid) cert doesn't validate the HTTP hostname + import ssl + server = self.make_server(CERT_fakehostname) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(CERT_fakehostname) + h = client.HTTPSConnection('localhost', server.port, context=context) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # Same with explicit check_hostname=True + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # With check_hostname=False, the mismatching is ignored + context.check_hostname = False + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=False) + h.request('GET', '/nonexistent') + resp = h.getresponse() + resp.close() + h.close() + self.assertEqual(resp.status, 404) + # The context's check_hostname setting is used if one isn't passed to + # HTTPSConnection. + context.check_hostname = False + h = client.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + resp.close() + h.close() + # Passing check_hostname to HTTPSConnection should override the + # context's setting. + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not available') + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPSConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:443", "www.python.org", 443), + ("www.python.org:", "www.python.org", 443), + ("www.python.org", "www.python.org", 443), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 443), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", + 443)): + c = client.HTTPSConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + +class RequestBodyTest(TestCase): + """Test cases where a request includes a message body.""" + + def setUp(self): + self.conn = client.HTTPConnection('example.com') + self.conn.sock = self.sock = FakeSocket("") + self.conn.sock = self.sock + + def get_headers_and_fp(self): + f = io.BytesIO(self.sock.data) + f.readline() # read the request line + message = client.parse_headers(f) + return message, f + + def test_manual_content_length(self): + # Set an incorrect content-length so that we can verify that + # it will not be over-ridden by the library. + self.conn.request("PUT", "/url", "body", + {"Content-Length": "42"}) + message, f = self.get_headers_and_fp() + self.assertEqual("42", message.get("content-length")) + self.assertEqual(4, len(f.read())) + + def test_ascii_body(self): + self.conn.request("PUT", "/url", "body") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("4", message.get("content-length")) + self.assertEqual(b'body', f.read()) + + def test_latin1_body(self): + self.conn.request("PUT", "/url", "body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_bytes_body(self): + self.conn.request("PUT", "/url", b"body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as f: + f.write("body") + with open(support.TESTFN) as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("4", message.get("content-length")) + self.assertEqual(b'body', f.read()) + + def test_binary_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "wb") as f: + f.write(b"body\xc1") + with open(support.TESTFN, "rb") as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + +class HTTPResponseTest(TestCase): + + def setUp(self): + body = "HTTP/1.1 200 Ok\r\nMy-Header: first-value\r\nMy-Header: \ + second-value\r\n\r\nText" + sock = FakeSocket(body) + self.resp = client.HTTPResponse(sock) + self.resp.begin() + + def test_getting_header(self): + header = self.resp.getheader('My-Header') + self.assertEqual(header, 'first-value, second-value') + + header = self.resp.getheader('My-Header', 'some default') + self.assertEqual(header, 'first-value, second-value') + + def test_getting_nonexistent_header_with_string_default(self): + header = self.resp.getheader('No-Such-Header', 'default-value') + self.assertEqual(header, 'default-value') + + def test_getting_nonexistent_header_with_iterable_default(self): + header = self.resp.getheader('No-Such-Header', ['default', 'values']) + self.assertEqual(header, 'default, values') + + header = self.resp.getheader('No-Such-Header', ('default', 'values')) + self.assertEqual(header, 'default, values') + + def test_getting_nonexistent_header_without_default(self): + header = self.resp.getheader('No-Such-Header') + self.assertEqual(header, None) + + def test_getting_header_defaultint(self): + header = self.resp.getheader('No-Such-Header',default=42) + self.assertEqual(header, 42) + +class TunnelTests(TestCase): + def setUp(self): + response_text = ( + 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n' # Reply to HEAD + 'Content-Length: 42\r\n\r\n' + ) + self.host = 'proxy.com' + self.conn = client.HTTPConnection(self.host) + self.conn._create_connection = self._create_connection(response_text) + + def tearDown(self): + self.conn.close() + + def _create_connection(self, response_text): + def create_connection(address, timeout=None, source_address=None): + return FakeSocket(response_text, host=address[0], port=address[1]) + return create_connection + + def test_set_tunnel_host_port_headers(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers) + + def test_disallow_set_tunnel_after_connect(self): + # Once connected, we shouldn't be able to tunnel anymore + self.conn.connect() + self.assertRaises(RuntimeError, self.conn.set_tunnel, + 'destination.com') + + def test_connect_with_tunnel(self): + self.conn.set_tunnel('destination.com') + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + # issue22095 + self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + # This test should be removed when CONNECT gets the HTTP/1.1 blessing + self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) + + def test_connect_put_request(self): + self.conn.set_tunnel('destination.com') + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + def test_tunnel_debuglog(self): + expected_header = 'X-Dummy: 1' + response_text = 'HTTP/1.0 200 OK\r\n{}\r\n\r\n'.format(expected_header) + + self.conn.set_debuglevel(1) + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + with support.captured_stdout() as output: + self.conn.request('PUT', '/', '') + lines = output.getvalue().splitlines() + self.assertIn('header: {}'.format(expected_header), lines) + + +@support.reap_threads +def test_main(verbose=None): + support.run_unittest(HeaderTests, OfflineTest, BasicTest, TimeoutTest, + PersistenceTest, + HTTPSTest, RequestBodyTest, SourceAddressTest, + HTTPResponseTest, ExtendedReadTest, + ExtendedReadTestChunked, TunnelTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/3.5/test_select.py b/src/greentest/3.5/test_select.py new file mode 100644 index 0000000..a973f3f --- /dev/null +++ b/src/greentest/3.5/test_select.py @@ -0,0 +1,82 @@ +import errno +import os +import select +import sys +import unittest +from test import support + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + try: + select.select([fd], [], [], 0) + except OSError as err: + self.assertEqual(err.errno, errno.EBADF) + else: + self.fail("exception not raised") + + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + def test_select(self): + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print('timeout =', tout) + rfd, wfd, xfd = select.select([p], [], [], tout) + if (rfd, wfd, xfd) == ([], [], []): + continue + if (rfd, wfd, xfd) == ([p], [], []): + line = p.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select():', rfd, wfd, xfd) + p.close() + + # Issue 16230: Crash on select resized list + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + self.assertEqual(select.select([], a, []), ([], a[:5], [])) + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5/test_selectors.py b/src/greentest/3.5/test_selectors.py new file mode 100644 index 0000000..852b2fe --- /dev/null +++ b/src/greentest/3.5/test_selectors.py @@ -0,0 +1,526 @@ +import errno +import os +import random +import selectors +import signal +import socket +import sys +from test import support +from time import sleep +import unittest +import unittest.mock +import tempfile +from time import monotonic as time +try: + import resource +except ImportError: + resource = None + + +if hasattr(socket, 'socketpair'): + socketpair = socket.socketpair +else: + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): + with socket.socket(family, type, proto) as l: + l.bind((support.HOST, 0)) + l.listen() + c = socket.socket(family, type, proto) + try: + c.connect(l.getsockname()) + caddr = c.getsockname() + while True: + a, addr = l.accept() + # check that we've got the correct client + if addr == caddr: + return c, a + a.close() + except OSError: + c.close() + raise + + +def find_ready_matching(ready, flag): + match = [] + for key, events in ready: + if events & flag: + match.append(key.fileobj) + return match + + +class BaseSelectorTestCase(unittest.TestCase): + + def make_socketpair(self): + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + return rd, wr + + def test_register(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIsInstance(key, selectors.SelectorKey) + self.assertEqual(key.fileobj, rd) + self.assertEqual(key.fd, rd.fileno()) + self.assertEqual(key.events, selectors.EVENT_READ) + self.assertEqual(key.data, "data") + + # register an unknown event + self.assertRaises(ValueError, s.register, 0, 999999) + + # register an invalid FD + self.assertRaises(ValueError, s.register, -10, selectors.EVENT_READ) + + # register twice + self.assertRaises(KeyError, s.register, rd, selectors.EVENT_READ) + + # register the same FD, but with a different object + self.assertRaises(KeyError, s.register, rd.fileno(), + selectors.EVENT_READ) + + def test_unregister(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + s.unregister(rd) + + # unregister an unknown file obj + self.assertRaises(KeyError, s.unregister, 999999) + + # unregister twice + self.assertRaises(KeyError, s.unregister, rd) + + def test_unregister_after_fd_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + r, w = rd.fileno(), wr.fileno() + s.register(r, selectors.EVENT_READ) + s.register(w, selectors.EVENT_WRITE) + rd.close() + wr.close() + s.unregister(r) + s.unregister(w) + + @unittest.skipUnless(os.name == 'posix', "requires posix") + def test_unregister_after_fd_close_and_reuse(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + r, w = rd.fileno(), wr.fileno() + s.register(r, selectors.EVENT_READ) + s.register(w, selectors.EVENT_WRITE) + rd2, wr2 = self.make_socketpair() + rd.close() + wr.close() + os.dup2(rd2.fileno(), r) + os.dup2(wr2.fileno(), w) + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + s.unregister(r) + s.unregister(w) + + def test_unregister_after_socket_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + rd.close() + wr.close() + s.unregister(rd) + s.unregister(wr) + + def test_modify(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ) + + # modify events + key2 = s.modify(rd, selectors.EVENT_WRITE) + self.assertNotEqual(key.events, key2.events) + self.assertEqual(key2, s.get_key(rd)) + + s.unregister(rd) + + # modify data + d1 = object() + d2 = object() + + key = s.register(rd, selectors.EVENT_READ, d1) + key2 = s.modify(rd, selectors.EVENT_READ, d2) + self.assertEqual(key.events, key2.events) + self.assertNotEqual(key.data, key2.data) + self.assertEqual(key2, s.get_key(rd)) + self.assertEqual(key2.data, d2) + + # modify unknown file obj + self.assertRaises(KeyError, s.modify, 999999, selectors.EVENT_READ) + + # modify use a shortcut + d3 = object() + s.register = unittest.mock.Mock() + s.unregister = unittest.mock.Mock() + + s.modify(rd, selectors.EVENT_READ, d3) + self.assertFalse(s.register.called) + self.assertFalse(s.unregister.called) + + def test_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + mapping = s.get_map() + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + + s.close() + self.assertRaises(RuntimeError, s.get_key, rd) + self.assertRaises(RuntimeError, s.get_key, wr) + self.assertRaises(KeyError, mapping.__getitem__, rd) + self.assertRaises(KeyError, mapping.__getitem__, wr) + + def test_get_key(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertEqual(key, s.get_key(rd)) + + # unknown file obj + self.assertRaises(KeyError, s.get_key, 999999) + + def test_get_map(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + keys = s.get_map() + self.assertFalse(keys) + self.assertEqual(len(keys), 0) + self.assertEqual(list(keys), []) + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIn(rd, keys) + self.assertEqual(key, keys[rd]) + self.assertEqual(len(keys), 1) + self.assertEqual(list(keys), [rd.fileno()]) + self.assertEqual(list(keys.values()), [key]) + + # unknown file obj + with self.assertRaises(KeyError): + keys[999999] + + # Read-only mapping + with self.assertRaises(TypeError): + del keys[rd] + + def test_select(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + wr_key = s.register(wr, selectors.EVENT_WRITE) + + result = s.select() + for key, events in result: + self.assertTrue(isinstance(key, selectors.SelectorKey)) + self.assertTrue(events) + self.assertFalse(events & ~(selectors.EVENT_READ | + selectors.EVENT_WRITE)) + + self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result) + + def test_context_manager(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + with s as sel: + sel.register(rd, selectors.EVENT_READ) + sel.register(wr, selectors.EVENT_WRITE) + + self.assertRaises(RuntimeError, s.get_key, rd) + self.assertRaises(RuntimeError, s.get_key, wr) + + def test_fileno(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + if hasattr(s, 'fileno'): + fd = s.fileno() + self.assertTrue(isinstance(fd, int)) + self.assertGreaterEqual(fd, 0) + + def test_selector(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + NUM_SOCKETS = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_SOCKETS): + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = s.select() + ready_writers = find_ready_matching(ready, selectors.EVENT_WRITE) + if not ready_writers: + self.fail("no sockets ready for writing") + wr = random.choice(ready_writers) + wr.send(MSG) + + for i in range(10): + ready = s.select() + ready_readers = find_ready_matching(ready, + selectors.EVENT_READ) + if ready_readers: + break + # there might be a delay between the write to the write end and + # the read end is reported ready + sleep(0.1) + else: + self.fail("no sockets ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = rd.recv(MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + s.unregister(r2w[rd]) + s.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_SOCKETS) + + @unittest.skipIf(sys.platform == 'win32', + 'select.select() cannot be used with empty fd sets') + def test_empty_select(self): + # Issue #23009: Make sure EpollSelector.select() works when no FD is + # registered. + s = self.SELECTOR() + self.addCleanup(s.close) + self.assertEqual(s.select(timeout=0), []) + + def test_timeout(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(wr, selectors.EVENT_WRITE) + t = time() + self.assertEqual(1, len(s.select(0))) + self.assertEqual(1, len(s.select(-1))) + self.assertLess(time() - t, 0.5) + + s.unregister(wr) + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(0)) + self.assertFalse(s.select(-1)) + self.assertLess(time() - t, 0.5) + + t0 = time() + self.assertFalse(s.select(1)) + t1 = time() + dt = t1 - t0 + # Tolerate 2.0 seconds for very slow buildbots + self.assertTrue(0.8 <= dt <= 2.0, dt) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_select_interrupt_exc(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + class InterruptSelect(Exception): + pass + + def handler(*args): + raise InterruptSelect + + orig_alrm_handler = signal.signal(signal.SIGALRM, handler) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(signal.alarm, 0) + + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + # select() is interrupted by a signal which raises an exception + with self.assertRaises(InterruptSelect): + s.select(30) + # select() was interrupted before the timeout of 30 seconds + self.assertLess(time() - t, 5.0) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_select_interrupt_noraise(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + orig_alrm_handler = signal.signal(signal.SIGALRM, lambda *args: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(signal.alarm, 0) + + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + # select() is interrupted by a signal, but the signal handler doesn't + # raise an exception, so select() should by retries with a recomputed + # timeout + self.assertFalse(s.select(1.5)) + self.assertGreaterEqual(time() - t, 1.0) + + +class ScalableSelectorMixIn: + + # see issue #18963 for why it's skipped on older OS X versions + @support.requires_mac_ver(10, 5) + @unittest.skipUnless(resource, "Test needs resource module") + def test_above_fd_setsize(self): + # A scalable implementation should have no problem with more than + # FD_SETSIZE file descriptors. Since we don't know the value, we just + # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling. + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, + (soft, hard)) + NUM_FDS = min(hard, 2**16) + except (OSError, ValueError): + NUM_FDS = soft + + # guard for already allocated FDs (stdin, stdout...) + NUM_FDS -= 32 + + s = self.SELECTOR() + self.addCleanup(s.close) + + for i in range(NUM_FDS // 2): + try: + rd, wr = self.make_socketpair() + except OSError: + # too many FDs, skip - note that we should only catch EMFILE + # here, but apparently *BSD and Solaris can fail upon connect() + # or bind() with EADDRNOTAVAIL, so let's be safe + self.skipTest("FD limit reached") + + try: + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + except OSError as e: + if e.errno == errno.ENOSPC: + # this can be raised by epoll if we go over + # fs.epoll.max_user_watches sysctl + self.skipTest("FD limit reached") + raise + + self.assertEqual(NUM_FDS // 2, len(s.select())) + + +class DefaultSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.DefaultSelector + + +class SelectSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.SelectSelector + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class PollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'PollSelector', None) + + +@unittest.skipUnless(hasattr(selectors, 'EpollSelector'), + "Test needs selectors.EpollSelector") +class EpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'EpollSelector', None) + + def test_register_file(self): + # epoll(7) returns EPERM when given a file to watch + s = self.SELECTOR() + with tempfile.NamedTemporaryFile() as f: + with self.assertRaises(IOError): + s.register(f, selectors.EVENT_READ) + # the SelectorKey has been removed + with self.assertRaises(KeyError): + s.get_key(f) + + +@unittest.skipUnless(hasattr(selectors, 'KqueueSelector'), + "Test needs selectors.KqueueSelector)") +class KqueueSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'KqueueSelector', None) + + def test_register_bad_fd(self): + # a file descriptor that's been closed should raise an OSError + # with EBADF + s = self.SELECTOR() + bad_f = support.make_bad_fd() + with self.assertRaises(OSError) as cm: + s.register(bad_f, selectors.EVENT_READ) + self.assertEqual(cm.exception.errno, errno.EBADF) + # the SelectorKey has been removed + with self.assertRaises(KeyError): + s.get_key(bad_f) + + +@unittest.skipUnless(hasattr(selectors, 'DevpollSelector'), + "Test needs selectors.DevpollSelector") +class DevpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'DevpollSelector', None) + + + +def test_main(): + tests = [DefaultSelectorTestCase, SelectSelectorTestCase, + PollSelectorTestCase, EpollSelectorTestCase, + KqueueSelectorTestCase, DevpollSelectorTestCase] + support.run_unittest(*tests) + support.reap_children() + + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.5/test_smtpd.py b/src/greentest/3.5/test_smtpd.py new file mode 100644 index 0000000..88dbfdf --- /dev/null +++ b/src/greentest/3.5/test_smtpd.py @@ -0,0 +1,1019 @@ +import unittest +import textwrap +from test import support, mock_socket +import socket +import io +import smtpd +import asyncore + + +class DummyServer(smtpd.SMTPServer): + def __init__(self, *args, **kwargs): + smtpd.SMTPServer.__init__(self, *args, **kwargs) + self.messages = [] + if self._decode_data: + self.return_status = 'return status' + else: + self.return_status = b'return status' + + def process_message(self, peer, mailfrom, rcpttos, data, **kw): + self.messages.append((peer, mailfrom, rcpttos, data)) + if data == self.return_status: + return '250 Okish' + if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']: + return '250 SMTPUTF8 message okish' + + +class DummyDispatcherBroken(Exception): + pass + + +class BrokenDummyServer(DummyServer): + def listen(self, num): + raise DummyDispatcherBroken() + + +class SMTPDServerTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def test_process_message_unimplemented(self): + server = smtpd.SMTPServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + + write_line(b'HELO example') + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') + + def test_decode_data_default_warns(self): + with self.assertWarns(DeprecationWarning): + smtpd.SMTPServer((support.HOST, 0), ('b', 0)) + + def test_decode_data_and_enable_SMTPUTF8_raises(self): + self.assertRaises( + ValueError, + smtpd.SMTPServer, + (support.HOST, 0), + ('b', 0), + enable_SMTPUTF8=True, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class DebuggingServerTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def send_data(self, channel, data, enable_SMTPUTF8=False): + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + write_line(b'EHLO example') + if enable_SMTPUTF8: + write_line(b'MAIL From:eggs@example BODY=8BITMIME SMTPUTF8') + else: + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + write_line(data) + write_line(b'.') + + def test_process_message_with_decode_data_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nhello\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + From: test + X-Peer: peer-address + + hello + ------------ END MESSAGE ------------ + """)) + + def test_process_message_with_decode_data_false(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + decode_data=False) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def test_process_message_with_enable_SMTPUTF8_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def test_process_SMTPUTF8_message_with_enable_SMTPUTF8_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n', + enable_SMTPUTF8=True) + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + mail options: ['BODY=8BITMIME', 'SMTPUTF8'] + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class TestFamilyDetection(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + @unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") + def test_socket_uses_IPv6(self): + server = smtpd.SMTPServer((support.HOSTv6, 0), (support.HOST, 0), + decode_data=False) + self.assertEqual(server.socket.family, socket.AF_INET6) + + def test_socket_uses_IPv4(self): + server = smtpd.SMTPServer((support.HOST, 0), (support.HOSTv6, 0), + decode_data=False) + self.assertEqual(server.socket.family, socket.AF_INET) + + +class TestRcptOptionParsing(unittest.TestCase): + error_response = (b'555 RCPT TO parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_params_rejected(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: foo=bar') + self.assertEqual(channel.socket.last, self.error_response) + + def test_nothing_accepted(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: ') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class TestMailOptionParsing(unittest.TestCase): + error_response = (b'555 MAIL FROM parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_with_decode_data_true(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + b'MAIL from: size=20 BODY=UNKNOWN', + b'MAIL from: size=20 body=8bitmime', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line(channel, b'MAIL from: size=20') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_decode_data_false(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=False) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=False) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line( + channel, + b'MAIL from: size=20 SMTPUTF8 BODY=UNKNOWN') + self.assertEqual( + channel.socket.last, + b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n') + self.write_line( + channel, b'MAIL from: size=20 body=8bitmime') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_enable_smtputf8_true(self): + server = DummyServer((support.HOST, 0), ('b', 0), enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + self.write_line(channel, b'EHLO example') + self.write_line( + channel, + b'MAIL from: size=20 body=8bitmime smtputf8') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class SMTPDChannelTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_broken_connect(self): + self.assertRaises( + DummyDispatcherBroken, BrokenDummyServer, + (support.HOST, 0), ('b', 0), decode_data=True) + + def test_decode_data_and_enable_SMTPUTF8_raises(self): + self.assertRaises( + ValueError, smtpd.SMTPChannel, + self.server, self.channel.conn, self.channel.addr, + enable_SMTPUTF8=True, decode_data=True) + + def test_server_accept(self): + self.server.handle_accept() + + def test_missing_data(self): + self.write_line(b'') + self.assertEqual(self.channel.socket.last, + b'500 Error: bad syntax\r\n') + + def test_EHLO(self): + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, b'250 HELP\r\n') + + def test_EHLO_bad_syntax(self): + self.write_line(b'EHLO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: EHLO hostname\r\n') + + def test_EHLO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_EHLO_HELO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO(self): + name = smtpd.socket.getfqdn() + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + '250 {}\r\n'.format(name).encode('ascii')) + + def test_HELO_EHLO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELP(self): + self.write_line(b'HELP') + self.assertEqual(self.channel.socket.last, + b'250 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELP_command(self): + self.write_line(b'HELP MAIL') + self.assertEqual(self.channel.socket.last, + b'250 Syntax: MAIL FROM:
\r\n') + + def test_HELP_command_unknown(self): + self.write_line(b'HELP SPAM') + self.assertEqual(self.channel.socket.last, + b'501 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELO_bad_syntax(self): + self.write_line(b'HELO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: HELO hostname\r\n') + + def test_HELO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO_parameter_rejected_when_extensions_not_enabled(self): + self.extended_smtp = False + self.write_line(b'HELO example') + self.write_line(b'MAIL from: SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_allows_space_after_colon(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_extended_MAIL_allows_space_after_colon(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: size=20') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_NOOP(self): + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_HELO_NOOP(self): + self.write_line(b'HELO example') + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_NOOP_bad_syntax(self): + self.write_line(b'NOOP hi') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: NOOP\r\n') + + def test_QUIT(self): + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_HELO_QUIT(self): + self.write_line(b'HELO example') + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_QUIT_arg_ignored(self): + self.write_line(b'QUIT bye bye') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_bad_state(self): + self.channel.smtp_state = 'BAD STATE' + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'451 Internal confusion\r\n') + + def test_command_too_long(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ' + + b'a' * self.channel.command_size_limit + + b'@example') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_limit_extended_with_SIZE(self): + self.write_line(b'EHLO example') + fill_len = self.channel.command_size_limit - len('MAIL from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 26) + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_rejects_SMTPUTF8_by_default(self): + self.write_line(b'EHLO example') + self.write_line( + b'MAIL from: BODY=8BITMIME SMTPUTF8') + self.assertEqual(self.channel.socket.last[0:1], b'5') + + def test_data_longer_than_default_data_size_limit(self): + # Hack the default so we don't have to generate so much data. + self.channel.data_size_limit = 1048 + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'A' * self.channel.data_size_limit + + b'A\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + def test_MAIL_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=512') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_MAIL_invalid_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=invalid') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_RCPT_unknown_parameters(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 MAIL FROM parameters not recognized or not implemented\r\n') + + self.write_line(b'MAIL FROM:') + self.write_line(b'RCPT TO: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 RCPT TO parameters not recognized or not implemented\r\n') + + def test_MAIL_size_parameter_larger_than_default_data_size_limit(self): + self.channel.data_size_limit = 1048 + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=2096') + self.assertEqual(self.channel.socket.last, + b'552 Error: message size exceeds fixed maximum message size\r\n') + + def test_need_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'RCPT to:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: need MAIL command\r\n') + + def test_MAIL_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_missing_address(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_chevrons(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_empty_chevrons(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from:<>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_quoted_localpart(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_nested_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:eggs@example') + self.write_line(b'MAIL from:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: nested MAIL command\r\n') + + def test_VRFY(self): + self.write_line(b'VRFY eggs@example') + self.assertEqual(self.channel.socket.last, + b'252 Cannot VRFY user, but will accept message and attempt ' + \ + b'delivery\r\n') + + def test_VRFY_syntax(self): + self.write_line(b'VRFY') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: VRFY
\r\n') + + def test_EXPN_not_implemented(self): + self.write_line(b'EXPN') + self.assertEqual(self.channel.socket.last, + b'502 EXPN not implemented\r\n') + + def test_no_HELO_MAIL(self): + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_need_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'503 Error: need RCPT command\r\n') + + def test_RCPT_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
\r\n') + + def test_RCPT_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
[SP ]\r\n') + + def test_RCPT_lowercase_to_OK(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to: ') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_no_HELO_RCPT(self): + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example'], + 'data\nmore')]) + + def test_DATA_syntax(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n') + + def test_no_HELO_DATA(self): + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_transparency_section_4_5_2(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'..\r\n.\r\n') + self.assertEqual(self.channel.received_data, '.') + + def test_multiple_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RCPT To:ham@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example','ham@example'], + 'data')]) + + def test_manual_status(self): + # checks that the Channel is able to return a custom status message + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'return status\r\n.') + self.assertEqual(self.channel.socket.last, b'250 Okish\r\n') + + def test_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'MAIL From:foo@example') + self.write_line(b'RCPT To:eggs@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'foo@example', + ['eggs@example'], + 'data')]) + + def test_HELO_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_RSET_syntax(self): + self.write_line(b'RSET hi') + self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n') + + def test_unknown_command(self): + self.write_line(b'UNKNOWN_CMD') + self.assertEqual(self.channel.socket.last, + b'500 Error: command "UNKNOWN_CMD" not ' + \ + b'recognized\r\n') + + def test_attribute_deprecations(self): + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__server + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__server = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__line + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__line = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__state + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__state = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__greeting + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__greeting = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__mailfrom + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__mailfrom = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__rcpttos + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__rcpttos = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__data + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__data = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__fqdn + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__fqdn = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__peer + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__peer = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__conn + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__conn = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__addr + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__addr = 'spam' + + def test_decode_data_default_warning(self): + with self.assertWarns(DeprecationWarning): + server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = self.server.accept() + with self.assertWarns(DeprecationWarning): + smtpd.SMTPChannel(server, conn, addr) + +@unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") +class SMTPDChannelIPv6Test(SMTPDChannelTest): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOSTv6, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + +class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set DATA size limit to 32 bytes for easy testing + self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_data_limit_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example'], + 'data\nmore')]) + + def test_data_limit_dialog_too_much_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'This message is longer than 32 bytes\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + +class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=False) + conn, addr = self.server.accept() + # Set decode_data to False + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=False) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, b'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87\n' + b'and some plain ascii') + + +class SMTPDChannelWithDecodeDataTrue(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set decode_data to True + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, 'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + 'utf8 enriched text: żźć\nand some plain ascii') + + +class SMTPDChannelTestWithEnableSMTPUTF8True(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + enable_SMTPUTF8=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_MAIL_command_accepts_SMTPUTF8_when_announced(self): + self.write_line(b'EHLO example') + self.write_line( + 'MAIL from: BODY=8BITMIME SMTPUTF8'.encode( + 'utf-8') + ) + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_process_smtputf8_message(self): + self.write_line(b'EHLO example') + for mail_parameters in [b'', b'BODY=8BITMIME SMTPUTF8']: + self.write_line(b'MAIL from: ' + mail_parameters) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'rcpt to:') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'data') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'c\r\n.') + if mail_parameters == b'': + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + else: + self.assertEqual(self.channel.socket.last, + b'250 SMTPUTF8 message okish\r\n') + + def test_utf8_data(self): + self.write_line(b'EHLO example') + self.write_line( + 'MAIL From: naïve@examplé BODY=8BITMIME SMTPUTF8'.encode('utf-8')) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line('RCPT To:späm@examplé'.encode('utf-8')) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + + def test_MAIL_command_limit_extended_with_SIZE_and_SMTPUTF8(self): + self.write_line(b'ehlo example') + fill_len = (512 + 26 + 10) - len('mail from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 1) + + b'@example>') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_multiple_emails_with_extended_command_length(self): + self.write_line(b'ehlo example') + fill_len = (512 + 26 + 10) - len('mail from:<@example>') + for char in [b'a', b'b', b'c']: + self.write_line(b'MAIL from:<' + char * fill_len + b'a@example>') + self.assertEqual(self.channel.socket.last[0:3], b'500') + self.write_line(b'MAIL from:<' + char * fill_len + b'@example>') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'rcpt to:') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'data') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'test\r\n.') + self.assertEqual(self.channel.socket.last[0:3], b'250') + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5/test_socket.py b/src/greentest/3.5/test_socket.py new file mode 100644 index 0000000..32ab2b6 --- /dev/null +++ b/src/greentest/3.5/test_socket.py @@ -0,0 +1,5394 @@ +import unittest +from test import support + +import errno +import io +import itertools +import socket +import select +import tempfile +import time +import traceback +import queue +import sys +import os +import array +import platform +import contextlib +from weakref import proxy +import signal +import math +import pickle +import struct +import random +import string +try: + import multiprocessing +except ImportError: + multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None + +HOST = support.HOST +MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return + +try: + import _thread as thread + import threading +except ImportError: + thread = None + threading = None +try: + import _socket +except ImportError: + _socket = None + + +def _have_socket_can(): + """Check whether CAN sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_rds(): + """Check whether RDS sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +HAVE_SOCKET_CAN = _have_socket_can() + +HAVE_SOCKET_RDS = _have_socket_rds() + +# Size in bytes of the int type +SIZEOF_INT = array.array("i").itemsize + +class SocketTCPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.serv.listen() + + def tearDown(self): + self.serv.close() + self.serv = None + +class SocketUDPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.port = support.bind_port(self.serv) + + def tearDown(self): + self.serv.close() + self.serv = None + +class ThreadSafeCleanupTestCase(unittest.TestCase): + """Subclass of unittest.TestCase with thread-safe cleanup methods. + + This subclass protects the addCleanup() and doCleanups() methods + with a recursive lock. + """ + + if threading: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cleanup_lock = threading.RLock() + + def addCleanup(self, *args, **kwargs): + with self._cleanup_lock: + return super().addCleanup(*args, **kwargs) + + def doCleanups(self, *args, **kwargs): + with self._cleanup_lock: + return super().doCleanups(*args, **kwargs) + +class SocketCANTest(unittest.TestCase): + + """To be able to run this test, a `vcan0` CAN interface can be created with + the following commands: + # modprobe vcan + # ip link add dev vcan0 type vcan + # ifconfig vcan0 up + """ + interface = 'vcan0' + bufsize = 128 + + """The CAN frame structure is defined in : + + struct can_frame { + canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ + __u8 can_dlc; /* data length code: 0 .. 8 */ + __u8 data[8] __attribute__((aligned(8))); + }; + """ + can_frame_fmt = "=IB3x8s" + can_frame_size = struct.calcsize(can_frame_fmt) + + """The Broadcast Management Command frame structure is defined + in : + + struct bcm_msg_head { + __u32 opcode; + __u32 flags; + __u32 count; + struct timeval ival1, ival2; + canid_t can_id; + __u32 nframes; + struct can_frame frames[0]; + } + + `bcm_msg_head` must be 8 bytes aligned because of the `frames` member (see + `struct can_frame` definition). Must use native not standard types for packing. + """ + bcm_cmd_msg_fmt = "@3I4l2I" + bcm_cmd_msg_fmt += "x" * (struct.calcsize(bcm_cmd_msg_fmt) % 8) + + def setUp(self): + self.s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + self.addCleanup(self.s.close) + try: + self.s.bind((self.interface,)) + except OSError: + self.skipTest('network interface `%s` does not exist' % + self.interface) + + +class SocketRDSTest(unittest.TestCase): + + """To be able to run this test, the `rds` kernel module must be loaded: + # modprobe rds + """ + bufsize = 8192 + + def setUp(self): + self.serv = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + self.addCleanup(self.serv.close) + try: + self.port = support.bind_port(self.serv) + except OSError: + self.skipTest('unable to bind RDS socket') + + +class ThreadableTest: + """Threadable Test class + + The ThreadableTest class makes it easy to create a threaded + client/server pair from an existing unit test. To create a + new threaded class from an existing unit test, use multiple + inheritance: + + class NewClass (OldClass, ThreadableTest): + pass + + This class defines two new fixture functions with obvious + purposes for overriding: + + clientSetUp () + clientTearDown () + + Any new test functions within the class must then define + tests in pairs, where the test name is preceded with a + '_' to indicate the client portion of the test. Ex: + + def testFoo(self): + # Server portion + + def _testFoo(self): + # Client portion + + Any exceptions raised by the clients during their tests + are caught and transferred to the main thread to alert + the testing framework. + + Note, the server setup function cannot call any blocking + functions that rely on the client thread during setup, + unless serverExplicitReady() is called just before + the blocking call (such as in setting up a client/server + connection and performing the accept() in setUp(). + """ + + def __init__(self): + # Swap the true setup function + self.__setUp = self.setUp + self.__tearDown = self.tearDown + self.setUp = self._setUp + self.tearDown = self._tearDown + + def serverExplicitReady(self): + """This method allows the server to explicitly indicate that + it wants the client thread to proceed. This is useful if the + server is about to execute a blocking routine that is + dependent upon the client thread during its setup routine.""" + self.server_ready.set() + + def _setUp(self): + self.server_ready = threading.Event() + self.client_ready = threading.Event() + self.done = threading.Event() + self.queue = queue.Queue(1) + self.server_crashed = False + + # Do some munging to start the client test. + methodname = self.id() + i = methodname.rfind('.') + methodname = methodname[i+1:] + test_method = getattr(self, '_' + methodname) + self.client_thread = thread.start_new_thread( + self.clientRun, (test_method,)) + + try: + self.__setUp() + except: + self.server_crashed = True + raise + finally: + self.server_ready.set() + self.client_ready.wait() + + def _tearDown(self): + self.__tearDown() + self.done.wait() + + if self.queue.qsize(): + exc = self.queue.get() + raise exc + + def clientRun(self, test_func): + self.server_ready.wait() + self.clientSetUp() + self.client_ready.set() + if self.server_crashed: + self.clientTearDown() + return + if not hasattr(test_func, '__call__'): + raise TypeError("test_func must be a callable function") + try: + test_func() + except BaseException as e: + self.queue.put(e) + finally: + self.clientTearDown() + + def clientSetUp(self): + raise NotImplementedError("clientSetUp must be implemented.") + + def clientTearDown(self): + self.done.set() + thread.exit() + +class ThreadedTCPSocketTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketUDPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedCANSocketTest(SocketCANTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketCANTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + try: + self.cli.bind((self.interface,)) + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedRDSSocketTest(SocketRDSTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketRDSTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + try: + # RDS sockets must be bound explicitly to send or receive data + self.cli.bind((HOST, 0)) + self.cli_addr = self.cli.getsockname() + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class SocketConnectedTest(ThreadedTCPSocketTest): + """Socket tests for client-server connection. + + self.cli_conn is a client socket connected to the server. The + setUp() method guarantees that it is connected to the server. + """ + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + ThreadedTCPSocketTest.setUp(self) + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + ThreadedTCPSocketTest.tearDown(self) + + def clientSetUp(self): + ThreadedTCPSocketTest.clientSetUp(self) + self.cli.connect((HOST, self.port)) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + ThreadedTCPSocketTest.clientTearDown(self) + +class SocketPairTest(unittest.TestCase, ThreadableTest): + + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.serv, self.cli = socket.socketpair() + + def tearDown(self): + self.serv.close() + self.serv = None + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +# The following classes are used by the sendmsg()/recvmsg() tests. +# Combining, for instance, ConnectedStreamTestMixin and TCPTestBase +# gives a drop-in replacement for SocketConnectedTest, but different +# address families can be used, and the attributes serv_addr and +# cli_addr will be set to the addresses of the endpoints. + +class SocketTestBase(unittest.TestCase): + """A base class for socket tests. + + Subclasses must provide methods newSocket() to return a new socket + and bindSock(sock) to bind it to an unused address. + + Creates a socket self.serv and sets self.serv_addr to its address. + """ + + def setUp(self): + self.serv = self.newSocket() + self.bindServer() + + def bindServer(self): + """Bind server socket and set self.serv_addr to its address.""" + self.bindSock(self.serv) + self.serv_addr = self.serv.getsockname() + + def tearDown(self): + self.serv.close() + self.serv = None + + +class SocketListeningTestMixin(SocketTestBase): + """Mixin to listen on the server socket.""" + + def setUp(self): + super().setUp() + self.serv.listen() + + +class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, + ThreadableTest): + """Mixin to add client socket and allow client/server tests. + + Client socket is self.cli and its address is self.cli_addr. See + ThreadableTest for usage information. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = self.newClientSocket() + self.bindClient() + + def newClientSocket(self): + """Return a new socket for use as client.""" + return self.newSocket() + + def bindClient(self): + """Bind client socket and set self.cli_addr to its address.""" + self.bindSock(self.cli) + self.cli_addr = self.cli.getsockname() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +class ConnectedStreamTestMixin(SocketListeningTestMixin, + ThreadedSocketTestMixin): + """Mixin to allow client/server stream tests with connected client. + + Server's socket representing connection to client is self.cli_conn + and client's connection to server is self.serv_conn. (Based on + SocketConnectedTest.) + """ + + def setUp(self): + super().setUp() + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + super().tearDown() + + def clientSetUp(self): + super().clientSetUp() + self.cli.connect(self.serv_addr) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + super().clientTearDown() + + +class UnixSocketTestBase(SocketTestBase): + """Base class for Unix-domain socket tests.""" + + # This class is used for file descriptor passing tests, so we + # create the sockets in a private directory so that other users + # can't send anything that might be problematic for a privileged + # user running the tests. + + def setUp(self): + self.dir_path = tempfile.mkdtemp() + self.addCleanup(os.rmdir, self.dir_path) + super().setUp() + + def bindSock(self, sock): + path = tempfile.mktemp(dir=self.dir_path) + sock.bind(path) + self.addCleanup(support.unlink, path) + +class UnixStreamBase(UnixSocketTestBase): + """Base class for Unix-domain SOCK_STREAM tests.""" + + def newSocket(self): + return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + +class InetTestBase(SocketTestBase): + """Base class for IPv4 socket tests.""" + + host = HOST + + def setUp(self): + super().setUp() + self.port = self.serv_addr[1] + + def bindSock(self, sock): + support.bind_port(sock, host=self.host) + +class TCPTestBase(InetTestBase): + """Base class for TCP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + +class UDPTestBase(InetTestBase): + """Base class for UDP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + +class SCTPStreamBase(InetTestBase): + """Base class for SCTP tests in one-to-one (SOCK_STREAM) mode.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM, + socket.IPPROTO_SCTP) + + +class Inet6TestBase(InetTestBase): + """Base class for IPv6 socket tests.""" + + host = support.HOSTv6 + +class UDP6TestBase(Inet6TestBase): + """Base class for UDP-over-IPv6 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + + +# Test-skipping decorators for use with ThreadableTest. + +def skipWithClientIf(condition, reason): + """Skip decorated test if condition is true, add client_skip decorator. + + If the decorated object is not a class, sets its attribute + "client_skip" to a decorator which will return an empty function + if the test is to be skipped, or the original function if it is + not. This can be used to avoid running the client part of a + skipped test when using ThreadableTest. + """ + def client_pass(*args, **kwargs): + pass + def skipdec(obj): + retval = unittest.skip(reason)(obj) + if not isinstance(obj, type): + retval.client_skip = lambda f: client_pass + return retval + def noskipdec(obj): + if not (isinstance(obj, type) or hasattr(obj, "client_skip")): + obj.client_skip = lambda f: f + return obj + return skipdec if condition else noskipdec + + +def requireAttrs(obj, *attributes): + """Skip decorated test if obj is missing any of the given attributes. + + Sets client_skip attribute as skipWithClientIf() does. + """ + missing = [name for name in attributes if not hasattr(obj, name)] + return skipWithClientIf( + missing, "don't have " + ", ".join(name for name in missing)) + + +def requireSocket(*args): + """Skip decorated test if a socket cannot be created with given arguments. + + When an argument is given as a string, will use the value of that + attribute of the socket module, or skip the test if it doesn't + exist. Sets client_skip attribute as skipWithClientIf() does. + """ + err = None + missing = [obj for obj in args if + isinstance(obj, str) and not hasattr(socket, obj)] + if missing: + err = "don't have " + ", ".join(name for name in missing) + else: + callargs = [getattr(socket, obj) if isinstance(obj, str) else obj + for obj in args] + try: + s = socket.socket(*callargs) + except OSError as e: + # XXX: check errno? + err = str(e) + else: + s.close() + return skipWithClientIf( + err is not None, + "can't create socket({0}): {1}".format( + ", ".join(str(o) for o in args), err)) + + +####################################################################### +## Begin Tests + +class GeneralModuleTests(unittest.TestCase): + + def test_SocketType_is_socketobject(self): + import _socket + self.assertTrue(socket.SocketType is _socket.socket) + s = socket.socket() + self.assertIsInstance(s, socket.SocketType) + s.close() + + def test_repr(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with s: + self.assertIn('fd=%i' % s.fileno(), repr(s)) + self.assertIn('family=%s' % socket.AF_INET, repr(s)) + self.assertIn('type=%s' % socket.SOCK_STREAM, repr(s)) + self.assertIn('proto=0', repr(s)) + self.assertNotIn('raddr', repr(s)) + s.bind(('127.0.0.1', 0)) + self.assertIn('laddr', repr(s)) + self.assertIn(str(s.getsockname()), repr(s)) + self.assertIn('[closed]', repr(s)) + self.assertNotIn('laddr', repr(s)) + + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_csocket_repr(self): + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + try: + expected = ('' + % (s.fileno(), s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + finally: + s.close() + expected = ('' + % (s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + + def test_weakref(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + p = proxy(s) + self.assertEqual(p.fileno(), s.fileno()) + s.close() + s = None + try: + p.fileno() + except ReferenceError: + pass + else: + self.fail('Socket proxy still exists') + + def testSocketError(self): + # Testing socket module exceptions + msg = "Error raising socket exception (%s)." + with self.assertRaises(OSError, msg=msg % 'OSError'): + raise OSError + with self.assertRaises(OSError, msg=msg % 'socket.herror'): + raise socket.herror + with self.assertRaises(OSError, msg=msg % 'socket.gaierror'): + raise socket.gaierror + + def testSendtoErrors(self): + # Testing that sendto doesn't mask failures. See #10169. + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind(('', 0)) + sockname = s.getsockname() + # 2 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'str'") + with self.assertRaises(TypeError) as cm: + s.sendto(5j, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'complex'") + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None) + self.assertIn('not NoneType',str(cm.exception)) + # 3 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', 0, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'str'") + with self.assertRaises(TypeError) as cm: + s.sendto(5j, 0, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'complex'") + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, None) + self.assertIn('not NoneType', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 'bar', sockname) + self.assertIn('an integer is required', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None, None) + self.assertIn('an integer is required', str(cm.exception)) + # wrong number of args + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo') + self.assertIn('(1 given)', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, sockname, 4) + self.assertIn('(4 given)', str(cm.exception)) + + def testCrucialConstants(self): + # Testing for mission critical constants + socket.AF_INET + socket.SOCK_STREAM + socket.SOCK_DGRAM + socket.SOCK_RAW + socket.SOCK_RDM + socket.SOCK_SEQPACKET + socket.SOL_SOCKET + socket.SO_REUSEADDR + + def testHostnameRes(self): + # Testing hostname resolution mechanisms + hostname = socket.gethostname() + try: + ip = socket.gethostbyname(hostname) + except OSError: + # Probably name lookup wasn't set up right; skip this test + self.skipTest('name lookup failure') + self.assertTrue(ip.find('.') >= 0, "Error resolving host to ip.") + try: + hname, aliases, ipaddrs = socket.gethostbyaddr(ip) + except OSError: + # Probably a similar problem as above; skip this test + self.skipTest('name lookup failure') + all_host_names = [hostname, hname] + aliases + fqhn = socket.getfqdn(ip) + if not fqhn in all_host_names: + self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + + def test_host_resolution(self): + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + self.assertRaises(OSError, socket.gethostbyname, addr) + self.assertRaises(OSError, socket.gethostbyaddr, addr) + + for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: + self.assertEqual(socket.gethostbyname(addr), addr) + + # we don't test support.HOSTv6 because there's a chance it doesn't have + # a matching name entry (e.g. 'ip6-localhost') + for host in [support.HOST]: + self.assertIn(host, socket.gethostbyaddr(host)[2]) + + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") + @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") + def test_sethostname(self): + oldhn = socket.gethostname() + try: + socket.sethostname('new') + except OSError as e: + if e.errno == errno.EPERM: + self.skipTest("test should be run as root") + else: + raise + try: + # running test as root! + self.assertEqual(socket.gethostname(), 'new') + # Should work with bytes objects too + socket.sethostname(b'bar') + self.assertEqual(socket.gethostname(), 'bar') + finally: + socket.sethostname(oldhn) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInterfaceNameIndex(self): + interfaces = socket.if_nameindex() + for index, name in interfaces: + self.assertIsInstance(index, int) + self.assertIsInstance(name, str) + # interface indices are non-zero integers + self.assertGreater(index, 0) + _index = socket.if_nametoindex(name) + self.assertIsInstance(_index, int) + self.assertEqual(index, _index) + _name = socket.if_indextoname(index) + self.assertIsInstance(_name, str) + self.assertEqual(name, _name) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInvalidInterfaceNameIndex(self): + # test nonexistent interface index/name + self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') + # test with invalid values + self.assertRaises(TypeError, socket.if_nametoindex, 0) + self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') + def testRefCountGetNameInfo(self): + # Testing reference count for getnameinfo + try: + # On some versions, this loses a reference + orig = sys.getrefcount(__name__) + socket.getnameinfo(__name__,0) + except TypeError: + if sys.getrefcount(__name__) != orig: + self.fail("socket.getnameinfo loses a reference") + + def testInterpreterCrash(self): + # Making sure getnameinfo doesn't crash the interpreter + try: + # On some versions, this crashes the interpreter. + socket.getnameinfo(('x', 0, 0, 0), 0) + except OSError: + pass + + def testNtoH(self): + # This just checks that htons etc. are their own inverse, + # when looking at the lower 16 or 32 bits. + sizes = {socket.htonl: 32, socket.ntohl: 32, + socket.htons: 16, socket.ntohs: 16} + for func, size in sizes.items(): + mask = (1<") + + def test_unusable_closed_socketio(self): + with socket.socket() as sock: + fp = sock.makefile("rb", buffering=0) + self.assertTrue(fp.readable()) + self.assertFalse(fp.writable()) + self.assertFalse(fp.seekable()) + fp.close() + self.assertRaises(ValueError, fp.readable) + self.assertRaises(ValueError, fp.writable) + self.assertRaises(ValueError, fp.seekable) + + def test_makefile_mode(self): + for mode in 'r', 'rb', 'rw', 'w', 'wb': + with self.subTest(mode=mode): + with socket.socket() as sock: + with sock.makefile(mode) as fp: + self.assertEqual(fp.mode, mode) + + def test_makefile_invalid_mode(self): + for mode in 'rt', 'x', '+', 'a': + with self.subTest(mode=mode): + with socket.socket() as sock: + with self.assertRaisesRegex(ValueError, 'invalid mode'): + sock.makefile(mode) + + def test_pickle(self): + sock = socket.socket() + with sock: + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, sock, protocol) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + family = pickle.loads(pickle.dumps(socket.AF_INET, protocol)) + self.assertEqual(family, socket.AF_INET) + type = pickle.loads(pickle.dumps(socket.SOCK_STREAM, protocol)) + self.assertEqual(type, socket.SOCK_STREAM) + + def test_listen_backlog(self): + for backlog in 0, -1: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv: + srv.bind((HOST, 0)) + srv.listen(backlog) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv: + srv.bind((HOST, 0)) + srv.listen() + + @support.cpython_only + def test_listen_backlog_overflow(self): + # Issue 15989 + import _testcapi + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.bind((HOST, 0)) + self.assertRaises(OverflowError, srv.listen, _testcapi.INT_MAX + 1) + srv.close() + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + def test_flowinfo(self): + self.assertRaises(OverflowError, socket.getnameinfo, + (support.HOSTv6, 0, 0xffffffff), 0) + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + self.assertRaises(OverflowError, s.bind, (support.HOSTv6, 0, -10)) + + def test_str_for_enums(self): + # Make sure that the AF_* and SOCK_* constants have enum-like string + # reprs. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + self.assertEqual(str(s.family), 'AddressFamily.AF_INET') + self.assertEqual(str(s.type), 'SocketKind.SOCK_STREAM') + + @unittest.skipIf(os.name == 'nt', 'Will not work on Windows') + def test_uknown_socket_family_repr(self): + # Test that when created with a family that's not one of the known + # AF_*/SOCK_* constants, socket.family just returns the number. + # + # To do this we fool socket.socket into believing it already has an + # open fd because on this path it doesn't actually verify the family and + # type and populates the socket object. + # + # On Windows this trick won't work, so the test is skipped. + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + with socket.socket(family=42424, type=13331, fileno=fd) as s: + self.assertEqual(s.family, 42424) + self.assertEqual(s.type, 13331) + + @unittest.skipUnless(hasattr(os, 'sendfile'), 'test needs os.sendfile()') + def test__sendfile_use_sendfile(self): + class File: + def __init__(self, fd): + self.fd = fd + + def fileno(self): + return self.fd + with socket.socket() as sock: + fd = os.open(os.curdir, os.O_RDONLY) + os.close(fd) + with self.assertRaises(socket._GiveupOnSendfile): + sock._sendfile_use_sendfile(File(fd)) + with self.assertRaises(OverflowError): + sock._sendfile_use_sendfile(File(2**1000)) + with self.assertRaises(TypeError): + sock._sendfile_use_sendfile(File(None)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +class BasicCANTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_CAN + socket.PF_CAN + socket.CAN_RAW + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCMConstants(self): + socket.CAN_BCM + + # opcodes + socket.CAN_BCM_TX_SETUP # create (cyclic) transmission task + socket.CAN_BCM_TX_DELETE # remove (cyclic) transmission task + socket.CAN_BCM_TX_READ # read properties of (cyclic) transmission task + socket.CAN_BCM_TX_SEND # send one CAN frame + socket.CAN_BCM_RX_SETUP # create RX content filter subscription + socket.CAN_BCM_RX_DELETE # remove RX content filter subscription + socket.CAN_BCM_RX_READ # read properties of RX content filter subscription + socket.CAN_BCM_TX_STATUS # reply to TX_READ request + socket.CAN_BCM_TX_EXPIRED # notification on performed transmissions (count=0) + socket.CAN_BCM_RX_STATUS # reply to RX_READ request + socket.CAN_BCM_RX_TIMEOUT # cyclic message is absent + socket.CAN_BCM_RX_CHANGED # updated CAN frame (detected content change) + + def testCreateSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + pass + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testCreateBCMSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) as s: + pass + + def testBindAny(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.bind(('', )) + + def testTooLongInterfaceName(self): + # most systems limit IFNAMSIZ to 16, take 1024 to be sure + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + self.assertRaisesRegex(OSError, 'interface name too long', + s.bind, ('x' * 1024,)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_LOOPBACK"), + 'socket.CAN_RAW_LOOPBACK required for this test.') + def testLoopback(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + for loopback in (0, 1): + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK, + loopback) + self.assertEqual(loopback, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_FILTER"), + 'socket.CAN_RAW_FILTER required for this test.') + def testFilter(self): + can_id, can_mask = 0x200, 0x700 + can_filter = struct.pack("=II", can_id, can_mask) + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, can_filter) + self.assertEqual(can_filter, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, 8)) + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, bytearray(can_filter)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class CANTest(ThreadedCANSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedCANSocketTest.__init__(self, methodName=methodName) + + @classmethod + def build_can_frame(cls, can_id, data): + """Build a CAN frame.""" + can_dlc = len(data) + data = data.ljust(8, b'\x00') + return struct.pack(cls.can_frame_fmt, can_id, can_dlc, data) + + @classmethod + def dissect_can_frame(cls, frame): + """Dissect a CAN frame.""" + can_id, can_dlc, data = struct.unpack(cls.can_frame_fmt, frame) + return (can_id, can_dlc, data[:can_dlc]) + + def testSendFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + self.assertEqual(addr[0], self.interface) + self.assertEqual(addr[1], socket.AF_CAN) + + def _testSendFrame(self): + self.cf = self.build_can_frame(0x00, b'\x01\x02\x03\x04\x05') + self.cli.send(self.cf) + + def testSendMaxFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + + def _testSendMaxFrame(self): + self.cf = self.build_can_frame(0x00, b'\x07' * 8) + self.cli.send(self.cf) + + def testSendMultiFrames(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf1, cf) + + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf2, cf) + + def _testSendMultiFrames(self): + self.cf1 = self.build_can_frame(0x07, b'\x44\x33\x22\x11') + self.cli.send(self.cf1) + + self.cf2 = self.build_can_frame(0x12, b'\x99\x22\x33') + self.cli.send(self.cf2) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def _testBCM(self): + cf, addr = self.cli.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + can_id, can_dlc, data = self.dissect_can_frame(cf) + self.assertEqual(self.can_id, can_id) + self.assertEqual(self.data, data) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCM(self): + bcm = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) + self.addCleanup(bcm.close) + bcm.connect((self.interface,)) + self.can_id = 0x123 + self.data = bytes([0xc0, 0xff, 0xee]) + self.cf = self.build_can_frame(self.can_id, self.data) + opcode = socket.CAN_BCM_TX_SEND + flags = 0 + count = 0 + ival1_seconds = ival1_usec = ival2_seconds = ival2_usec = 0 + bcm_can_id = 0x0222 + nframes = 1 + assert len(self.cf) == 16 + header = struct.pack(self.bcm_cmd_msg_fmt, + opcode, + flags, + count, + ival1_seconds, + ival1_usec, + ival2_seconds, + ival2_usec, + bcm_can_id, + nframes, + ) + header_plus_frame = header + self.cf + bytes_sent = bcm.send(header_plus_frame) + self.assertEqual(bytes_sent, len(header_plus_frame)) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +class BasicRDSTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_RDS + socket.PF_RDS + + def testCreateSocket(self): + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + pass + + def testSocketBufferSize(self): + bufsize = 16384 + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, bufsize) + s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, bufsize) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class RDSTest(ThreadedRDSSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedRDSSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + super().setUp() + self.evt = threading.Event() + + def testSendAndRecv(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + self.assertEqual(self.cli_addr, addr) + + def _testSendAndRecv(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testPeek(self): + data, addr = self.serv.recvfrom(self.bufsize, socket.MSG_PEEK) + self.assertEqual(self.data, data) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testPeek(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + @requireAttrs(socket.socket, 'recvmsg') + def testSendAndRecvMsg(self): + data, ancdata, msg_flags, addr = self.serv.recvmsg(self.bufsize) + self.assertEqual(self.data, data) + + @requireAttrs(socket.socket, 'sendmsg') + def _testSendAndRecvMsg(self): + self.data = b'hello ' * 10 + self.cli.sendmsg([self.data], (), 0, (HOST, self.port)) + + def testSendAndRecvMulti(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data1, data) + + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data2, data) + + def _testSendAndRecvMulti(self): + self.data1 = b'bacon' + self.cli.sendto(self.data1, 0, (HOST, self.port)) + + self.data2 = b'egg' + self.cli.sendto(self.data2, 0, (HOST, self.port)) + + def testSelect(self): + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testSelect(self): + self.data = b'select' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testCongestion(self): + # wait until the sender is done + self.evt.wait() + + def _testCongestion(self): + # test the behavior in case of congestion + self.data = b'fill' + self.cli.setblocking(False) + try: + # try to lower the receiver's socket buffer size + self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384) + except OSError: + pass + with self.assertRaises(OSError) as cm: + try: + # fill the receiver's socket buffer + while True: + self.cli.sendto(self.data, 0, (HOST, self.port)) + finally: + # signal the receiver we're done + self.evt.set() + # sendto() should have failed with ENOBUFS + self.assertEqual(cm.exception.errno, errno.ENOBUFS) + # and we should have received a congestion notification through poll + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicTCPTest(SocketConnectedTest): + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecv(self): + # Testing large receive over TCP + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.serv_conn.send(MSG) + + def testOverFlowRecv(self): + # Testing receive in chunks over TCP + seg1 = self.cli_conn.recv(len(MSG) - 3) + seg2 = self.cli_conn.recv(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecv(self): + self.serv_conn.send(MSG) + + def testRecvFrom(self): + # Testing large recvfrom() over TCP + msg, addr = self.cli_conn.recvfrom(1024) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.serv_conn.send(MSG) + + def testOverFlowRecvFrom(self): + # Testing recvfrom() in chunks over TCP + seg1, addr = self.cli_conn.recvfrom(len(MSG)-3) + seg2, addr = self.cli_conn.recvfrom(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecvFrom(self): + self.serv_conn.send(MSG) + + def testSendAll(self): + # Testing sendall() with a 2048 byte string over TCP + msg = b'' + while 1: + read = self.cli_conn.recv(1024) + if not read: + break + msg += read + self.assertEqual(msg, b'f' * 2048) + + def _testSendAll(self): + big_chunk = b'f' * 2048 + self.serv_conn.sendall(big_chunk) + + def testFromFd(self): + # Testing fromfd() + fd = self.cli_conn.fileno() + sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + self.assertIsInstance(sock, socket.socket) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testFromFd(self): + self.serv_conn.send(MSG) + + def testDup(self): + # Testing dup() + sock = self.cli_conn.dup() + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDup(self): + self.serv_conn.send(MSG) + + def testShutdown(self): + # Testing shutdown() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + # wait for _testShutdown to finish: on OS X, when the server + # closes the connection the client also becomes disconnected, + # and the client's shutdown call will fail. (Issue #4397.) + self.done.wait() + + def _testShutdown(self): + self.serv_conn.send(MSG) + self.serv_conn.shutdown(2) + + testShutdown_overflow = support.cpython_only(testShutdown) + + @support.cpython_only + def _testShutdown_overflow(self): + import _testcapi + self.serv_conn.send(MSG) + # Issue 15989 + self.assertRaises(OverflowError, self.serv_conn.shutdown, + _testcapi.INT_MAX + 1) + self.assertRaises(OverflowError, self.serv_conn.shutdown, + 2 + (_testcapi.UINT_MAX + 1)) + self.serv_conn.shutdown(2) + + def testDetach(self): + # Testing detach() + fileno = self.cli_conn.fileno() + f = self.cli_conn.detach() + self.assertEqual(f, fileno) + # cli_conn cannot be used anymore... + self.assertTrue(self.cli_conn._closed) + self.assertRaises(OSError, self.cli_conn.recv, 1024) + self.cli_conn.close() + # ...but we can create another socket using the (still open) + # file descriptor + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=f) + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDetach(self): + self.serv_conn.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicUDPTest(ThreadedUDPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedUDPSocketTest.__init__(self, methodName=methodName) + + def testSendtoAndRecv(self): + # Testing sendto() and Recv() over UDP + msg = self.serv.recv(len(MSG)) + self.assertEqual(msg, MSG) + + def _testSendtoAndRecv(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFrom(self): + # Testing recvfrom() over UDP + msg, addr = self.serv.recvfrom(len(MSG)) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFromNegative(self): + # Negative lengths passed to recvfrom should give ValueError. + self.assertRaises(ValueError, self.serv.recvfrom, -1) + + def _testRecvFromNegative(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + +# Tests for the sendmsg()/recvmsg() interface. Where possible, the +# same test code is used with different families and types of socket +# (e.g. stream, datagram), and tests using recvmsg() are repeated +# using recvmsg_into(). +# +# The generic test classes such as SendmsgTests and +# RecvmsgGenericTests inherit from SendrecvmsgBase and expect to be +# supplied with sockets cli_sock and serv_sock representing the +# client's and the server's end of the connection respectively, and +# attributes cli_addr and serv_addr holding their (numeric where +# appropriate) addresses. +# +# The final concrete test classes combine these with subclasses of +# SocketTestBase which set up client and server sockets of a specific +# type, and with subclasses of SendrecvmsgBase such as +# SendrecvmsgDgramBase and SendrecvmsgConnectedBase which map these +# sockets to cli_sock and serv_sock and override the methods and +# attributes of SendrecvmsgBase to fill in destination addresses if +# needed when sending, check for specific flags in msg_flags, etc. +# +# RecvmsgIntoMixin provides a version of doRecvmsg() implemented using +# recvmsg_into(). + +# XXX: like the other datagram (UDP) tests in this module, the code +# here assumes that datagram delivery on the local machine will be +# reliable. + +class SendrecvmsgBase(ThreadSafeCleanupTestCase): + # Base class for sendmsg()/recvmsg() tests. + + # Time in seconds to wait before considering a test failed, or + # None for no timeout. Not all tests actually set a timeout. + fail_timeout = 3.0 + + def setUp(self): + self.misc_event = threading.Event() + super().setUp() + + def sendToServer(self, msg): + # Send msg to the server. + return self.cli_sock.send(msg) + + # Tuple of alternative default arguments for sendmsg() when called + # via sendmsgToServer() (e.g. to include a destination address). + sendmsg_to_server_defaults = () + + def sendmsgToServer(self, *args): + # Call sendmsg() on self.cli_sock with the given arguments, + # filling in any arguments which are not supplied with the + # corresponding items of self.sendmsg_to_server_defaults, if + # any. + return self.cli_sock.sendmsg( + *(args + self.sendmsg_to_server_defaults[len(args):])) + + def doRecvmsg(self, sock, bufsize, *args): + # Call recvmsg() on sock with given arguments and return its + # result. Should be used for tests which can use either + # recvmsg() or recvmsg_into() - RecvmsgIntoMixin overrides + # this method with one which emulates it using recvmsg_into(), + # thus allowing the same test to be used for both methods. + result = sock.recvmsg(bufsize, *args) + self.registerRecvmsgResult(result) + return result + + def registerRecvmsgResult(self, result): + # Called by doRecvmsg() with the return value of recvmsg() or + # recvmsg_into(). Can be overridden to arrange cleanup based + # on the returned ancillary data, for instance. + pass + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer. + self.assertEqual(addr1, addr2) + + # Flags that are normally unset in msg_flags + msg_flags_common_unset = 0 + for name in ("MSG_CTRUNC", "MSG_OOB"): + msg_flags_common_unset |= getattr(socket, name, 0) + + # Flags that are normally set + msg_flags_common_set = 0 + + # Flags set when a complete record has been received (e.g. MSG_EOR + # for SCTP) + msg_flags_eor_indicator = 0 + + # Flags set when a complete record has not been received + # (e.g. MSG_TRUNC for datagram sockets) + msg_flags_non_eor_indicator = 0 + + def checkFlags(self, flags, eor=None, checkset=0, checkunset=0, ignore=0): + # Method to check the value of msg_flags returned by recvmsg[_into](). + # + # Checks that all bits in msg_flags_common_set attribute are + # set in "flags" and all bits in msg_flags_common_unset are + # unset. + # + # The "eor" argument specifies whether the flags should + # indicate that a full record (or datagram) has been received. + # If "eor" is None, no checks are done; otherwise, checks + # that: + # + # * if "eor" is true, all bits in msg_flags_eor_indicator are + # set and all bits in msg_flags_non_eor_indicator are unset + # + # * if "eor" is false, all bits in msg_flags_non_eor_indicator + # are set and all bits in msg_flags_eor_indicator are unset + # + # If "checkset" and/or "checkunset" are supplied, they require + # the given bits to be set or unset respectively, overriding + # what the attributes require for those bits. + # + # If any bits are set in "ignore", they will not be checked, + # regardless of the other inputs. + # + # Will raise Exception if the inputs require a bit to be both + # set and unset, and it is not ignored. + + defaultset = self.msg_flags_common_set + defaultunset = self.msg_flags_common_unset + + if eor: + defaultset |= self.msg_flags_eor_indicator + defaultunset |= self.msg_flags_non_eor_indicator + elif eor is not None: + defaultset |= self.msg_flags_non_eor_indicator + defaultunset |= self.msg_flags_eor_indicator + + # Function arguments override defaults + defaultset &= ~checkunset + defaultunset &= ~checkset + + # Merge arguments with remaining defaults, and check for conflicts + checkset |= defaultset + checkunset |= defaultunset + inboth = checkset & checkunset & ~ignore + if inboth: + raise Exception("contradictory set, unset requirements for flags " + "{0:#x}".format(inboth)) + + # Compare with given msg_flags value + mask = (checkset | checkunset) & ~ignore + self.assertEqual(flags & mask, checkset & mask) + + +class RecvmsgIntoMixin(SendrecvmsgBase): + # Mixin to implement doRecvmsg() using recvmsg_into(). + + def doRecvmsg(self, sock, bufsize, *args): + buf = bytearray(bufsize) + result = sock.recvmsg_into([buf], *args) + self.registerRecvmsgResult(result) + self.assertGreaterEqual(result[0], 0) + self.assertLessEqual(result[0], bufsize) + return (bytes(buf[:result[0]]),) + result[1:] + + +class SendrecvmsgDgramFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for datagram sockets. + + @property + def msg_flags_non_eor_indicator(self): + return super().msg_flags_non_eor_indicator | socket.MSG_TRUNC + + +class SendrecvmsgSCTPFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for SCTP sockets. + + @property + def msg_flags_eor_indicator(self): + return super().msg_flags_eor_indicator | socket.MSG_EOR + + +class SendrecvmsgConnectionlessBase(SendrecvmsgBase): + # Base class for tests on connectionless-mode sockets. Users must + # supply sockets on attributes cli and serv to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.serv + + @property + def cli_sock(self): + return self.cli + + @property + def sendmsg_to_server_defaults(self): + return ([], [], 0, self.serv_addr) + + def sendToServer(self, msg): + return self.cli_sock.sendto(msg, self.serv_addr) + + +class SendrecvmsgConnectedBase(SendrecvmsgBase): + # Base class for tests on connected sockets. Users must supply + # sockets on attributes serv_conn and cli_conn (representing the + # connections *to* the server and the client), to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.cli_conn + + @property + def cli_sock(self): + return self.serv_conn + + def checkRecvmsgAddress(self, addr1, addr2): + # Address is currently "unspecified" for a connected socket, + # so we don't examine it + pass + + +class SendrecvmsgServerTimeoutBase(SendrecvmsgBase): + # Base class to set a timeout on server's socket. + + def setUp(self): + super().setUp() + self.serv_sock.settimeout(self.fail_timeout) + + +class SendmsgTests(SendrecvmsgServerTimeoutBase): + # Tests for sendmsg() which can use any socket type and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsg(self): + # Send a simple message with sendmsg(). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG]), len(MSG)) + + def testSendmsgDataGenerator(self): + # Send from buffer obtained from a generator (not a sequence). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgDataGenerator(self): + self.assertEqual(self.sendmsgToServer((o for o in [MSG])), + len(MSG)) + + def testSendmsgAncillaryGenerator(self): + # Gather (empty) ancillary data from a generator. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgAncillaryGenerator(self): + self.assertEqual(self.sendmsgToServer([MSG], (o for o in [])), + len(MSG)) + + def testSendmsgArray(self): + # Send data from an array instead of the usual bytes object. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgArray(self): + self.assertEqual(self.sendmsgToServer([array.array("B", MSG)]), + len(MSG)) + + def testSendmsgGather(self): + # Send message data from more than one buffer (gather write). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgGather(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + def testSendmsgBadArgs(self): + # Check that sendmsg() rejects invalid arguments. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadArgs(self): + self.assertRaises(TypeError, self.cli_sock.sendmsg) + self.assertRaises(TypeError, self.sendmsgToServer, + b"not in an iterable") + self.assertRaises(TypeError, self.sendmsgToServer, + object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG, object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], 0, object()) + self.sendToServer(b"done") + + def testSendmsgBadCmsg(self): + # Check that invalid ancillary data items are rejected. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(object(), 0, b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, object(), b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, object())]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0)]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b"data", 42)]) + self.sendToServer(b"done") + + @requireAttrs(socket, "CMSG_SPACE") + def testSendmsgBadMultiCmsg(self): + # Check that invalid ancillary data items are rejected when + # more than one item is present. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + @testSendmsgBadMultiCmsg.client_skip + def _testSendmsgBadMultiCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [0, 0, b""]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b""), object()]) + self.sendToServer(b"done") + + def testSendmsgExcessCmsgReject(self): + # Check that sendmsg() rejects excess ancillary data items + # when the number that can be sent is limited. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgExcessCmsgReject(self): + if not hasattr(socket, "CMSG_SPACE"): + # Can only send one item + with self.assertRaises(OSError) as cm: + self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")]) + self.assertIsNone(cm.exception.errno) + self.sendToServer(b"done") + + def testSendmsgAfterClose(self): + # Check that sendmsg() fails on a closed socket. + pass + + def _testSendmsgAfterClose(self): + self.cli_sock.close() + self.assertRaises(OSError, self.sendmsgToServer, [MSG]) + + +class SendmsgStreamTests(SendmsgTests): + # Tests for sendmsg() which require a stream socket and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsgExplicitNoneAddr(self): + # Check that peer address can be specified as None. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgExplicitNoneAddr(self): + self.assertEqual(self.sendmsgToServer([MSG], [], 0, None), len(MSG)) + + def testSendmsgTimeout(self): + # Check that timeout works with sendmsg(). + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + def _testSendmsgTimeout(self): + try: + self.cli_sock.settimeout(0.03) + with self.assertRaises(socket.timeout): + while True: + self.sendmsgToServer([b"a"*512]) + finally: + self.misc_event.set() + + # XXX: would be nice to have more tests for sendmsg flags argument. + + # Linux supports MSG_DONTWAIT when sending, but in general, it + # only works when receiving. Could add other platforms if they + # support it too. + @skipWithClientIf(sys.platform not in {"linux"}, + "MSG_DONTWAIT not known to work on this platform when " + "sending") + def testSendmsgDontWait(self): + # Check that MSG_DONTWAIT in flags causes non-blocking behaviour. + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @testSendmsgDontWait.client_skip + def _testSendmsgDontWait(self): + try: + with self.assertRaises(OSError) as cm: + while True: + self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT) + self.assertIn(cm.exception.errno, + (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + self.misc_event.set() + + +class SendmsgConnectionlessTests(SendmsgTests): + # Tests for sendmsg() which require a connectionless-mode + # (e.g. datagram) socket, and do not involve recvmsg() or + # recvmsg_into(). + + def testSendmsgNoDestAddr(self): + # Check that sendmsg() fails when no destination address is + # given for unconnected socket. + pass + + def _testSendmsgNoDestAddr(self): + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG]) + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG], [], 0, None) + + +class RecvmsgGenericTests(SendrecvmsgBase): + # Tests for recvmsg() which can also be emulated using + # recvmsg_into(), and can use any socket type. + + def testRecvmsg(self): + # Receive a simple message with recvmsg[_into](). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsg(self): + self.sendToServer(MSG) + + def testRecvmsgExplicitDefaults(self): + # Test recvmsg[_into]() with default arguments provided explicitly. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgExplicitDefaults(self): + self.sendToServer(MSG) + + def testRecvmsgShorter(self): + # Receive a message smaller than buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) + 42) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShorter(self): + self.sendToServer(MSG) + + # FreeBSD < 8 doesn't always set the MSG_TRUNC flag when a truncated + # datagram is received (issue #13001). + @support.requires_freebsd_version(8) + def testRecvmsgTrunc(self): + # Receive part of message, check for truncation indicators. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + @support.requires_freebsd_version(8) + def _testRecvmsgTrunc(self): + self.sendToServer(MSG) + + def testRecvmsgShortAncillaryBuf(self): + # Test ancillary data buffer too small to hold any ancillary data. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 1) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShortAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgLongAncillaryBuf(self): + # Test large ancillary data buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgLongAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgAfterClose(self): + # Check that recvmsg[_into]() fails on a closed socket. + self.serv_sock.close() + self.assertRaises(OSError, self.doRecvmsg, self.serv_sock, 1024) + + def _testRecvmsgAfterClose(self): + pass + + def testRecvmsgTimeout(self): + # Check that timeout works. + try: + self.serv_sock.settimeout(0.03) + self.assertRaises(socket.timeout, + self.doRecvmsg, self.serv_sock, len(MSG)) + finally: + self.misc_event.set() + + def _testRecvmsgTimeout(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @requireAttrs(socket, "MSG_PEEK") + def testRecvmsgPeek(self): + # Check that MSG_PEEK in flags enables examination of pending + # data without consuming it. + + # Receive part of data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3, 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + # Ignoring MSG_TRUNC here (so this test is the same for stream + # and datagram sockets). Some wording in POSIX seems to + # suggest that it needn't be set when peeking, but that may + # just be a slip. + self.checkFlags(flags, eor=False, + ignore=getattr(socket, "MSG_TRUNC", 0)) + + # Receive all data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + # Check that the same data can still be received normally. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgPeek.client_skip + def _testRecvmsgPeek(self): + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + def testRecvmsgFromSendmsg(self): + # Test receiving with recvmsg[_into]() when message is sent + # using sendmsg(). + self.serv_sock.settimeout(self.fail_timeout) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgFromSendmsg.client_skip + def _testRecvmsgFromSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + +class RecvmsgGenericStreamTests(RecvmsgGenericTests): + # Tests which require a stream socket and can use either recvmsg() + # or recvmsg_into(). + + def testRecvmsgEOF(self): + # Receive end-of-stream indicator (b"", peer socket closed). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.assertEqual(msg, b"") + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=None) # Might not have end-of-record marker + + def _testRecvmsgEOF(self): + self.cli_sock.close() + + def testRecvmsgOverflow(self): + # Receive a message in more than one chunk. + seg1, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + seg2, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testRecvmsgOverflow(self): + self.sendToServer(MSG) + + +class RecvmsgTests(RecvmsgGenericTests): + # Tests for recvmsg() which can use any socket type. + + def testRecvmsgBadArgs(self): + # Check that recvmsg() rejects invalid arguments. + self.assertRaises(TypeError, self.serv_sock.recvmsg) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + -1, 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + len(MSG), -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + [bytearray(10)], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + object(), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), 0, object()) + + msg, ancdata, flags, addr = self.serv_sock.recvmsg(len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgBadArgs(self): + self.sendToServer(MSG) + + +class RecvmsgIntoTests(RecvmsgIntoMixin, RecvmsgGenericTests): + # Tests for recvmsg_into() which can use any socket type. + + def testRecvmsgIntoBadArgs(self): + # Check that recvmsg_into() rejects invalid arguments. + buf = bytearray(len(MSG)) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + len(MSG), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + buf, 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [object()], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [b"I'm not writable"], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf, object()], 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg_into, + [buf], -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], 0, object()) + + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf], 0, 0) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoBadArgs(self): + self.sendToServer(MSG) + + def testRecvmsgIntoGenerator(self): + # Receive into buffer obtained from a generator (not a sequence). + buf = bytearray(len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + (o for o in [buf])) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoGenerator(self): + self.sendToServer(MSG) + + def testRecvmsgIntoArray(self): + # Receive into an array rather than the usual bytearray. + buf = array.array("B", [0] * len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf]) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf.tobytes(), MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoArray(self): + self.sendToServer(MSG) + + def testRecvmsgIntoScatter(self): + # Receive into multiple buffers (scatter write). + b1 = bytearray(b"----") + b2 = bytearray(b"0123456789") + b3 = bytearray(b"--------------") + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + [b1, memoryview(b2)[2:9], b3]) + self.assertEqual(nbytes, len(b"Mary had a little lamb")) + self.assertEqual(b1, bytearray(b"Mary")) + self.assertEqual(b2, bytearray(b"01 had a 9")) + self.assertEqual(b3, bytearray(b"little lamb---")) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoScatter(self): + self.sendToServer(b"Mary had a little lamb") + + +class CmsgMacroTests(unittest.TestCase): + # Test the functions CMSG_LEN() and CMSG_SPACE(). Tests + # assumptions used by sendmsg() and recvmsg[_into](), which share + # code with these functions. + + # Match the definition in socketmodule.c + try: + import _testcapi + except ImportError: + socklen_t_limit = 0x7fffffff + else: + socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) + + @requireAttrs(socket, "CMSG_LEN") + def testCMSG_LEN(self): + # Test CMSG_LEN() with various valid and invalid values, + # checking the assumptions used by recvmsg() and sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_LEN(0) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(socket.CMSG_LEN(0), array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_LEN(n) + # This is how recvmsg() calculates the data size + self.assertEqual(ret - socket.CMSG_LEN(0), n) + self.assertLessEqual(ret, self.socklen_t_limit) + + self.assertRaises(OverflowError, socket.CMSG_LEN, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_LEN, toobig) + self.assertRaises(OverflowError, socket.CMSG_LEN, sys.maxsize) + + @requireAttrs(socket, "CMSG_SPACE") + def testCMSG_SPACE(self): + # Test CMSG_SPACE() with various valid and invalid values, + # checking the assumptions used by sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + last = socket.CMSG_SPACE(0) + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(last, array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_SPACE(n) + self.assertGreaterEqual(ret, last) + self.assertGreaterEqual(ret, socket.CMSG_LEN(n)) + self.assertGreaterEqual(ret, n + socket.CMSG_LEN(0)) + self.assertLessEqual(ret, self.socklen_t_limit) + last = ret + + self.assertRaises(OverflowError, socket.CMSG_SPACE, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_SPACE, toobig) + self.assertRaises(OverflowError, socket.CMSG_SPACE, sys.maxsize) + + +class SCMRightsTest(SendrecvmsgServerTimeoutBase): + # Tests for file descriptor passing on Unix-domain sockets. + + # Invalid file descriptor value that's unlikely to evaluate to a + # real FD even if one of its bytes is replaced with a different + # value (which shouldn't actually happen). + badfd = -0x5555 + + def newFDs(self, n): + # Return a list of n file descriptors for newly-created files + # containing their list indices as ASCII numbers. + fds = [] + for i in range(n): + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + self.addCleanup(os.close, fd) + os.write(fd, str(i).encode()) + fds.append(fd) + return fds + + def checkFDs(self, fds): + # Check that the file descriptors in the given list contain + # their correct list indices as ASCII numbers. + for n, fd in enumerate(fds): + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(os.read(fd, 1024), str(n).encode()) + + def registerRecvmsgResult(self, result): + self.addCleanup(self.closeRecvmsgFDs, result) + + def closeRecvmsgFDs(self, recvmsg_result): + # Close all file descriptors specified in the ancillary data + # of the given return value from recvmsg() or recvmsg_into(). + for cmsg_level, cmsg_type, cmsg_data in recvmsg_result[1]: + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + for fd in fds: + os.close(fd) + + def createAndSendFDs(self, n): + # Send n new file descriptors created by newFDs() to the + # server, with the constant MSG as the non-ancillary data. + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(n)))]), + len(MSG)) + + def checkRecvmsgFDs(self, numfds, result, maxcmsgs=1, ignoreflags=0): + # Check that constant MSG was received with numfds file + # descriptors in a maximum of maxcmsgs control messages (which + # must contain only complete integers). By default, check + # that MSG_CTRUNC is unset, but ignore any flags in + # ignoreflags. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertIsInstance(ancdata, list) + self.assertLessEqual(len(ancdata), maxcmsgs) + fds = array.array("i") + for item in ancdata: + self.assertIsInstance(item, tuple) + cmsg_level, cmsg_type, cmsg_data = item + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data) % SIZEOF_INT, 0) + fds.frombytes(cmsg_data) + + self.assertEqual(len(fds), numfds) + self.checkFDs(fds) + + def testFDPassSimple(self): + # Pass a single FD (array read from bytes object). + self.checkRecvmsgFDs(1, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testFDPassSimple(self): + self.assertEqual( + self.sendmsgToServer( + [MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(1)).tobytes())]), + len(MSG)) + + def testMultipleFDPass(self): + # Pass multiple FDs in a single array. + self.checkRecvmsgFDs(4, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testMultipleFDPass(self): + self.createAndSendFDs(4) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassCMSG_SPACE(self): + # Test using CMSG_SPACE() to calculate ancillary buffer size. + self.checkRecvmsgFDs( + 4, self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(4 * SIZEOF_INT))) + + @testFDPassCMSG_SPACE.client_skip + def _testFDPassCMSG_SPACE(self): + self.createAndSendFDs(4) + + def testFDPassCMSG_LEN(self): + # Test using CMSG_LEN() to calculate ancillary buffer size. + self.checkRecvmsgFDs(1, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(4 * SIZEOF_INT)), + # RFC 3542 says implementations may set + # MSG_CTRUNC if there isn't enough space + # for trailing padding. + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassCMSG_LEN(self): + self.createAndSendFDs(1) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparate(self): + # Pass two FDs in two separate arrays. Arrays may be combined + # into a single control message by the OS. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), 10240), + maxcmsgs=2) + + @testFDPassSeparate.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparate(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparateMinSpace(self): + # Pass two FDs in two separate arrays, receiving them into the + # minimum space for two arrays. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(SIZEOF_INT)), + maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) + + @testFDPassSeparateMinSpace.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparateMinSpace(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + def sendAncillaryIfPossible(self, msg, ancdata): + # Try to send msg and ancdata to server, but if the system + # call fails, just send msg with no ancillary data. + try: + nbytes = self.sendmsgToServer([msg], ancdata) + except OSError as e: + # Check that it was the system call that failed + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer([msg]) + self.assertEqual(nbytes, len(msg)) + + def testFDPassEmpty(self): + # Try to pass an empty FD array. Can receive either no array + # or an empty array. + self.checkRecvmsgFDs(0, self.doRecvmsg(self.serv_sock, + len(MSG), 10240), + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassEmpty(self): + self.sendAncillaryIfPossible(MSG, [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + b"")]) + + def testFDPassPartialInt(self): + # Try to pass a truncated FD array. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 1) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + def _testFDPassPartialInt(self): + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [self.badfd]).tobytes()[:-1])]) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassPartialIntInMiddle(self): + # Try to pass two FD arrays, the first of which is truncated. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 2) + fds = array.array("i") + # Arrays may have been combined in a single control message + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.assertLessEqual(len(fds), 2) + self.checkFDs(fds) + + @testFDPassPartialIntInMiddle.client_skip + def _testFDPassPartialIntInMiddle(self): + fd0, fd1 = self.newFDs(2) + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0, self.badfd]).tobytes()[:-1]), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]) + + def checkTruncatedHeader(self, result, ignoreflags=0): + # Check that no ancillary data items are returned when data is + # truncated inside the cmsghdr structure. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no buffer size + # is specified. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG)), + # BSD seems to set MSG_CTRUNC only + # if an item has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTruncNoBufSize(self): + self.createAndSendFDs(1) + + def testCmsgTrunc0(self): + # Check that no ancillary data is received when buffer size is 0. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTrunc0(self): + self.createAndSendFDs(1) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + def testCmsgTrunc1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + + def _testCmsgTrunc1(self): + self.createAndSendFDs(1) + + def testCmsgTrunc2Int(self): + # The cmsghdr structure has at least three members, two of + # which are ints, so we still shouldn't see any ancillary + # data. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + SIZEOF_INT * 2)) + + def _testCmsgTrunc2Int(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Minus1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(0) - 1)) + + def _testCmsgTruncLen0Minus1(self): + self.createAndSendFDs(1) + + # The following tests try to truncate the control message in the + # middle of the FD array. + + def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): + # Check that file descriptor data is truncated to between + # mindata and maxdata bytes when received with buffer size + # ancbuf, and that any complete file descriptor numbers are + # valid. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + if mindata == 0 and ancdata == []: + return + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertGreaterEqual(len(cmsg_data), mindata) + self.assertLessEqual(len(cmsg_data), maxdata) + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.checkFDs(fds) + + def testCmsgTruncLen0(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + + def _testCmsgTruncLen0(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Plus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + + def _testCmsgTruncLen0Plus1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), + maxdata=SIZEOF_INT) + + def _testCmsgTruncLen1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen2Minus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, + maxdata=(2 * SIZEOF_INT) - 1) + + def _testCmsgTruncLen2Minus1(self): + self.createAndSendFDs(2) + + +class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase): + # Test sendmsg() and recvmsg[_into]() using the ancillary data + # features of the RFC 3542 Advanced Sockets API for IPv6. + # Currently we can only handle certain data items (e.g. traffic + # class, hop limit, MTU discovery and fragmentation settings) + # without resorting to unportable means such as the struct module, + # but the tests here are aimed at testing the ancillary data + # handling in sendmsg() and recvmsg() rather than the IPv6 API + # itself. + + # Test value to use when setting hop limit of packet + hop_limit = 2 + + # Test value to use when setting traffic class of packet. + # -1 means "use kernel default". + traffic_class = -1 + + def ancillaryMapping(self, ancdata): + # Given ancillary data list ancdata, return a mapping from + # pairs (cmsg_level, cmsg_type) to corresponding cmsg_data. + # Check that no (level, type) pair appears more than once. + d = {} + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertNotIn((cmsg_level, cmsg_type), d) + d[(cmsg_level, cmsg_type)] = cmsg_data + return d + + def checkHopLimit(self, ancbufsize, maxhop=255, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space. Check that data is MSG, ancillary data is not + # truncated (but ignore any flags in ignoreflags), and hop + # limit is between 0 and maxhop inclusive. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + self.assertIsInstance(ancdata[0], tuple) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimit(self): + # Test receiving the packet hop limit as ancillary data. + self.checkHopLimit(ancbufsize=10240) + + @testRecvHopLimit.client_skip + def _testRecvHopLimit(self): + # Need to wait until server has asked to receive ancillary + # data, as implementations are not required to buffer it + # otherwise. + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimitCMSG_SPACE(self): + # Test receiving hop limit, using CMSG_SPACE to calculate buffer size. + self.checkHopLimit(ancbufsize=socket.CMSG_SPACE(SIZEOF_INT)) + + @testRecvHopLimitCMSG_SPACE.client_skip + def _testRecvHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Could test receiving into buffer sized using CMSG_LEN, but RFC + # 3542 says portable applications must provide space for trailing + # padding. Implementations may set MSG_CTRUNC if there isn't + # enough space for the padding. + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSetHopLimit(self): + # Test setting hop limit on outgoing packet and receiving it + # at the other end. + self.checkHopLimit(ancbufsize=10240, maxhop=self.hop_limit) + + @testSetHopLimit.client_skip + def _testSetHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + def checkTrafficClassAndHopLimit(self, ancbufsize, maxhop=255, + ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space. Check that data is MSG, ancillary + # data is not truncated (but ignore any flags in ignoreflags), + # and traffic class and hop limit are in range (hop limit no + # more than maxhop). + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + self.assertEqual(len(ancdata), 2) + ancmap = self.ancillaryMapping(ancdata) + + tcdata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_TCLASS)] + self.assertEqual(len(tcdata), SIZEOF_INT) + a = array.array("i") + a.frombytes(tcdata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + hldata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT)] + self.assertEqual(len(hldata), SIZEOF_INT) + a = array.array("i") + a.frombytes(hldata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimit(self): + # Test receiving traffic class and hop limit as ancillary data. + self.checkTrafficClassAndHopLimit(ancbufsize=10240) + + @testRecvTrafficClassAndHopLimit.client_skip + def _testRecvTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + # Test receiving traffic class and hop limit, using + # CMSG_SPACE() to calculate buffer size. + self.checkTrafficClassAndHopLimit( + ancbufsize=socket.CMSG_SPACE(SIZEOF_INT) * 2) + + @testRecvTrafficClassAndHopLimitCMSG_SPACE.client_skip + def _testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSetTrafficClassAndHopLimit(self): + # Test setting traffic class and hop limit on outgoing packet, + # and receiving them at the other end. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testSetTrafficClassAndHopLimit.client_skip + def _testSetTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testOddCmsgSize(self): + # Try to send ancillary data with first item one byte too + # long. Fall back to sending with correct size if this fails, + # and check that second item was handled correctly. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testOddCmsgSize.client_skip + def _testOddCmsgSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + try: + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class]).tobytes() + b"\x00"), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + except OSError as e: + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + self.assertEqual(nbytes, len(MSG)) + + # Tests for proper handling of truncated ancillary data + + def checkHopLimitTruncatedHeader(self, ancbufsize, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space, which should be too small to contain the ancillary + # data header (if ancbufsize is None, pass no second argument + # to recvmsg()). Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and no ancillary data is + # returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + args = () if ancbufsize is None else (ancbufsize,) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), *args) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no ancillary + # buffer size is provided. + self.checkHopLimitTruncatedHeader(ancbufsize=None, + # BSD seems to set + # MSG_CTRUNC only if an item + # has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + @testCmsgTruncNoBufSize.client_skip + def _testCmsgTruncNoBufSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc0(self): + # Check that no ancillary data is received when ancillary + # buffer size is zero. + self.checkHopLimitTruncatedHeader(ancbufsize=0, + ignoreflags=socket.MSG_CTRUNC) + + @testSingleCmsgTrunc0.client_skip + def _testSingleCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=1) + + @testSingleCmsgTrunc1.client_skip + def _testSingleCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc2Int(self): + self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) + + @testSingleCmsgTrunc2Int.client_skip + def _testSingleCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncLen0Minus1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) + + @testSingleCmsgTruncLen0Minus1.client_skip + def _testSingleCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncInData(self): + # Test truncation of a control message inside its associated + # data. The message may be returned with its data truncated, + # or not returned at all. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + self.assertLessEqual(len(ancdata), 1) + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + @testSingleCmsgTruncInData.client_skip + def _testSingleCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + def checkTruncatedSecondHeader(self, ancbufsize, ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space, which should be large enough to + # contain the first item, but too small to contain the header + # of the second. Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and only one ancillary + # data item is returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertIn(cmsg_type, {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT}) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + # Try the above test with various buffer sizes. + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc0(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT), + ignoreflags=socket.MSG_CTRUNC) + + @testSecondCmsgTrunc0.client_skip + def _testSecondCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) + + @testSecondCmsgTrunc1.client_skip + def _testSecondCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc2Int(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + 2 * SIZEOF_INT) + + @testSecondCmsgTrunc2Int.client_skip + def _testSecondCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTruncLen0Minus1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(0) - 1) + + @testSecondCmsgTruncLen0Minus1.client_skip + def _testSecondCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecomdCmsgTruncInData(self): + # Test truncation of the second of two control messages inside + # its associated data. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + cmsg_types = {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT} + + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + self.assertEqual(ancdata, []) + + @testSecomdCmsgTruncInData.client_skip + def _testSecomdCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + +# Derive concrete test classes for different socket types. + +class SendrecvmsgUDPTestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDPTest(SendmsgConnectionlessTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDPTest(RecvmsgTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDPTest(RecvmsgIntoTests, SendrecvmsgUDPTestBase): + pass + + +class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDP6TestBase): + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer, ignoring scope ID + self.assertEqual(addr1[:-1], addr2[:-1]) + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDP6Test(SendmsgConnectionlessTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDP6Test(RecvmsgTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDP6Test(RecvmsgIntoTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgRFC3542AncillaryUDP6Test(RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoRFC3542AncillaryUDP6Test(RecvmsgIntoMixin, + RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + + +class SendrecvmsgTCPTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, TCPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgTCPTest(SendmsgStreamTests, SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgTCPTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoTCPTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + + +class SendrecvmsgSCTPStreamTestBase(SendrecvmsgSCTPFlagsBase, + SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, SCTPStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgSCTPStreamTest(SendmsgStreamTests, SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCTPStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + +@requireAttrs(socket.socket, "recvmsg_into") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCTPStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgIntoSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + + +class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, UnixStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUnixStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUnixStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCMRightsStreamTest(SCMRightsTest, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg_into") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCMRightsStreamTest(RecvmsgIntoMixin, SCMRightsTest, + SendrecvmsgUnixStreamTestBase): + pass + + +# Test interrupting the interruptible send/receive methods with a +# signal when a timeout is set. These tests avoid having multiple +# threads alive during the test so that the OS cannot deliver the +# signal to the wrong one. + +class InterruptedTimeoutBase(unittest.TestCase): + # Base class for interrupted send/receive tests. Installs an + # empty handler for SIGALRM and removes it on teardown, along with + # any scheduled alarms. + + def setUp(self): + super().setUp() + orig_alrm_handler = signal.signal(signal.SIGALRM, + lambda signum, frame: 1 / 0) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(self.setAlarm, 0) + + # Timeout for socket operations + timeout = 4.0 + + # Provide setAlarm() method to schedule delivery of SIGALRM after + # given number of seconds, or cancel it if zero, and an + # appropriate time value to use. Use setitimer() if available. + if hasattr(signal, "setitimer"): + alarm_time = 0.05 + + def setAlarm(self, seconds): + signal.setitimer(signal.ITIMER_REAL, seconds) + else: + # Old systems may deliver the alarm up to one second early + alarm_time = 2 + + def setAlarm(self, seconds): + signal.alarm(seconds) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase): + # Test interrupting the recv*() methods with signals when a + # timeout is set. + + def setUp(self): + super().setUp() + self.serv.settimeout(self.timeout) + + def checkInterruptedRecv(self, func, *args, **kwargs): + # Check that func(*args, **kwargs) raises + # errno of EINTR when interrupted by a signal. + self.setAlarm(self.alarm_time) + with self.assertRaises(ZeroDivisionError) as cm: + func(*args, **kwargs) + + def testInterruptedRecvTimeout(self): + self.checkInterruptedRecv(self.serv.recv, 1024) + + def testInterruptedRecvIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recv_into, bytearray(1024)) + + def testInterruptedRecvfromTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom, 1024) + + def testInterruptedRecvfromIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom_into, bytearray(1024)) + + @requireAttrs(socket.socket, "recvmsg") + def testInterruptedRecvmsgTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg, 1024) + + @requireAttrs(socket.socket, "recvmsg_into") + def testInterruptedRecvmsgIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg_into, [bytearray(1024)]) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +@unittest.skipUnless(thread, 'Threading required for this test.') +class InterruptedSendTimeoutTest(InterruptedTimeoutBase, + ThreadSafeCleanupTestCase, + SocketListeningTestMixin, TCPTestBase): + # Test interrupting the interruptible send*() methods with signals + # when a timeout is set. + + def setUp(self): + super().setUp() + self.serv_conn = self.newSocket() + self.addCleanup(self.serv_conn.close) + # Use a thread to complete the connection, but wait for it to + # terminate before running the test, so that there is only one + # thread to accept the signal. + cli_thread = threading.Thread(target=self.doConnect) + cli_thread.start() + self.cli_conn, addr = self.serv.accept() + self.addCleanup(self.cli_conn.close) + cli_thread.join() + self.serv_conn.settimeout(self.timeout) + + def doConnect(self): + self.serv_conn.connect(self.serv_addr) + + def checkInterruptedSend(self, func, *args, **kwargs): + # Check that func(*args, **kwargs), run in a loop, raises + # OSError with an errno of EINTR when interrupted by a + # signal. + with self.assertRaises(ZeroDivisionError) as cm: + while True: + self.setAlarm(self.alarm_time) + func(*args, **kwargs) + + # Issue #12958: The following tests have problems on OS X prior to 10.7 + @support.requires_mac_ver(10, 7) + def testInterruptedSendTimeout(self): + self.checkInterruptedSend(self.serv_conn.send, b"a"*512) + + @support.requires_mac_ver(10, 7) + def testInterruptedSendtoTimeout(self): + # Passing an actual address here as Python's wrapper for + # sendto() doesn't allow passing a zero-length one; POSIX + # requires that the address is ignored since the socket is + # connection-mode, however. + self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512, + self.serv_addr) + + @support.requires_mac_ver(10, 7) + @requireAttrs(socket.socket, "sendmsg") + def testInterruptedSendmsgTimeout(self): + self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512]) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class TCPCloserTest(ThreadedTCPSocketTest): + + def testClose(self): + conn, addr = self.serv.accept() + conn.close() + + sd = self.cli + read, write, err = select.select([sd], [], [], 1.0) + self.assertEqual(read, [sd]) + self.assertEqual(sd.recv(1), b'') + + # Calling close() many times should be safe. + conn.close() + conn.close() + + def _testClose(self): + self.cli.connect((HOST, self.port)) + time.sleep(1.0) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicSocketPairTest(SocketPairTest): + + def __init__(self, methodName='runTest'): + SocketPairTest.__init__(self, methodName=methodName) + + def _check_defaults(self, sock): + self.assertIsInstance(sock, socket.socket) + if hasattr(socket, 'AF_UNIX'): + self.assertEqual(sock.family, socket.AF_UNIX) + else: + self.assertEqual(sock.family, socket.AF_INET) + self.assertEqual(sock.type, socket.SOCK_STREAM) + self.assertEqual(sock.proto, 0) + + def _testDefaults(self): + self._check_defaults(self.cli) + + def testDefaults(self): + self._check_defaults(self.serv) + + def testRecv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.cli.send(MSG) + + def testSend(self): + self.serv.send(MSG) + + def _testSend(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NonBlockingTCPTests(ThreadedTCPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def testSetBlocking(self): + # Testing whether set blocking works + self.serv.setblocking(True) + self.assertIsNone(self.serv.gettimeout()) + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error setting non-blocking mode.") + + def _testSetBlocking(self): + pass + + @support.cpython_only + def testSetBlocking_overflow(self): + # Issue 15989 + import _testcapi + if _testcapi.UINT_MAX >= _testcapi.ULONG_MAX: + self.skipTest('needs UINT_MAX < ULONG_MAX') + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.serv.setblocking(_testcapi.UINT_MAX + 1) + self.assertIsNone(self.serv.gettimeout()) + + _testSetBlocking_overflow = support.cpython_only(_testSetBlocking) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'test needs socket.SOCK_NONBLOCK') + @support.requires_linux_version(2, 6, 28) + def testInitNonBlocking(self): + # reinit server socket + self.serv.close() + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM | + socket.SOCK_NONBLOCK) + self.port = support.bind_port(self.serv) + self.serv.listen() + # actual testing + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error creating with non-blocking mode.") + + def _testInitNonBlocking(self): + pass + + def testInheritFlags(self): + # Issue #7995: when calling accept() on a listening socket with a + # timeout, the resulting socket should not be non-blocking. + self.serv.settimeout(10) + try: + conn, addr = self.serv.accept() + message = conn.recv(len(MSG)) + finally: + conn.close() + self.serv.settimeout(None) + + def _testInheritFlags(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + time.sleep(0.5) + self.cli.send(MSG) + + def testAccept(self): + # Testing non-blocking accept + self.serv.setblocking(0) + try: + conn, addr = self.serv.accept() + except OSError: + pass + else: + self.fail("Error trying to do non-blocking accept.") + read, write, err = select.select([self.serv], [], []) + if self.serv in read: + conn, addr = self.serv.accept() + self.assertIsNone(conn.gettimeout()) + conn.close() + else: + self.fail("Error trying to do accept after select.") + + def _testAccept(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + + def testConnect(self): + # Testing non-blocking connect + conn, addr = self.serv.accept() + conn.close() + + def _testConnect(self): + self.cli.settimeout(10) + self.cli.connect((HOST, self.port)) + + def testRecv(self): + # Testing non-blocking recv + conn, addr = self.serv.accept() + conn.setblocking(0) + try: + msg = conn.recv(len(MSG)) + except OSError: + pass + else: + self.fail("Error trying to do non-blocking recv.") + read, write, err = select.select([conn], [], []) + if conn in read: + msg = conn.recv(len(MSG)) + conn.close() + self.assertEqual(msg, MSG) + else: + self.fail("Error during select call to non-blocking socket.") + + def _testRecv(self): + self.cli.connect((HOST, self.port)) + time.sleep(0.1) + self.cli.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class FileObjectClassTestCase(SocketConnectedTest): + """Unit tests for the object returned by socket.makefile() + + self.read_file is the io object returned by makefile() on + the client connection. You can read from this file to + get output from the server. + + self.write_file is the io object returned by makefile() on the + server connection. You can write to this file to send output + to the client. + """ + + bufsize = -1 # Use default buffer size + encoding = 'utf-8' + errors = 'strict' + newline = None + + read_mode = 'rb' + read_msg = MSG + write_mode = 'wb' + write_msg = MSG + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def setUp(self): + self.evt1, self.evt2, self.serv_finished, self.cli_finished = [ + threading.Event() for i in range(4)] + SocketConnectedTest.setUp(self) + self.read_file = self.cli_conn.makefile( + self.read_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def tearDown(self): + self.serv_finished.set() + self.read_file.close() + self.assertTrue(self.read_file.closed) + self.read_file = None + SocketConnectedTest.tearDown(self) + + def clientSetUp(self): + SocketConnectedTest.clientSetUp(self) + self.write_file = self.serv_conn.makefile( + self.write_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def clientTearDown(self): + self.cli_finished.set() + self.write_file.close() + self.assertTrue(self.write_file.closed) + self.write_file = None + SocketConnectedTest.clientTearDown(self) + + def testReadAfterTimeout(self): + # Issue #7322: A file object must disallow further reads + # after a timeout has occurred. + self.cli_conn.settimeout(1) + self.read_file.read(3) + # First read raises a timeout + self.assertRaises(socket.timeout, self.read_file.read, 1) + # Second read is disallowed + with self.assertRaises(OSError) as ctx: + self.read_file.read(1) + self.assertIn("cannot read from timed out object", str(ctx.exception)) + + def _testReadAfterTimeout(self): + self.write_file.write(self.write_msg[0:3]) + self.write_file.flush() + self.serv_finished.wait() + + def testSmallRead(self): + # Performing small file read test + first_seg = self.read_file.read(len(self.read_msg)-3) + second_seg = self.read_file.read(3) + msg = first_seg + second_seg + self.assertEqual(msg, self.read_msg) + + def _testSmallRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testFullRead(self): + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testFullRead(self): + self.write_file.write(self.write_msg) + self.write_file.close() + + def testUnbufferedRead(self): + # Performing unbuffered file read test + buf = type(self.read_msg)() + while 1: + char = self.read_file.read(1) + if not char: + break + buf += char + self.assertEqual(buf, self.read_msg) + + def _testUnbufferedRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testReadline(self): + # Performing file readline test + line = self.read_file.readline() + self.assertEqual(line, self.read_msg) + + def _testReadline(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testCloseAfterMakefile(self): + # The file returned by makefile should keep the socket open. + self.cli_conn.close() + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testCloseAfterMakefile(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileAfterMakefileClose(self): + self.read_file.close() + msg = self.cli_conn.recv(len(MSG)) + if isinstance(self.read_msg, str): + msg = msg.decode() + self.assertEqual(msg, self.read_msg) + + def _testMakefileAfterMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testClosedAttr(self): + self.assertTrue(not self.read_file.closed) + + def _testClosedAttr(self): + self.assertTrue(not self.write_file.closed) + + def testAttributes(self): + self.assertEqual(self.read_file.mode, self.read_mode) + self.assertEqual(self.read_file.name, self.cli_conn.fileno()) + + def _testAttributes(self): + self.assertEqual(self.write_file.mode, self.write_mode) + self.assertEqual(self.write_file.name, self.serv_conn.fileno()) + + def testRealClose(self): + self.read_file.close() + self.assertRaises(ValueError, self.read_file.fileno) + self.cli_conn.close() + self.assertRaises(OSError, self.cli_conn.getsockname) + + def _testRealClose(self): + pass + + +class UnbufferedFileObjectClassTestCase(FileObjectClassTestCase): + + """Repeat the tests from FileObjectClassTestCase with bufsize==0. + + In this case (and in this case only), it should be possible to + create a file object, read a line from it, create another file + object, read another line from it, without loss of data in the + first file object's buffer. Note that http.client relies on this + when reading multiple requests from the same socket.""" + + bufsize = 0 # Use unbuffered mode + + def testUnbufferedReadline(self): + # Read a line, create a new file object, read another line with it + line = self.read_file.readline() # first line + self.assertEqual(line, b"A. " + self.write_msg) # first line + self.read_file = self.cli_conn.makefile('rb', 0) + line = self.read_file.readline() # second line + self.assertEqual(line, b"B. " + self.write_msg) # second line + + def _testUnbufferedReadline(self): + self.write_file.write(b"A. " + self.write_msg) + self.write_file.write(b"B. " + self.write_msg) + self.write_file.flush() + + def testMakefileClose(self): + # The file returned by makefile should keep the socket open... + self.cli_conn.close() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, self.read_msg) + # ...until the file is itself closed + self.read_file.close() + self.assertRaises(OSError, self.cli_conn.recv, 1024) + + def _testMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileCloseSocketDestroy(self): + refcount_before = sys.getrefcount(self.cli_conn) + self.read_file.close() + refcount_after = sys.getrefcount(self.cli_conn) + self.assertEqual(refcount_before - 1, refcount_after) + + def _testMakefileCloseSocketDestroy(self): + pass + + # Non-blocking ops + # NOTE: to set `read_file` as non-blocking, we must call + # `cli_conn.setblocking` and vice-versa (see setUp / clientSetUp). + + def testSmallReadNonBlocking(self): + self.cli_conn.setblocking(False) + self.assertEqual(self.read_file.readinto(bytearray(10)), None) + self.assertEqual(self.read_file.read(len(self.read_msg) - 3), None) + self.evt1.set() + self.evt2.wait(1.0) + first_seg = self.read_file.read(len(self.read_msg) - 3) + if first_seg is None: + # Data not arrived (can happen under Windows), wait a bit + time.sleep(0.5) + first_seg = self.read_file.read(len(self.read_msg) - 3) + buf = bytearray(10) + n = self.read_file.readinto(buf) + self.assertEqual(n, 3) + msg = first_seg + buf[:n] + self.assertEqual(msg, self.read_msg) + self.assertEqual(self.read_file.readinto(bytearray(16)), None) + self.assertEqual(self.read_file.read(1), None) + + def _testSmallReadNonBlocking(self): + self.evt1.wait(1.0) + self.write_file.write(self.write_msg) + self.write_file.flush() + self.evt2.set() + # Avoid cloding the socket before the server test has finished, + # otherwise system recv() will return 0 instead of EWOULDBLOCK. + self.serv_finished.wait(5.0) + + def testWriteNonBlocking(self): + self.cli_finished.wait(5.0) + # The client thread can't skip directly - the SkipTest exception + # would appear as a failure. + if self.serv_skipped: + self.skipTest(self.serv_skipped) + + def _testWriteNonBlocking(self): + self.serv_skipped = None + self.serv_conn.setblocking(False) + # Try to saturate the socket buffer pipe with repeated large writes. + BIG = b"x" * support.SOCK_MAX_SIZE + LIMIT = 10 + # The first write() succeeds since a chunk of data can be buffered + n = self.write_file.write(BIG) + self.assertGreater(n, 0) + for i in range(LIMIT): + n = self.write_file.write(BIG) + if n is None: + # Succeeded + break + self.assertGreater(n, 0) + else: + # Let us know that this test didn't manage to establish + # the expected conditions. This is not a failure in itself but, + # if it happens repeatedly, the test should be fixed. + self.serv_skipped = "failed to saturate the socket buffer" + + +class LineBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 1 # Default-buffered for reading; line-buffered for writing + + +class SmallBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 2 # Exercise the buffering code + + +class UnicodeReadFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'wb' + write_msg = MSG + newline = '' + + +class UnicodeWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'rb' + read_msg = MSG + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class UnicodeReadWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class NetworkConnectionTest(object): + """Prove network connection.""" + + def clientSetUp(self): + # We're inherited below by BasicTCPTest2, which also inherits + # BasicTCPTest, which defines self.port referenced below. + self.cli = socket.create_connection((HOST, self.port)) + self.serv_conn = self.cli + +class BasicTCPTest2(NetworkConnectionTest, BasicTCPTest): + """Tests that NetworkConnection does not break existing TCP functionality. + """ + +class NetworkConnectionNoServer(unittest.TestCase): + + class MockSocket(socket.socket): + def connect(self, *args): + raise socket.timeout('timed out') + + @contextlib.contextmanager + def mocked_socket_module(self): + """Return a socket which times out on connect""" + old_socket = socket.socket + socket.socket = self.MockSocket + try: + yield + finally: + socket.socket = old_socket + + def test_connect(self): + port = support.find_unused_port() + cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(cli.close) + with self.assertRaises(OSError) as cm: + cli.connect((HOST, port)) + self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + + def test_create_connection(self): + # Issue #9792: errors raised by create_connection() should have + # a proper errno attribute. + port = support.find_unused_port() + with self.assertRaises(OSError) as cm: + socket.create_connection((HOST, port)) + + # Issue #16257: create_connection() calls getaddrinfo() against + # 'localhost'. This may result in an IPV6 addr being returned + # as well as an IPV4 one: + # >>> socket.getaddrinfo('localhost', port, 0, SOCK_STREAM) + # >>> [(2, 2, 0, '', ('127.0.0.1', 41230)), + # (26, 2, 0, '', ('::1', 41230, 0, 0))] + # + # create_connection() enumerates through all the addresses returned + # and if it doesn't successfully bind to any of them, it propagates + # the last exception it encountered. + # + # On Solaris, ENETUNREACH is returned in this circumstance instead + # of ECONNREFUSED. So, if that errno exists, add it to our list of + # expected errnos. + expected_errnos = [ errno.ECONNREFUSED, ] + if hasattr(errno, 'ENETUNREACH'): + expected_errnos.append(errno.ENETUNREACH) + + self.assertIn(cm.exception.errno, expected_errnos) + + def test_create_connection_timeout(self): + # Issue #9792: create_connection() should not recast timeout errors + # as generic socket errors. + with self.mocked_socket_module(): + with self.assertRaises(socket.timeout): + socket.create_connection((HOST, 1234)) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.source_port = support.find_unused_port() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def _justAccept(self): + conn, addr = self.serv.accept() + conn.close() + + testFamily = _justAccept + def _testFamily(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.family, 2) + + testSourceAddress = _justAccept + def _testSourceAddress(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30, + source_address=('', self.source_port)) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.getsockname()[1], self.source_port) + # The port number being used is sufficient to show that the bind() + # call happened. + + testTimeoutDefault = _justAccept + def _testTimeoutDefault(self): + # passing no explicit timeout uses socket's global default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(42) + try: + self.cli = socket.create_connection((HOST, self.port)) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), 42) + + testTimeoutNone = _justAccept + def _testTimeoutNone(self): + # None timeout means the same as sock.settimeout(None) + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + self.cli = socket.create_connection((HOST, self.port), timeout=None) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), None) + + testTimeoutValueNamed = _justAccept + def _testTimeoutValueNamed(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.assertEqual(self.cli.gettimeout(), 30) + + testTimeoutValueNonamed = _justAccept + def _testTimeoutValueNonamed(self): + self.cli = socket.create_connection((HOST, self.port), 30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.gettimeout(), 30) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionBehaviourTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def testInsideTimeout(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + time.sleep(3) + conn.send(b"done!") + testOutsideTimeout = testInsideTimeout + + def _testInsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port)) + data = sock.recv(5) + self.assertEqual(data, b"done!") + + def _testOutsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port), timeout=1) + self.assertRaises(socket.timeout, lambda: sock.recv(5)) + + +class TCPTimeoutTest(SocketTCPTest): + + def testTCPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.accept() + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (TCP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of error (TCP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (TCP)") + if not ok: + self.fail("accept() returned success when we did not expect it") + + @unittest.skipUnless(hasattr(signal, 'alarm'), + 'test needs signal.alarm()') + def testInterruptedTimeout(self): + # XXX I don't know how to do this test on MSWindows or any other + # plaform that doesn't support signal.alarm() or os.kill(), though + # the bug should have existed on all platforms. + self.serv.settimeout(5.0) # must be longer than alarm + class Alarm(Exception): + pass + def alarm_handler(signal, frame): + raise Alarm + old_alarm = signal.signal(signal.SIGALRM, alarm_handler) + try: + signal.alarm(2) # POSIX allows alarm to be up to 1 second early + try: + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of Alarm") + except Alarm: + pass + except: + self.fail("caught other exception instead of Alarm:" + " %s(%s):\n%s" % + (sys.exc_info()[:2] + (traceback.format_exc(),))) + else: + self.fail("nothing caught") + finally: + signal.alarm(0) # shut off alarm + except Alarm: + self.fail("got Alarm in wrong place") + finally: + # no alarm can be pending. Safe to restore old handler. + signal.signal(signal.SIGALRM, old_alarm) + +class UDPTimeoutTest(SocketUDPTest): + + def testUDPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.recv(1024) + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (UDP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.recv(1024) + except socket.timeout: + self.fail("caught timeout instead of error (UDP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (UDP)") + if not ok: + self.fail("recv() returned success when we did not expect it") + +class TestExceptions(unittest.TestCase): + + def testExceptionTree(self): + self.assertTrue(issubclass(OSError, Exception)) + self.assertTrue(issubclass(socket.herror, OSError)) + self.assertTrue(issubclass(socket.gaierror, OSError)) + self.assertTrue(issubclass(socket.timeout, OSError)) + +@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +class TestLinuxAbstractNamespace(unittest.TestCase): + + UNIX_PATH_MAX = 108 + + def testLinuxAbstractNamespace(self): + address = b"\x00python-test-hello\x00\xff" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s1: + s1.bind(address) + s1.listen() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s2: + s2.connect(s1.getsockname()) + with s1.accept()[0] as s3: + self.assertEqual(s1.getsockname(), address) + self.assertEqual(s2.getpeername(), address) + + def testMaxName(self): + address = b"\x00" + b"h" * (self.UNIX_PATH_MAX - 1) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(address) + self.assertEqual(s.getsockname(), address) + + def testNameOverflow(self): + address = "\x00" + "h" * self.UNIX_PATH_MAX + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + self.assertRaises(OSError, s.bind, address) + + def testStrName(self): + # Check that an abstract name can be passed as a string. + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s.bind("\x00python\x00test\x00") + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + finally: + s.close() + + def testBytearrayName(self): + # Check that an abstract name can be passed as a bytearray. + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(bytearray(b"\x00python\x00test\x00")) + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + +@unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'test needs socket.AF_UNIX') +class TestUnixDomain(unittest.TestCase): + + def setUp(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + def tearDown(self): + self.sock.close() + + def encoded(self, path): + # Return the given path encoded in the file system encoding, + # or skip the test if this is not possible. + try: + return os.fsencode(path) + except UnicodeEncodeError: + self.skipTest( + "Pathname {0!a} cannot be represented in file " + "system encoding {1!r}".format( + path, sys.getfilesystemencoding())) + + def bind(self, sock, path): + # Bind the socket + try: + sock.bind(path) + except OSError as e: + if str(e) == "AF_UNIX path too long": + self.skipTest( + "Pathname {0!a} is too long to serve as an AF_UNIX path" + .format(path)) + else: + raise + + def testStrAddr(self): + # Test binding to and retrieving a normal string pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testBytesAddr(self): + # Test binding to a bytes pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, self.encoded(path)) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testSurrogateescapeBind(self): + # Test binding to a valid non-ASCII pathname, with the + # non-ASCII bytes supplied using surrogateescape encoding. + path = os.path.abspath(support.TESTFN_UNICODE) + b = self.encoded(path) + self.bind(self.sock, b.decode("ascii", "surrogateescape")) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testUnencodableAddr(self): + # Test binding to a pathname that cannot be encoded in the + # file system encoding. + if support.TESTFN_UNENCODABLE is None: + self.skipTest("No unencodable filename available") + path = os.path.abspath(support.TESTFN_UNENCODABLE) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BufferIOTest(SocketConnectedTest): + """ + Test the buffer versions of socket.recv() and socket.send(). + """ + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecvIntoArray(self): + buf = array.array("B", [0] * len(MSG)) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + buf = buf.tobytes() + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvIntoBytearray(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoBytearray = _testRecvIntoArray + + def testRecvIntoMemoryview(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoMemoryview = _testRecvIntoArray + + def testRecvFromIntoArray(self): + buf = array.array("B", [0] * len(MSG)) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + buf = buf.tobytes() + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvFromIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoBytearray(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoBytearray = _testRecvFromIntoArray + + def testRecvFromIntoMemoryview(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoMemoryview = _testRecvFromIntoArray + + def testRecvFromIntoSmallBuffer(self): + # See issue #20246. + buf = bytearray(8) + self.assertRaises(ValueError, self.cli_conn.recvfrom_into, buf, 1024) + + def _testRecvFromIntoSmallBuffer(self): + self.serv_conn.send(MSG) + + def testRecvFromIntoEmptyBuffer(self): + buf = bytearray() + self.cli_conn.recvfrom_into(buf) + self.cli_conn.recvfrom_into(buf, 0) + + _testRecvFromIntoEmptyBuffer = _testRecvFromIntoArray + + +TIPC_STYPE = 2000 +TIPC_LOWER = 200 +TIPC_UPPER = 210 + +def isTipcAvailable(): + """Check if the TIPC module is loaded + + The TIPC module is not loaded automatically on Ubuntu and probably + other Linux distros. + """ + if not hasattr(socket, "AF_TIPC"): + return False + try: + f = open("/proc/modules") + except IOError as e: + # It's ok if the file does not exist, is a directory or if we + # have not the permission to read it. In any other case it's a + # real error, so raise it again. + if e.errno in (errno.ENOENT, errno.EISDIR, errno.EACCES): + return False + else: + raise + with f: + for line in f: + if line.startswith("tipc "): + return True + return False + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCTest(unittest.TestCase): + def testRDM(self): + srv = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + cli = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + self.addCleanup(srv.close) + self.addCleanup(cli.close) + + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + srv.bind(srvaddr) + + sendaddr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + cli.sendto(MSG, sendaddr) + + msg, recvaddr = srv.recvfrom(1024) + + self.assertEqual(cli.getsockname(), recvaddr) + self.assertEqual(msg, MSG) + + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCThreadableTest(unittest.TestCase, ThreadableTest): + def __init__(self, methodName = 'runTest'): + unittest.TestCase.__init__(self, methodName = methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.srv = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.srv.close) + self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + self.srv.bind(srvaddr) + self.srv.listen() + self.serverExplicitReady() + self.conn, self.connaddr = self.srv.accept() + self.addCleanup(self.conn.close) + + def clientSetUp(self): + # There is a hittable race between serverExplicitReady() and the + # accept() call; sleep a little while to avoid it, otherwise + # we could get an exception + time.sleep(0.1) + self.cli = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.cli.close) + addr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + self.cli.connect(addr) + self.cliaddr = self.cli.getsockname() + + def testStream(self): + msg = self.conn.recv(1024) + self.assertEqual(msg, MSG) + self.assertEqual(self.cliaddr, self.connaddr) + + def _testStream(self): + self.cli.send(MSG) + self.cli.close() + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class ContextManagersTest(ThreadedTCPSocketTest): + + def _testSocketClass(self): + # base test + with socket.socket() as sock: + self.assertFalse(sock._closed) + self.assertTrue(sock._closed) + # close inside with block + with socket.socket() as sock: + sock.close() + self.assertTrue(sock._closed) + # exception inside with block + with socket.socket() as sock: + self.assertRaises(OSError, sock.sendall, b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionBase(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionBase(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + self.assertFalse(sock._closed) + sock.sendall(b'foo') + self.assertEqual(sock.recv(1024), b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionClose(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionClose(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + sock.close() + self.assertTrue(sock._closed) + self.assertRaises(OSError, sock.sendall, b'foo') + + +class InheritanceTest(unittest.TestCase): + @unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), + "SOCK_CLOEXEC not defined") + @support.requires_linux_version(2, 6, 28) + def test_SOCK_CLOEXEC(self): + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: + self.assertTrue(s.type & socket.SOCK_CLOEXEC) + self.assertFalse(s.get_inheritable()) + + def test_default_inheritable(self): + sock = socket.socket() + with sock: + self.assertEqual(sock.get_inheritable(), False) + + def test_dup(self): + sock = socket.socket() + with sock: + newsock = sock.dup() + sock.close() + with newsock: + self.assertEqual(newsock.get_inheritable(), False) + + def test_set_inheritable(self): + sock = socket.socket() + with sock: + sock.set_inheritable(True) + self.assertEqual(sock.get_inheritable(), True) + + sock.set_inheritable(False) + self.assertEqual(sock.get_inheritable(), False) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + + @unittest.skipUnless(hasattr(socket, "socketpair"), + "need socket.socketpair()") + def test_socketpair(self): + s1, s2 = socket.socketpair() + self.addCleanup(s1.close) + self.addCleanup(s2.close) + self.assertEqual(s1.get_inheritable(), False) + self.assertEqual(s2.get_inheritable(), False) + + +@unittest.skipUnless(hasattr(socket, "SOCK_NONBLOCK"), + "SOCK_NONBLOCK not defined") +class NonblockConstantTest(unittest.TestCase): + def checkNonblock(self, s, nonblock=True, timeout=0.0): + if nonblock: + self.assertTrue(s.type & socket.SOCK_NONBLOCK) + self.assertEqual(s.gettimeout(), timeout) + else: + self.assertFalse(s.type & socket.SOCK_NONBLOCK) + self.assertEqual(s.gettimeout(), None) + + @support.requires_linux_version(2, 6, 28) + def test_SOCK_NONBLOCK(self): + # a lot of it seems silly and redundant, but I wanted to test that + # changing back and forth worked ok + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) as s: + self.checkNonblock(s) + s.setblocking(1) + self.checkNonblock(s, False) + s.setblocking(0) + self.checkNonblock(s) + s.settimeout(None) + self.checkNonblock(s, False) + s.settimeout(2.0) + self.checkNonblock(s, timeout=2.0) + s.setblocking(1) + self.checkNonblock(s, False) + # defaulttimeout + t = socket.getdefaulttimeout() + socket.setdefaulttimeout(0.0) + with socket.socket() as s: + self.checkNonblock(s) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(2.0) + with socket.socket() as s: + self.checkNonblock(s, timeout=2.0) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(t) + + +@unittest.skipUnless(os.name == "nt", "Windows specific") +@unittest.skipUnless(multiprocessing, "need multiprocessing") +class TestSocketSharing(SocketTCPTest): + # This must be classmethod and not staticmethod or multiprocessing + # won't be able to bootstrap it. + @classmethod + def remoteProcessServer(cls, q): + # Recreate socket from shared data + sdata = q.get() + message = q.get() + + s = socket.fromshare(sdata) + s2, c = s.accept() + + # Send the message + s2.sendall(message) + s2.close() + s.close() + + def testShare(self): + # Transfer the listening server socket to another process + # and service it from there. + + # Create process: + q = multiprocessing.Queue() + p = multiprocessing.Process(target=self.remoteProcessServer, args=(q,)) + p.start() + + # Get the shared socket data + data = self.serv.share(p.pid) + + # Pass the shared socket to the other process + addr = self.serv.getsockname() + self.serv.close() + q.put(data) + + # The data that the server will send us + message = b"slapmahfro" + q.put(message) + + # Connect + s = socket.create_connection(addr) + # listen for the data + m = [] + while True: + data = s.recv(100) + if not data: + break + m.append(data) + s.close() + received = b"".join(m) + self.assertEqual(received, message) + p.join() + + def testShareLength(self): + data = self.serv.share(os.getpid()) + self.assertRaises(ValueError, socket.fromshare, data[:-1]) + self.assertRaises(ValueError, socket.fromshare, data+b"foo") + + def compareSockets(self, org, other): + # socket sharing is expected to work only for blocking socket + # since the internal python timeout value isn't transferred. + self.assertEqual(org.gettimeout(), None) + self.assertEqual(org.gettimeout(), other.gettimeout()) + + self.assertEqual(org.family, other.family) + self.assertEqual(org.type, other.type) + # If the user specified "0" for proto, then + # internally windows will have picked the correct value. + # Python introspection on the socket however will still return + # 0. For the shared socket, the python value is recreated + # from the actual value, so it may not compare correctly. + if org.proto != 0: + self.assertEqual(org.proto, other.proto) + + def testShareLocal(self): + data = self.serv.share(os.getpid()) + s = socket.fromshare(data) + try: + self.compareSockets(self.serv, s) + finally: + s.close() + + def testTypes(self): + families = [socket.AF_INET, socket.AF_INET6] + types = [socket.SOCK_STREAM, socket.SOCK_DGRAM] + for f in families: + for t in types: + try: + source = socket.socket(f, t) + except OSError: + continue # This combination is not supported + try: + data = source.share(os.getpid()) + shared = socket.fromshare(data) + try: + self.compareSockets(source, shared) + finally: + shared.close() + finally: + source.close() + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendfileUsingSendTest(ThreadedTCPSocketTest): + """ + Test the send() implementation of socket.sendfile(). + """ + + FILESIZE = (10 * 1024 * 1024) # 10MB + BUFSIZE = 8192 + FILEDATA = b"" + TIMEOUT = 2 + + @classmethod + def setUpClass(cls): + def chunks(total, step): + assert total >= step + while total > step: + yield step + total -= step + if total: + yield total + + chunk = b"".join([random.choice(string.ascii_letters).encode() + for i in range(cls.BUFSIZE)]) + with open(support.TESTFN, 'wb') as f: + for csize in chunks(cls.FILESIZE, cls.BUFSIZE): + f.write(chunk) + with open(support.TESTFN, 'rb') as f: + cls.FILEDATA = f.read() + assert len(cls.FILEDATA) == cls.FILESIZE + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + + def accept_conn(self): + self.serv.settimeout(self.TIMEOUT) + conn, addr = self.serv.accept() + conn.settimeout(self.TIMEOUT) + self.addCleanup(conn.close) + return conn + + def recv_data(self, conn): + received = [] + while True: + chunk = conn.recv(self.BUFSIZE) + if not chunk: + break + received.append(chunk) + return b''.join(received) + + def meth_from_sock(self, sock): + # Depending on the mixin class being run return either send() + # or sendfile() method implementation. + return getattr(sock, "_sendfile_use_send") + + # regular file + + def _testRegularFile(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, self.FILESIZE) + self.assertEqual(file.tell(), self.FILESIZE) + + def testRegularFile(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # non regular file + + def _testNonRegularFile(self): + address = self.serv.getsockname() + file = io.BytesIO(self.FILEDATA) + with socket.create_connection(address) as sock, file as file: + sent = sock.sendfile(file) + self.assertEqual(sent, self.FILESIZE) + self.assertEqual(file.tell(), self.FILESIZE) + self.assertRaises(socket._GiveupOnSendfile, + sock._sendfile_use_sendfile, file) + + def testNonRegularFile(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # empty file + + def _testEmptyFileSend(self): + address = self.serv.getsockname() + filename = support.TESTFN + "2" + with open(filename, 'wb'): + self.addCleanup(support.unlink, filename) + file = open(filename, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, 0) + self.assertEqual(file.tell(), 0) + + def testEmptyFileSend(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(data, b"") + + # offset + + def _testOffset(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file, offset=5000) + self.assertEqual(sent, self.FILESIZE - 5000) + self.assertEqual(file.tell(), self.FILESIZE) + + def testOffset(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE - 5000) + self.assertEqual(data, self.FILEDATA[5000:]) + + # count + + def _testCount(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 5000007 + meth = self.meth_from_sock(sock) + sent = meth(file, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count) + + def testCount(self): + count = 5000007 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[:count]) + + # count small + + def _testCountSmall(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 1 + meth = self.meth_from_sock(sock) + sent = meth(file, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count) + + def testCountSmall(self): + count = 1 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[:count]) + + # count + offset + + def _testCountWithOffset(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 100007 + meth = self.meth_from_sock(sock) + sent = meth(file, offset=2007, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count + 2007) + + def testCountWithOffset(self): + count = 100007 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[2007:count+2007]) + + # non blocking sockets are not supposed to work + + def _testNonBlocking(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + sock.setblocking(False) + meth = self.meth_from_sock(sock) + self.assertRaises(ValueError, meth, file) + self.assertRaises(ValueError, sock.sendfile, file) + + def testNonBlocking(self): + conn = self.accept_conn() + if conn.recv(8192): + self.fail('was not supposed to receive any data') + + # timeout (non-triggered) + + def _testWithTimeout(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, self.FILESIZE) + + def testWithTimeout(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # timeout (triggered) + + def _testWithTimeoutTriggeredSend(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=0.01) as sock, \ + file as file: + meth = self.meth_from_sock(sock) + self.assertRaises(socket.timeout, meth, file) + + def testWithTimeoutTriggeredSend(self): + conn = self.accept_conn() + conn.recv(88192) + + # errors + + def _test_errors(self): + pass + + def test_errors(self): + with open(support.TESTFN, 'rb') as file: + with socket.socket(type=socket.SOCK_DGRAM) as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex( + ValueError, "SOCK_STREAM", meth, file) + with open(support.TESTFN, 'rt') as file: + with socket.socket() as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex( + ValueError, "binary mode", meth, file) + with open(support.TESTFN, 'rb') as file: + with socket.socket() as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex(TypeError, "positive integer", + meth, file, count='2') + self.assertRaisesRegex(TypeError, "positive integer", + meth, file, count=0.1) + self.assertRaisesRegex(ValueError, "positive integer", + meth, file, count=0) + self.assertRaisesRegex(ValueError, "positive integer", + meth, file, count=-1) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +@unittest.skipUnless(hasattr(os, "sendfile"), + 'os.sendfile() required for this test.') +@unittest.skipUnless(hasattr(os, 'gevent_uses_sendfile'), + 'gevent sockets do not support this') +class SendfileUsingSendfileTest(SendfileUsingSendTest): + """ + Test the sendfile() implementation of socket.sendfile(). + """ + def meth_from_sock(self, sock): + return getattr(sock, "_sendfile_use_sendfile") + + +def test_main(): + tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest, + TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ] + + tests.extend([ + NonBlockingTCPTests, + FileObjectClassTestCase, + UnbufferedFileObjectClassTestCase, + LineBufferedFileObjectClassTestCase, + SmallBufferedFileObjectClassTestCase, + UnicodeReadFileObjectClassTestCase, + UnicodeWriteFileObjectClassTestCase, + UnicodeReadWriteFileObjectClassTestCase, + NetworkConnectionNoServer, + NetworkConnectionAttributesTest, + NetworkConnectionBehaviourTest, + ContextManagersTest, + InheritanceTest, + NonblockConstantTest + ]) + tests.append(BasicSocketPairTest) + tests.append(TestUnixDomain) + tests.append(TestLinuxAbstractNamespace) + tests.extend([TIPCTest, TIPCThreadableTest]) + tests.extend([BasicCANTest, CANTest]) + tests.extend([BasicRDSTest, RDSTest]) + tests.extend([ + CmsgMacroTests, + SendmsgUDPTest, + RecvmsgUDPTest, + RecvmsgIntoUDPTest, + SendmsgUDP6Test, + RecvmsgUDP6Test, + RecvmsgRFC3542AncillaryUDP6Test, + RecvmsgIntoRFC3542AncillaryUDP6Test, + RecvmsgIntoUDP6Test, + SendmsgTCPTest, + RecvmsgTCPTest, + RecvmsgIntoTCPTest, + SendmsgSCTPStreamTest, + RecvmsgSCTPStreamTest, + RecvmsgIntoSCTPStreamTest, + SendmsgUnixStreamTest, + RecvmsgUnixStreamTest, + RecvmsgIntoUnixStreamTest, + RecvmsgSCMRightsStreamTest, + RecvmsgIntoSCMRightsStreamTest, + # These are slow when setitimer() is not available + InterruptedRecvTimeoutTest, + InterruptedSendTimeoutTest, + TestSocketSharing, + SendfileUsingSendTest, + SendfileUsingSendfileTest, + ]) + + thread_info = support.threading_setup() + support.run_unittest(*tests) + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.5/test_ssl.py b/src/greentest/3.5/test_ssl.py new file mode 100644 index 0000000..5515583 --- /dev/null +++ b/src/greentest/3.5/test_ssl.py @@ -0,0 +1,3450 @@ +# Test the support for SSL and sockets + +import sys +import unittest +from test import support +import socket +import select +import time +import datetime +import gc +import os +import errno +import pprint +import tempfile +import urllib.request +import traceback +import asyncore +import weakref +import platform +import functools + +ssl = support.import_module("ssl") + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = support.HOST +IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL') +IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0) + + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/make_ssl_certs.py. +# Other certificates are simply fetched from the Internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = os.fsencode(CERTFILE) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = os.fsencode(ONLYCERT) +BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = os.fsencode(CAPATH) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") + + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNING_CA = data_file("pycacert.pem") +# cert with all kinds of subject alt names +ALLSANFILE = data_file("allsans.pem") + +REMOTE_HOST = "self-signed.pythontest.net" +REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +NONEXISTINGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") + +DHFILE = data_file("dh1024.pem") +BYTES_DHFILE = os.fsencode(DHFILE) + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + +def can_clear_options(): + # 0.9.8m or higher + return ssl._OPENSSL_API_VERSION >= (0, 9, 8, 13, 15) + +def no_sslv2_implies_sslv3_hello(): + # 0.9.7h or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) + +def have_verify_flags(): + # 0.9.8 or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 8, 0, 15) + +def utc_offset(): #NOTE: ignore issues like #1647654 + # local time = utc time + utc offset + if time.daylight and time.localtime().tm_isdst > 0: + return -time.altzone # seconds + return -time.timezone + +def asn1time(cert_time): + # Some versions of OpenSSL ignore seconds, see #18207 + # 0.9.8.i + if ssl._OPENSSL_API_VERSION == (0, 9, 8, 9, 15): + fmt = "%b %d %H:%M:%S %Y GMT" + dt = datetime.datetime.strptime(cert_time, fmt) + dt = dt.replace(second=0) + cert_time = dt.strftime(fmt) + # %d adds leading zero but ASN1_TIME_print() uses leading space + if cert_time[4] == "0": + cert_time = cert_time[:4] + " " + cert_time[5:] + + return cert_time + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): + if hasattr(ssl, 'PROTOCOL_SSLv2'): + @functools.wraps(func) + def f(*args, **kwargs): + try: + ssl.SSLContext(ssl.PROTOCOL_SSLv2) + except ssl.SSLError: + if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and + platform.linux_distribution() == ('debian', 'squeeze/sid', '')): + raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") + return func(*args, **kwargs) + return f + else: + return func + +needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + if ssl.HAS_ECDH: + ssl.OP_SINGLE_ECDH_USE + if ssl.OPENSSL_VERSION_INFO >= (1, 0): + ssl.OP_NO_COMPRESSION + self.assertIn(ssl.HAS_SNI, {True, False}) + self.assertIn(ssl.HAS_ECDH, {True, False}) + + def test_str_for_enums(self): + # Make sure that the PROTOCOL_* constants have enum-like string + # reprs. + proto = ssl.PROTOCOL_TLS + self.assertEqual(str(proto), '_SSLMethod.PROTOCOL_TLS') + ctx = ssl.SSLContext(proto) + self.assertIs(ctx.protocol, proto) + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + + data, is_cryptographic = ssl.RAND_pseudo_bytes(16) + self.assertEqual(len(data), 16) + self.assertEqual(is_cryptographic, v == 1) + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + + # negative num is invalid + self.assertRaises(ValueError, ssl.RAND_bytes, -5) + self.assertRaises(ValueError, ssl.RAND_pseudo_bytes, -5) + + if hasattr(ssl, 'RAND_egd'): + self.assertRaises(TypeError, ssl.RAND_egd, 1) + self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1) + ssl.RAND_add("this is a random string", 75.0) + ssl.RAND_add(b"this is a random bytes object", 75.0) + ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) + + @unittest.skipUnless(os.name == 'posix', 'requires posix') + def test_random_fork(self): + status = ssl.RAND_status() + if not status: + self.fail("OpenSSL's PRNG has insufficient randomness") + + rfd, wfd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(rfd) + child_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(child_random), 16) + os.write(wfd, child_random) + os.close(wfd) + except BaseException: + os._exit(1) + else: + os._exit(0) + else: + os.close(wfd) + self.addCleanup(os.close, rfd) + _, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + child_random = os.read(rfd, 16) + self.assertEqual(len(child_random), 16) + parent_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(parent_random), 16) + + self.assertNotEqual(child_random, parent_random) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + p = ssl._ssl._test_decode_cert(CERTFILE) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['issuer'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + # Note the next three asserts will fail if the keys are regenerated + self.assertEqual(p['notAfter'], asn1time('Oct 5 23:01:56 2020 GMT')) + self.assertEqual(p['notBefore'], asn1time('Oct 8 23:01:56 2010 GMT')) + self.assertEqual(p['serialNumber'], 'D7C7381919AFC24E') + self.assertEqual(p['subject'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + + def test_parse_all_sans(self): + p = ssl._ssl._test_decode_cert(ALLSANFILE) + self.assertEqual(p['subjectAltName'], + ( + ('DNS', 'allsans'), + ('othername', ''), + ('othername', ''), + ('email', 'user@example.org'), + ('DNS', 'www.example.org'), + ('DirName', + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'dirname example'),))), + ('URI', 'https://www.python.org/'), + ('IP Address', '127.0.0.1'), + ('IP Address', '0:0:0:0:0:0:0:1\n'), + ('Registered ID', '1.2.3.4.5') + ) + ) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, int) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 0.9 + self.assertGreaterEqual(n, 0x900000) + # < 3.0 + self.assertLess(n, 0x30000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 0) + self.assertLess(major, 3) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + # Version string as returned by {Open,Libre}SSL, the format might change + if IS_LIBRESSL: + self.assertTrue(s.startswith("LibreSSL {:d}".format(major)), + (s, t, hex(n))) + else: + self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)), + (s, t, hex(n))) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + wr = weakref.ref(ss) + with support.check_warnings(("", ResourceWarning)): + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # OSError raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s) as ss: + self.assertRaises(OSError, ss.recv, 1) + self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) + self.assertRaises(OSError, ss.recvfrom, 1) + self.assertRaises(OSError, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(OSError, ss.send, b'x') + self.assertRaises(OSError, ss.sendto, b'x', ('0.0.0.0', 0)) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with ssl.wrap_socket(s) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + def test_errors(self): + sock = socket.socket() + self.assertRaisesRegex(ValueError, + "certfile must be specified", + ssl.wrap_socket, sock, keyfile=CERTFILE) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True, certfile="") + with ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) as s: + self.assertRaisesRegex(ValueError, "can't connect in server-side mode", + s.connect, (HOST, 8080)) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, + certfile=CERTFILE, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, + certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def bad_cert_test(self, certfile): + """Check that trying to use the given client certificate fails""" + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + certfile) + sock = socket.socket() + self.addCleanup(sock.close) + with self.assertRaises(ssl.SSLError): + ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) + + def test_empty_cert(self): + """Wrapping with an empty cert file""" + self.bad_cert_test("nullcert.pem") + + def test_malformed_cert(self): + """Wrapping with a badly formatted certificate (syntax error)""" + self.bad_cert_test("badcert.pem") + + def test_malformed_key(self): + """Wrapping with a badly formatted key (syntax error)""" + self.bad_cert_test("badkey.pem") + + def test_match_hostname(self): + def ok(cert, hostname): + ssl.match_hostname(cert, hostname) + def fail(cert, hostname): + self.assertRaises(ssl.CertificateError, + ssl.match_hostname, cert, hostname) + + # -- Hostname matching -- + + cert = {'subject': ((('commonName', 'example.com'),),)} + ok(cert, 'example.com') + ok(cert, 'ExAmple.cOm') + fail(cert, 'www.example.com') + fail(cert, '.example.com') + fail(cert, 'example.org') + fail(cert, 'exampleXcom') + + cert = {'subject': ((('commonName', '*.a.com'),),)} + ok(cert, 'foo.a.com') + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + # only match one left-most wildcard + cert = {'subject': ((('commonName', 'f*.com'),),)} + ok(cert, 'foo.com') + ok(cert, 'f.com') + fail(cert, 'bar.com') + fail(cert, 'foo.a.com') + fail(cert, 'bar.foo.com') + + # NULL bytes are bad, CVE-2013-4073 + cert = {'subject': ((('commonName', + 'null.python.org\x00example.org'),),)} + ok(cert, 'null.python.org\x00example.org') # or raise an error? + fail(cert, 'example.org') + fail(cert, 'null.python.org') + + # error cases with wildcards + cert = {'subject': ((('commonName', '*.*.a.com'),),)} + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + cert = {'subject': ((('commonName', 'a.*.com'),),)} + fail(cert, 'a.foo.com') + fail(cert, 'a..com') + fail(cert, 'a.com') + + # wildcard doesn't match IDNA prefix 'xn--' + idna = 'püthon.python.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, idna) + cert = {'subject': ((('commonName', 'x*.python.org'),),)} + fail(cert, idna) + cert = {'subject': ((('commonName', 'xn--p*.python.org'),),)} + fail(cert, idna) + + # wildcard in first fragment and IDNA A-labels in sequent fragments + # are supported. + idna = 'www*.pythön.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, 'www.pythön.org'.encode("idna").decode("ascii")) + ok(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'ftp.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'pythön.org'.encode("idna").decode("ascii")) + + # Slightly fake real-world example + cert = {'notAfter': 'Jun 26 21:41:46 2011 GMT', + 'subject': ((('commonName', 'linuxfrz.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('DNS', 'linuxfr.com'), + ('othername', ''))} + ok(cert, 'linuxfr.org') + ok(cert, 'linuxfr.com') + # Not a "DNS" entry + fail(cert, '') + # When there is a subjectAltName, commonName isn't used + fail(cert, 'linuxfrz.org') + + # A pristine real-world example + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),), + (('commonName', 'mail.google.com'),))} + ok(cert, 'mail.google.com') + fail(cert, 'gmail.com') + # Only commonName is considered + fail(cert, 'California') + + # -- IPv4 matching -- + cert = {'subject': ((('commonName', 'example.com'),),), + 'subjectAltName': (('DNS', 'example.com'), + ('IP Address', '10.11.12.13'), + ('IP Address', '14.15.16.17'))} + ok(cert, '10.11.12.13') + ok(cert, '14.15.16.17') + fail(cert, '14.15.16.18') + fail(cert, 'example.net') + + # -- IPv6 matching -- + cert = {'subject': ((('commonName', 'example.com'),),), + 'subjectAltName': (('DNS', 'example.com'), + ('IP Address', '2001:0:0:0:0:0:0:CAFE\n'), + ('IP Address', '2003:0:0:0:0:0:0:BABA\n'))} + ok(cert, '2001::cafe') + ok(cert, '2003::baba') + fail(cert, '2003::bebe') + fail(cert, 'example.net') + + # -- Miscellaneous -- + + # Neither commonName nor subjectAltName + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),))} + fail(cert, 'mail.google.com') + + # No DNS entry in subjectAltName but a commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('commonName', 'mail.google.com'),)), + 'subjectAltName': (('othername', 'blabla'), )} + ok(cert, 'mail.google.com') + + # No DNS entry subjectAltName and no commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),)), + 'subjectAltName': (('othername', 'blabla'),)} + fail(cert, 'google.com') + + # Empty cert / no cert + self.assertRaises(ValueError, ssl.match_hostname, None, 'example.com') + self.assertRaises(ValueError, ssl.match_hostname, {}, 'example.com') + + # Issue #17980: avoid denials of service by refusing more than one + # wildcard per fragment. + cert = {'subject': ((('commonName', 'a*b.com'),),)} + ok(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b.co*'),),)} + fail(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b*.com'),),)} + with self.assertRaises(ssl.CertificateError) as cm: + ssl.match_hostname(cert, 'axxbxxc.com') + self.assertIn("too many wildcards", str(cm.exception)) + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with socket.socket() as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + s.bind(('127.0.0.1', 0)) + s.listen() + c = socket.socket(socket.AF_INET) + c.connect(s.getsockname()) + with ssl.wrap_socket(c, do_handshake_on_connect=False) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + s.close() + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + def test_dealloc_warn(self): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + r = repr(ss) + with self.assertWarns(ResourceWarning) as cm: + ss = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (set, bool)) + if isinstance(trust, set): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegex(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegex(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + def cert_time_ok(self, timestring, timestamp): + self.assertEqual(ssl.cert_time_to_seconds(timestring), timestamp) + + def cert_time_fail(self, timestring): + with self.assertRaises(ValueError): + ssl.cert_time_to_seconds(timestring) + + @unittest.skipUnless(utc_offset(), + 'local time needs to be different from UTC') + def test_cert_time_to_seconds_timezone(self): + # Issue #19940: ssl.cert_time_to_seconds() returns wrong + # results if local timezone is not UTC + self.cert_time_ok("May 9 00:00:00 2007 GMT", 1178668800.0) + self.cert_time_ok("Jan 5 09:34:43 2018 GMT", 1515144883.0) + + def test_cert_time_to_seconds(self): + timestring = "Jan 5 09:34:43 2018 GMT" + ts = 1515144883.0 + self.cert_time_ok(timestring, ts) + # accept keyword parameter, assert its name + self.assertEqual(ssl.cert_time_to_seconds(cert_time=timestring), ts) + # accept both %e and %d (space or zero generated by strftime) + self.cert_time_ok("Jan 05 09:34:43 2018 GMT", ts) + # case-insensitive + self.cert_time_ok("JaN 5 09:34:43 2018 GmT", ts) + self.cert_time_fail("Jan 5 09:34 2018 GMT") # no seconds + self.cert_time_fail("Jan 5 09:34:43 2018") # no GMT + self.cert_time_fail("Jan 5 09:34:43 2018 UTC") # not GMT timezone + self.cert_time_fail("Jan 35 09:34:43 2018 GMT") # invalid day + self.cert_time_fail("Jon 5 09:34:43 2018 GMT") # invalid month + self.cert_time_fail("Jan 5 24:00:00 2018 GMT") # invalid hour + self.cert_time_fail("Jan 5 09:60:43 2018 GMT") # invalid minute + + newyear_ts = 1230768000.0 + # leap seconds + self.cert_time_ok("Dec 31 23:59:60 2008 GMT", newyear_ts) + # same timestamp + self.cert_time_ok("Jan 1 00:00:00 2009 GMT", newyear_ts) + + self.cert_time_ok("Jan 5 09:34:59 2018 GMT", 1515144899) + # allow 60th second (even if it is not a leap second) + self.cert_time_ok("Jan 5 09:34:60 2018 GMT", 1515144900) + # allow 2nd leap second for compatibility with time.strptime() + self.cert_time_ok("Jan 5 09:34:61 2018 GMT", 1515144901) + self.cert_time_fail("Jan 5 09:34:62 2018 GMT") # invalid seconds + + # no special treatement for the special value: + # 99991231235959Z (rfc 5280) + self.cert_time_ok("Dec 31 23:59:59 9999 GMT", 253402300799.0) + + @support.run_with_locale('LC_ALL', '') + def test_cert_time_to_seconds_locale(self): + # `cert_time_to_seconds()` should be locale independent + + def local_february_name(): + return time.strftime('%b', (1, 2, 3, 4, 5, 6, 0, 0, 0)) + + if local_february_name().lower() == 'feb': + self.skipTest("locale-specific month name needs to be " + "different from C locale") + + # locale-independent + self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) + self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + + +class ContextTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_constructor(self): + for protocol in PROTOCOLS: + ssl.SSLContext(protocol) + ctx = ssl.SSLContext() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + @skip_if_broken_ubuntu_ssl + def test_protocol(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.protocol, proto) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @skip_if_broken_ubuntu_ssl + def test_options(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0): + default |= ssl.OP_NO_COMPRESSION + self.assertEqual(default, ctx.options) + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + if can_clear_options(): + ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) + self.assertEqual(default, ctx.options) + ctx.options = 0 + # Ubuntu has OP_NO_SSLv3 forced on by default + self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + else: + with self.assertRaises(ValueError): + ctx.options = 0 + + def test_verify_mode(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read() + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read() + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegex(ssl.SSLError, "no start line"): + ctx.load_verify_locations(cadata="broken") + with self.assertRaisesRegex(ssl.SSLError, "not enough data"): + ctx.load_verify_locations(cadata=b"broken") + + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_dh_params(DHFILE) + if os.name != 'nt': + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(FileNotFoundError) as cm: + ctx.load_dh_params(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + @skip_if_broken_ubuntu_ssl + def test_session_stats(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + @needs_sni + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @needs_sni + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': asn1time('Mar 29 12:29:49 2033 GMT'), + 'notBefore': asn1time('Mar 30 12:29:49 2003 GMT'), + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + with open(SIGNING_CA) as f: + cadata = f.read() + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_DH_USE", 0), + getattr(ssl, "OP_SINGLE_DH_USE", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + ) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + def test_check_hostname(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertFalse(ctx.check_hostname) + + # Requires CERT_REQUIRED or CERT_OPTIONAL + with self.assertRaises(ValueError): + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + self.assertEqual(cm.exception.library, 'PEM') + self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) + self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen() + c = socket.socket() + c.connect(s.getsockname()) + c.setblocking(False) + with ctx.wrap_socket(c, False, do_handshake_on_connect=False) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertTrue(s.startswith("The operation did not complete (read)"), s) + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + +class MemoryBIOTests(unittest.TestCase): + + def test_read_write(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + self.assertEqual(bio.read(), b'') + bio.write(b'foo') + bio.write(b'bar') + self.assertEqual(bio.read(), b'foobar') + self.assertEqual(bio.read(), b'') + bio.write(b'baz') + self.assertEqual(bio.read(2), b'ba') + self.assertEqual(bio.read(1), b'z') + self.assertEqual(bio.read(1), b'') + + def test_eof(self): + bio = ssl.MemoryBIO() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertFalse(bio.eof) + bio.write(b'foo') + self.assertFalse(bio.eof) + bio.write_eof() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(2), b'fo') + self.assertFalse(bio.eof) + self.assertEqual(bio.read(1), b'o') + self.assertTrue(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertTrue(bio.eof) + + def test_pending(self): + bio = ssl.MemoryBIO() + self.assertEqual(bio.pending, 0) + bio.write(b'foo') + self.assertEqual(bio.pending, 3) + for i in range(3): + bio.read(1) + self.assertEqual(bio.pending, 3-i-1) + for i in range(3): + bio.write(b'x') + self.assertEqual(bio.pending, i+1) + bio.read() + self.assertEqual(bio.pending, 0) + + def test_buffer_types(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + bio.write(bytearray(b'bar')) + self.assertEqual(bio.read(), b'bar') + bio.write(memoryview(b'baz')) + self.assertEqual(bio.read(), b'baz') + + def test_error_types(self): + bio = ssl.MemoryBIO() + self.assertRaises(TypeError, bio.write, 'foo') + self.assertRaises(TypeError, bio.write, None) + self.assertRaises(TypeError, bio.write, True) + self.assertRaises(TypeError, bio.write, 1) + + +class NetworkedTests(unittest.TestCase): + + def test_connect(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) + try: + s.connect((REMOTE_HOST, 443)) + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + + # this should fail because we have no verification certs + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + + # this should succeed because we specify the root cert + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + s.connect((REMOTE_HOST, 443)) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.setblocking(False) + rc = s.connect_ex((REMOTE_HOST, 443)) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + s.close() + + def test_connect_ex_error(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + rc = s.connect_ex((REMOTE_HOST, 444)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + finally: + s.close() + + def test_connect_with_context(self): + with support.transient_internet(REMOTE_HOST): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + # Same with a server hostname + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=REMOTE_HOST) + s.connect((REMOTE_HOST, 443)) + s.close() + # This should fail because we have no verification certs + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + # This should succeed because we specify the root cert + ctx.load_verify_locations(REMOTE_ROOT_CERT) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_cadata(self): + with open(REMOTE_ROOT_CERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + with support.transient_internet(REMOTE_HOST): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss.connect((REMOTE_HOST, 443)) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + with support.transient_internet(REMOTE_HOST): + s = socket.socket(socket.AF_INET) + s.connect((REMOTE_HOST, 443)) + s.setblocking(False) + s = ssl.wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + s.close() + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + def _test_get_server_certificate(host, port, cert=None): + with support.transient_internet(host): + pem = ssl.get_server_certificate((host, port)) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + + try: + pem = ssl.get_server_certificate((host, port), + ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + pem = ssl.get_server_certificate((host, port), + ca_certs=cert) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + + _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) + if support.IPV6_ENABLED: + _test_get_server_certificate('ipv6.google.com', 443) + + def test_ciphers(self): + remote = (REMOTE_HOST, 443) + with support.transient_internet(remote[0]): + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(remote) + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(remote) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(remote) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + # sha256.tbs-internet.com needs SNI to use the correct certificate + if not ssl.HAS_SNI: + self.skipTest("SNI needed for this test") + # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with support.transient_internet("sha256.tbs-internet.com"): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(sha256_cert) + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="sha256.tbs-internet.com") + try: + s.connect(remote) + if support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + self.assertEqual(len(ctx.get_ca_certs()), 1) + + @needs_sni + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + with support.transient_internet(REMOTE_HOST): + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s) as ss: + ss.connect((REMOTE_HOST, 443)) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + + +class NetworkedBIOTests(unittest.TestCase): + + def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): + # A simple IO loop. Call func(*args) depending on the error we get + # (WANT_READ or WANT_WRITE) move data between the socket and the BIOs. + timeout = kwargs.get('timeout', 10) + count = 0 + while True: + errno = None + count += 1 + try: + ret = func(*args) + except ssl.SSLError as e: + if e.errno not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + raise + errno = e.errno + # Get any data from the outgoing BIO irrespective of any error, and + # send it to the socket. + buf = outgoing.read() + sock.sendall(buf) + # If there's no error, we're done. For WANT_READ, we need to get + # data from the socket and put it in the incoming BIO. + if errno is None: + break + elif errno == ssl.SSL_ERROR_WANT_READ: + buf = sock.recv(32768) + if buf: + incoming.write(buf) + else: + incoming.write_eof() + if support.verbose: + sys.stdout.write("Needed %d calls to complete %s().\n" + % (count, func.__name__)) + return ret + + def test_handshake(self): + with support.transient_internet(REMOTE_HOST): + sock = socket.socket(socket.AF_INET) + sock.connect((REMOTE_HOST, 443)) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(REMOTE_ROOT_CERT) + ctx.check_hostname = True + sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST) + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # self-signed.pythontest.net probably shuts down the TCP + # connection without sending a secure shutdown message, and + # this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + sock.close() + + def test_read_write_data(self): + with support.transient_internet(REMOTE_HOST): + sock = socket.socket(socket.AF_INET) + sock.connect((REMOTE_HOST, 443)) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'GET / HTTP/1.0\r\n\r\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf[:5], b'HTTP/') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + sock.close() + + +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + + from test.ssl_servers import make_https_server + + class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(1) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) + self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) + except (ssl.SSLError, ConnectionResetError) as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + self.server.conn_errors.append(e) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.server.stop() + self.close() + return False + else: + self.server.shared_ciphers.append(self.sslconn.shared_ciphers()) + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + sys.stdout.write(" cert binary is " + str(len(cert_binary)) + " bytes\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + sys.stdout.write(" server: selected protocol is now " + + str(self.sslconn.selected_npn_protocol()) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except OSError: + if self.server.chatty: + handle_error("Test server failure:\n") + self.close() + self.running = False + # normally, we'd just stop here, but for the test + # harness, we want to stop the server + self.server.stop() + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + npn_protocols=None, alpn_protocols=None, + ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLSv1) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if npn_protocols: + self.context.set_npn_protocols(npn_protocols) + if alpn_protocols: + self.context.set_alpn_protocols(alpn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = support.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_npn_protocols = [] + self.selected_alpn_protocols = [] + self.shared_ciphers = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.sock.settimeout(0.05) + self.sock.listen() + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except socket.timeout: + pass + except KeyboardInterrupt: + self.stop() + self.sock.close() + + def stop(self): + self.active = False + + class AsyncoreEchoServer(threading.Thread): + + # this one's based on asyncore.dispatcher + + class EchoServer (asyncore.dispatcher): + + class ConnectionHandler (asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = ssl.wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accepted(self, sock_obj, addr): + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + + def start (self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + + def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=sni_name) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_alpn_protocol': s.selected_alpn_protocol(), + 'client_npn_protocol': s.selected_npn_protocol(), + 'version': s.version(), + }) + s.close() + stats['server_alpn_protocols'] = server.selected_alpn_protocols + stats['server_npn_protocols'] = server.selected_npn_protocols + stats['server_shared_ciphers'] = server.shared_ciphers + return stats + + def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_SSLv23: + client_context.set_ciphers("ALL") + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(CERTFILE) + ctx.load_verify_locations(CERTFILE) + try: + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except OSError as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) + + + class ThreadedTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + for protocol in PROTOCOLS: + with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): + context = ssl.SSLContext(protocol) + context.load_cert_chain(CERTFILE) + server_params_test(context, context, + chatty=True, connectionchatty=True) + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + s = context.wrap_socket(socket.socket(), + do_handshake_on_connect=False) + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + s.close() + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(SIGNING_CA) + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + with self.assertRaisesRegex(ssl.SSLError, + "certificate verify failed"): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="localhost") as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: + with self.assertRaisesRegex(ssl.CertificateError, + "hostname 'invalid' doesn't match 'localhost'"): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with socket.socket() as s: + with self.assertRaisesRegex(ValueError, + "check_hostname requires server_hostname"): + context.wrap_socket(s) + + def test_wrong_cert(self): + """Connecting when the server rejects the client's certificate + + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with a wrong client certificate fails. + """ + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + "wrongcert.pem") + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_REQUIRED, + cacerts=CERTFILE, chatty=False, + connectionchatty=False) + with server, \ + socket.socket() as sock, \ + ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) as s: + try: + # Expect either an SSL error about the server rejecting + # the connection, or a low-level connection reset (which + # sometimes happens on Windows) + s.connect((HOST, server.port)) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = support.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen() + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with socket.socket() as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = ssl.wrap_socket(c) + except OSError: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") + def test_protocol_sslv2(self): + """Connecting to an SSLv2 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) + # SSLv23 client with specific SSL options + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv2) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv23(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) + except OSError as x: + # this fails on some older versions of OpenSSL (0.9.7l, for instance) + if support.verbose: + sys.stdout.write( + " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" + % str(x)) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + + # Server with specific SSL options + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), + "OpenSSL is compiled without SSLv3 support") + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, + False, client_options=ssl.OP_NO_SSLv2) + + @skip_if_broken_ubuntu_ssl + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), + "TLS version 1.1 not supported.") + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), + "TLS version 1.2 not supported.") + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(1) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using socketserver to create and manage SSL connections.""" + server = make_https_server(self, certfile=CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + with open(CERTFILE, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = 'https://localhost:%d/%s' % ( + server.port, os.path.split(CERTFILE)[1]) + context = ssl.create_default_context(cafile=CERTFILE) + f = urllib.request.urlopen(url, context=context) + try: + dlen = f.info().get("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = ssl.wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, whether to expect success, *args) + send_methods = [ + ('send', s.send, True, []), + ('sendto', s.sendto, False, ["some.address"]), + ('sendall', s.sendall, True, []), + ] + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = "PREFIX_" + + for meth_name, send_meth, expect_success, args in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + send_meth(indata, *args) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # read(-1, buffer) is supported, even though read(-1) is not + data = b"data" + s.send(data) + buffer = bytearray(len(data)) + self.assertEqual(s.read(-1, buffer), len(data)) + self.assertEqual(buffer, data) + + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, bytearray(100)) + + s.write(b"over\n") + + self.assertRaises(ValueError, s.recv, -1) + self.assertRaises(ValueError, s.read, -1) + + s.close() + + def test_recv_zero(self): + server = ThreadedEchoServer(CERTFILE) + server.__enter__() + self.addCleanup(server.__exit__, None, None) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = ssl.wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + # recv/read(0) should return no data + s.send(b"data") + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.read(0), b"") + self.assertEqual(s.read(), b"data") + + # Should not block if the other end sends no data + s.setblocking(False) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.recv_into(bytearray()), 0) + + def test_nonblocking_send(self): + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + s.setblocking(False) + + # If we keep sending data, at some point the buffers + # will be full and the call will block + buf = bytearray(8192) + def fill_buffer(): + while True: + s.send(buf) + self.assertRaises((ssl.SSLWantWriteError, + ssl.SSLWantReadError), fill_buffer) + + # Now read all the output and discard it + s.setblocking(True) + s.close() + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen() + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + ssl.wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = ssl.wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + server = context.wrap_socket(server, server_side=True) + + evt = threading.Event() + remote = None + peer = None + def serve(): + nonlocal remote, peer + server.listen() + # Block on the accept and wait on the connection to close. + evt.set() + remote, peer = server.accept() + remote.recv(1) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = context.wrap_socket(socket.socket()) + client.connect((host, port)) + client_addr = client.getsockname() + client.close() + t.join() + remote.close() + server.close() + # Sanity checks. + self.assertIsInstance(remote, ssl.SSLSocket) + self.assertEqual(peer, client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_default_ciphers(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + try: + # Force a set of weak ciphers on our client context + context.set_ciphers("DES") + except ssl.SSLError: + self.skipTest("no DES cipher available") + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_SSLv23, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) + self.assertIn("no shared cipher", str(server.conn_errors[0])) + + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + self.assertIs(s.version(), None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1') + self.assertIs(s.version(), None) + + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain(CERTFILE) + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + if ssl.OPENSSL_VERSION_INFO < (1, 0, 0): + context.set_ciphers("ECCdraft:ECDH") + with ThreadedEchoServer(context=context) as server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got channel binding data: {0!r}\n" + .format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + s.close() + + # now, again + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got another channel binding data: {0!r}\n" + .format(new_cb_data)) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + s.close() + + def test_compression(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['compression'], None) + + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.load_dh_params(DHFILE) + context.set_ciphers("kEDH") + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: + self.fail("Non-DH cipher: " + cipher[0]) + + def test_selected_alpn_protocol(self): + # selected_alpn_protocol() is None unless ALPN is used. + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support required") + def test_selected_alpn_protocol_if_server_uses_alpn(self): + # selected_alpn_protocol() is None unless ALPN is used by the client. + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_verify_locations(CERTFILE) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(['foo', 'bar']) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support needed for this test") + def test_alpn_protocols(self): + server_protocols = ['foo', 'bar', 'milkshake'] + protocol_tests = [ + (['foo', 'bar'], 'foo'), + (['bar', 'foo'], 'foo'), + (['milkshake'], 'milkshake'), + (['http/3.0', 'http/4.0'], None) + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + client_context.load_cert_chain(CERTFILE) + client_context.set_alpn_protocols(client_protocols) + + try: + stats = server_params_test(client_context, + server_context, + chatty=True, + connectionchatty=True) + except ssl.SSLError as e: + stats = e + + if expected is None and IS_OPENSSL_1_1: + # OpenSSL 1.1.0 raises handshake error + self.assertIsInstance(stats, ssl.SSLError) + else: + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_alpn_protocol'] + self.assertEqual(client_result, expected, + msg % (client_result, "client")) + server_result = stats['server_alpn_protocols'][-1] \ + if len(stats['server_alpn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, + msg % (server_result, "server")) + + def test_selected_npn_protocol(self): + # selected_npn_protocol() is None unless NPN is used + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_npn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test") + def test_npn_protocols(self): + server_protocols = ['http/1.1', 'spdy/2'] + protocol_tests = [ + (['http/1.1', 'spdy/2'], 'http/1.1'), + (['spdy/2', 'http/1.1'], 'http/1.1'), + (['spdy/2', 'test'], 'spdy/2'), + (['abc', 'def'], 'abc') + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_npn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_cert_chain(CERTFILE) + client_context.set_npn_protocols(client_protocols) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_npn_protocol'] + self.assertEqual(client_result, expected, msg % (client_result, "client")) + server_result = stats['server_npn_protocols'][-1] \ + if len(stats['server_npn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, msg % (server_result, "server")) + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + @needs_sni + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, 'localhost') + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, 'localhost') + self.assertEqual(calls, []) + + @needs_sni + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @needs_sni + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1/0 + server_context.set_servername_callback(cb_raising) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertIn("ZeroDivisionError", stderr.getvalue()) + + @needs_sni + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertIn("TypeError", stderr.getvalue()) + + def test_shared_ciphers(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): + client_context.set_ciphers("AES128:AES256") + server_context.set_ciphers("AES256") + alg1 = "AES256" + alg2 = "AES-256" + else: + client_context.set_ciphers("AES:3DES") + server_context.set_ciphers("3DES") + alg1 = "3DES" + alg2 = "DES-CBC3" + + stats = server_params_test(client_context, server_context) + ciphers = stats['server_shared_ciphers'][0] + self.assertGreater(len(ciphers), 0) + for name, tls_version, bits in ciphers: + if not alg1 in name.split("-") and alg2 not in name: + self.fail(name) + + def test_read_write_after_close_raises_valuerror(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + + with server: + s = context.wrap_socket(socket.socket()) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + def test_sendfile(self): + TEST_DATA = b"x" * 512 + with open(support.TESTFN, 'wb') as f: + f.write(TEST_DATA) + self.addCleanup(support.unlink, support.TESTFN) + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + with open(support.TESTFN, 'rb') as file: + s.sendfile(file) + self.assertEqual(s.recv(1024), TEST_DATA) + + +def test_main(verbose=False): + if support.verbose: + import warnings + plats = { + 'Linux': platform.linux_distribution, + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', + 'dist\(\) and linux_distribution\(\) ' + 'functions are deprecated .*', + PendingDeprecationWarning, + ) + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + tests = [ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests] + + if support.is_resource_enabled('network'): + tests.append(NetworkedTests) + tests.append(NetworkedBIOTests) + + if _have_threads: + thread_info = support.threading_setup() + if thread_info: + tests.append(ThreadedTests) + + try: + support.run_unittest(*tests) + finally: + if _have_threads: + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.5/test_subprocess.py b/src/greentest/3.5/test_subprocess.py new file mode 100644 index 0000000..98b737c --- /dev/null +++ b/src/greentest/3.5/test_subprocess.py @@ -0,0 +1,2751 @@ +import unittest +from unittest import mock +from test.support import script_helper +from test import support +import subprocess +import sys +import signal +import io +import locale +import os +import errno +import tempfile +import time +import re +import selectors +import sysconfig +import warnings +import select +import shutil +import gc +import textwrap + +try: + import threading +except ImportError: + threading = None + +mswindows = (sys.platform == "win32") + +# +# Depends on the following external programs: Python +# + +if mswindows: + SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), ' + 'os.O_BINARY);') +else: + SETBINARY = '' + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + # Try to minimize the number of children we have so this test + # doesn't crash on some buildbots (Alphas in particular). + support.reap_children() + + def tearDown(self): + for inst in subprocess._active: + inst.wait() + subprocess._cleanup() + self.assertFalse(subprocess._active, "subprocess._active not empty") + + def assertStderrEqual(self, stderr, expected, msg=None): + # In a debug build, stuff like "[6580 refs]" is printed to stderr at + # shutdown time. That frustrates tests trying to check stderr produced + # from a spawned Python process. + actual = support.strip_python_stderr(stderr) + # strip_python_stderr also strips whitespace, so we do too. + expected = expected.strip() + self.assertEqual(actual, expected, msg) + + +class PopenTestException(Exception): + pass + + +class PopenExecuteChildRaises(subprocess.Popen): + """Popen subclass for testing cleanup of subprocess.PIPE filehandles when + _execute_child fails. + """ + def _execute_child(self, *args, **kwargs): + raise PopenTestException("Forced Exception for Test") + + +class ProcessTestCase(BaseTestCase): + + def test_io_buffered_by_default(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + self.assertIsInstance(p.stdin, io.BufferedIOBase) + self.assertIsInstance(p.stdout, io.BufferedIOBase) + self.assertIsInstance(p.stderr, io.BufferedIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_io_unbuffered_works(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0) + try: + self.assertIsInstance(p.stdin, io.RawIOBase) + self.assertIsInstance(p.stdout, io.RawIOBase) + self.assertIsInstance(p.stderr, io.RawIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_call_seq(self): + # call() function with sequence argument + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(rc, 47) + + def test_call_timeout(self): + # call() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.call waits for the + # child. + self.assertRaises(subprocess.TimeoutExpired, subprocess.call, + [sys.executable, "-c", "while True: pass"], + timeout=0.1) + + def test_check_call_zero(self): + # check_call() function with zero return code + rc = subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(0)"]) + self.assertEqual(rc, 0) + + def test_check_call_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(c.exception.returncode, 47) + + def test_check_output(self): + # check_output() function with zero return code + output = subprocess.check_output( + [sys.executable, "-c", "print('BDFL')"]) + self.assertIn(b'BDFL', output) + + def test_check_output_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_output( + [sys.executable, "-c", "import sys; sys.exit(5)"]) + self.assertEqual(c.exception.returncode, 5) + + def test_check_output_stderr(self): + # check_output() function stderr redirected to stdout + output = subprocess.check_output( + [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], + stderr=subprocess.STDOUT) + self.assertIn(b'BDFL', output) + + def test_check_output_stdin_arg(self): + # check_output() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + stdin=tf) + self.assertIn(b'PEAR', output) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + input=b'pear') + self.assertIn(b'PEAR', output) + + def test_check_output_stdout_arg(self): + # check_output() refuses to accept 'stdout' argument + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdout=sys.stdout) + self.fail("Expected ValueError when stdout arg supplied.") + self.assertIn('stdout', c.exception.args[0]) + + def test_check_output_stdin_with_input_arg(self): + # check_output() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdin=tf, input=b'hare') + self.fail("Expected ValueError when stdin and input args supplied.") + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + # check_output() function with timeout arg + with self.assertRaises(subprocess.TimeoutExpired) as c: + output = subprocess.check_output( + [sys.executable, "-c", + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"], + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3) + self.fail("Expected TimeoutExpired.") + self.assertEqual(c.exception.output, b'BDFL') + + def test_call_kwargs(self): + # call() function with keyword args + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + rc = subprocess.call([sys.executable, "-c", + 'import sys, os;' + 'sys.exit(os.getenv("FRUIT")=="banana")'], + env=newenv) + self.assertEqual(rc, 1) + + def test_invalid_args(self): + # Popen() called with invalid arguments should raise TypeError + # but Popen.__del__ should not complain (issue #12085) + with support.captured_stderr() as s: + self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) + argcount = subprocess.Popen.__init__.__code__.co_argcount + too_many_args = [0] * (argcount + 1) + self.assertRaises(TypeError, subprocess.Popen, *too_many_args) + self.assertEqual(s.getvalue(), '') + + def test_stdin_none(self): + # .stdin is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + p.wait() + self.assertEqual(p.stdin, None) + + def test_stdout_none(self): + # .stdout is None when not redirected, and the child's stdout will + # be inherited from the parent. In order to test this we run a + # subprocess in a subprocess: + # this_test + # \-- subprocess created by this test (parent) + # \-- subprocess created by the parent subprocess (child) + # The parent doesn't specify stdout, so the child will use the + # parent's stdout. This test checks that the message printed by the + # child goes to the parent stdout. The parent also checks that the + # child's stdout is None. See #11963. + code = ('import sys; from subprocess import Popen, PIPE;' + 'p = Popen([sys.executable, "-c", "print(\'test_stdout_none\')"],' + ' stdin=PIPE, stderr=PIPE);' + 'p.wait(); assert p.stdout is None;') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test_stdout_none') + + def test_stderr_none(self): + # .stderr is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + p.wait() + self.assertEqual(p.stderr, None) + + def _assert_python(self, pre_args, **kwargs): + # We include sys.exit() to prevent the test runner from hanging + # whenever python is found. + args = pre_args + ["import sys; sys.exit(47)"] + p = subprocess.Popen(args, **kwargs) + p.wait() + self.assertEqual(47, p.returncode) + + def test_executable(self): + # Check that the executable argument works. + # + # On Unix (non-Mac and non-Windows), Python looks at args[0] to + # determine where its standard library is, so we need the directory + # of args[0] to be valid for the Popen() call to Python to succeed. + # See also issue #16170 and issue #7774. + doesnotexist = os.path.join(os.path.dirname(sys.executable), + "doesnotexist") + self._assert_python([doesnotexist, "-c"], executable=sys.executable) + + def test_executable_takes_precedence(self): + # Check that the executable argument takes precedence over args[0]. + # + # Verify first that the call succeeds without the executable arg. + pre_args = [sys.executable, "-c"] + self._assert_python(pre_args) + self.assertRaises(FileNotFoundError, self._assert_python, pre_args, + executable="doesnotexist") + + @unittest.skipIf(mswindows, "executable argument replaces shell") + def test_executable_replaces_shell(self): + # Check that the executable argument replaces the default shell + # when shell=True. + self._assert_python([], executable=sys.executable, shell=True) + + # For use in the test_cwd* tests below. + def _normalize_cwd(self, cwd): + # Normalize an expected cwd (for Tru64 support). + # We can't use os.path.realpath since it doesn't expand Tru64 {memb} + # strings. See bug #1063571. + with support.change_cwd(cwd): + return os.getcwd() + + # For use in the test_cwd* tests below. + def _split_python_path(self): + # Return normalized (python_dir, python_base). + python_path = os.path.realpath(sys.executable) + return os.path.split(python_path) + + # For use in the test_cwd* tests below. + def _assert_cwd(self, expected_cwd, python_arg, **kwargs): + # Invoke Python via Popen, and assert that (1) the call succeeds, + # and that (2) the current working directory of the child process + # matches *expected_cwd*. + p = subprocess.Popen([python_arg, "-c", + "import os, sys; " + "sys.stdout.write(os.getcwd()); " + "sys.exit(47)"], + stdout=subprocess.PIPE, + **kwargs) + self.addCleanup(p.stdout.close) + p.wait() + self.assertEqual(47, p.returncode) + normcase = os.path.normcase + self.assertEqual(normcase(expected_cwd), + normcase(p.stdout.read().decode("utf-8"))) + + def test_cwd(self): + # Check that cwd changes the cwd for the child process. + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_arg(self): + # Check that Popen looks for args[0] relative to cwd if args[0] + # is relative. + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + with support.temp_cwd('test_cwd_with_relative_arg', quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python]) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, rel_python, cwd=python_dir) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_executable(self): + # Check that Popen looks for executable relative to cwd if executable + # is relative (and that executable takes precedence over args[0]). + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + doesntexist = "somethingyoudonthave" + with support.temp_cwd('test_cwd_with_relative_executable', quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python, + cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, doesntexist, executable=rel_python, + cwd=python_dir) + + def test_cwd_with_absolute_arg(self): + # Check that Popen can find the executable when the cwd is wrong + # if args[0] is an absolute path. + python_dir, python_base = self._split_python_path() + abs_python = os.path.join(python_dir, python_base) + rel_python = os.path.join(os.curdir, python_base) + with support.temp_dir() as wrong_dir: + # Before calling with an absolute path, confirm that using a + # relative path fails. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + wrong_dir = self._normalize_cwd(wrong_dir) + self._assert_cwd(wrong_dir, abs_python, cwd=wrong_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + def test_executable_with_cwd(self): + python_dir, python_base = self._split_python_path() + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, "somethingyoudonthave", + executable=sys.executable, cwd=python_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + @unittest.skipIf(sysconfig.is_python_build(), + "need an installed Python. See #7774") + def test_executable_without_cwd(self): + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + self._assert_cwd(os.getcwd(), "somethingyoudonthave", + executable=sys.executable) + + def test_stdin_pipe(self): + # stdin redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.stdin.write(b"pear") + p.stdin.close() + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_filedes(self): + # stdin is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + os.write(d, b"pear") + os.lseek(d, 0, 0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=d) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_fileobj(self): + # stdin is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b"pear") + tf.seek(0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=tf) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdout_pipe(self): + # stdout redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), b"orange") + + def test_stdout_filedes(self): + # stdout is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=d) + p.wait() + os.lseek(d, 0, 0) + self.assertEqual(os.read(d, 1024), b"orange") + + def test_stdout_fileobj(self): + # stdout is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=tf) + p.wait() + tf.seek(0) + self.assertEqual(tf.read(), b"orange") + + def test_stderr_pipe(self): + # stderr redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=subprocess.PIPE) + self.addCleanup(p.stderr.close) + self.assertStderrEqual(p.stderr.read(), b"strawberry") + + def test_stderr_filedes(self): + # stderr is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=d) + p.wait() + os.lseek(d, 0, 0) + self.assertStderrEqual(os.read(d, 1024), b"strawberry") + + def test_stderr_fileobj(self): + # stderr is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"strawberry") + + def test_stderr_redirect_with_no_stdout_redirect(self): + # test stderr=STDOUT while stdout=None (not set) + + # - grandchild prints to stderr + # - child redirects grandchild's stderr to its stdout + # - the parent should get grandchild's stderr in child's stdout + p = subprocess.Popen([sys.executable, "-c", + 'import sys, subprocess;' + 'rc = subprocess.call([sys.executable, "-c",' + ' "import sys;"' + ' "sys.stderr.write(\'42\')"],' + ' stderr=subprocess.STDOUT);' + 'sys.exit(rc)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + #NOTE: stdout should get stderr from grandchild + self.assertStderrEqual(stdout, b'42') + self.assertStderrEqual(stderr, b'') # should be empty + self.assertEqual(p.returncode, 0) + + def test_stdout_stderr_pipe(self): + # capture stdout and stderr to the same pipe + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + self.addCleanup(p.stdout.close) + self.assertStderrEqual(p.stdout.read(), b"appleorange") + + def test_stdout_stderr_file(self): + # capture stdout and stderr to the same open file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=tf, + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"appleorange") + + def test_stdout_filedes_of_stdout(self): + # stdout is set to 1 (#1531862). + # To avoid printing the text on stdout, we do something similar to + # test_stdout_none (see above). The parent subprocess calls the child + # subprocess passing stdout=1, and this test uses stdout=PIPE in + # order to capture and check the output of the parent. See #11963. + code = ('import sys, subprocess; ' + 'rc = subprocess.call([sys.executable, "-c", ' + ' "import os, sys; sys.exit(os.write(sys.stdout.fileno(), ' + 'b\'test with stdout=1\'))"], stdout=1); ' + 'assert rc == 18') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test with stdout=1') + + def test_stdout_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'for i in range(10240):' + 'print("x" * 1024)'], + stdout=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdout, None) + + def test_stderr_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys\n' + 'for i in range(10240):' + 'sys.stderr.write("x" * 1024)'], + stderr=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stderr, None) + + def test_stdin_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdin.read(1)'], + stdin=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdin, None) + + def test_env(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "orange" + with subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange") + + # Windows requires at least the SYSTEMROOT environment variable to start + # Python + @unittest.skipIf(sys.platform == 'win32', + 'cannot test an empty env on Windows') + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') is not None, + 'the python library cannot be loaded ' + 'with an empty environment') + def test_empty_env(self): + with subprocess.Popen([sys.executable, "-c", + 'import os; ' + 'print(list(os.environ.keys()))'], + stdout=subprocess.PIPE, + env={}) as p: + stdout, stderr = p.communicate() + self.assertIn(stdout.strip(), + (b"[]", + # Mac OS X adds __CF_USER_TEXT_ENCODING variable to an empty + # environment + b"['__CF_USER_TEXT_ENCODING']")) + + def test_communicate_stdin(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.communicate(b"pear") + self.assertEqual(p.returncode, 1) + + def test_communicate_stdout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("pineapple")'], + stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, b"pineapple") + self.assertEqual(stderr, None) + + def test_communicate_stderr(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("pineapple")'], + stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple");' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(b"banana") + self.assertEqual(stdout, b"banana") + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate_timeout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stderr.write("pineapple\\n");' + 'time.sleep(1);' + 'sys.stderr.write("pear\\n");' + 'sys.stdout.write(sys.stdin.read())'], + universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, "banana", + timeout=0.3) + # Make sure we can keep waiting for it, and that we get the whole output + # after it completes. + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr.encode(), b"pineapple\npear\n") + + def test_communicate_timeout_large_output(self): + # Test an expiring timeout while the child is outputting lots of data. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));'], + stdout=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, timeout=0.4) + (stdout, _) = p.communicate() + self.assertEqual(len(stdout), 4 * 64 * 1024) + + # Test for the fd leak reported in http://bugs.python.org/issue2791. + def test_communicate_pipe_fd_leak(self): + for stdin_pipe in (False, True): + for stdout_pipe in (False, True): + for stderr_pipe in (False, True): + options = {} + if stdin_pipe: + options['stdin'] = subprocess.PIPE + if stdout_pipe: + options['stdout'] = subprocess.PIPE + if stderr_pipe: + options['stderr'] = subprocess.PIPE + if not options: + continue + p = subprocess.Popen((sys.executable, "-c", "pass"), **options) + p.communicate() + if p.stdin is not None: + self.assertTrue(p.stdin.closed) + if p.stdout is not None: + self.assertTrue(p.stdout.closed) + if p.stderr is not None: + self.assertTrue(p.stderr.closed) + + def test_communicate_returns(self): + # communicate() should return None if no redirection is active + p = subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(47)"]) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertEqual(stderr, None) + + def test_communicate_pipe_buf(self): + # communicate() with writes larger than pipe_buf + # This test will probably deadlock rather than fail, if + # communicate() does not work properly. + x, y = os.pipe() + os.close(x) + os.close(y) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read(47));' + 'sys.stderr.write("x" * %d);' + 'sys.stdout.write(sys.stdin.read())' % + support.PIPE_MAX_SIZE], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + string_to_write = b"a" * support.PIPE_MAX_SIZE + (stdout, stderr) = p.communicate(string_to_write) + self.assertEqual(stdout, string_to_write) + + def test_writes_before_communicate(self): + # stdin.write before communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.stdin.write(b"banana") + (stdout, stderr) = p.communicate(b"split") + self.assertEqual(stdout, b"bananasplit") + self.assertStderrEqual(stderr, b"") + + def test_universal_newlines(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(sys.stdin.readline().encode());' + 'buf.flush();' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(sys.stdin.read().encode());' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + p.stdin.write("line1\n") + p.stdin.flush() + self.assertEqual(p.stdout.readline(), "line1\n") + p.stdin.write("line3\n") + p.stdin.close() + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.readline(), + "line2\n") + self.assertEqual(p.stdout.read(6), + "line3\n") + self.assertEqual(p.stdout.read(), + "line4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate(self): + # universal newlines through communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, + "line2\nline4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate_stdin(self): + # universal newlines through communicate(), with only stdin + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.readline() + assert s == "line1\\n", repr(s) + s = sys.stdin.read() + assert s == "line3\\n", repr(s) + ''')], + stdin=subprocess.PIPE, + universal_newlines=1) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_input_none(self): + # Test communicate(input=None) with universal newlines. + # + # We set stdout to PIPE because, as of this writing, a different + # code path is tested when the number of pipes is zero or one. + p = subprocess.Popen([sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + p.communicate() + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_stdin_stdout_stderr(self): + # universal newlines through communicate(), with stdin, stdout, stderr + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.buffer.readline() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line2\\r") + sys.stderr.buffer.write(b"eline2\\n") + s = sys.stdin.buffer.read() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line4\\n") + sys.stdout.buffer.write(b"line5\\r\\n") + sys.stderr.buffer.write(b"eline6\\r") + sys.stderr.buffer.write(b"eline7\\r\\nz") + ''')], + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) + # Python debug build push something like "[42442 refs]\n" + # to stderr at exit of subprocess. + # Don't use assertStderrEqual because it strips CR and LF from output. + self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + + def test_universal_newlines_communicate_encodings(self): + # Check that universal newlines mode works for various encodings, + # in particular for encodings in the UTF-16 and UTF-32 families. + # See issue #15595. + # + # UTF-16 and UTF-32-BE are sufficient to check both with BOM and + # without, and UTF-16 and UTF-32. + import _bootlocale + for encoding in ['utf-16', 'utf-32-be']: + old_getpreferredencoding = _bootlocale.getpreferredencoding + # Indirectly via io.TextIOWrapper, Popen() defaults to + # locale.getpreferredencoding(False) and earlier in Python 3.2 to + # locale.getpreferredencoding(). + def getpreferredencoding(do_setlocale=True): + return encoding + code = ("import sys; " + r"sys.stdout.buffer.write('1\r\n2\r3\n4'.encode('%s'))" % + encoding) + args = [sys.executable, '-c', code] + try: + _bootlocale.getpreferredencoding = getpreferredencoding + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout, stderr = popen.communicate(input='') + finally: + _bootlocale.getpreferredencoding = old_getpreferredencoding + self.assertEqual(stdout, '1\n2\n3\n4') + + def test_no_leaking(self): + # Make sure we leak no resources + if not mswindows: + max_handles = 1026 # too much for most UNIX systems + else: + max_handles = 2050 # too much for (at least some) Windows setups + handles = [] + tmpdir = tempfile.mkdtemp() + try: + for i in range(max_handles): + try: + tmpfile = os.path.join(tmpdir, support.TESTFN) + handles.append(os.open(tmpfile, os.O_WRONLY|os.O_CREAT)) + except OSError as e: + if e.errno != errno.EMFILE: + raise + break + else: + self.skipTest("failed to reach the file descriptor limit " + "(tried %d)" % max_handles) + # Close a couple of them (should be enough for a subprocess) + for i in range(10): + os.close(handles.pop()) + # Loop creating some subprocesses. If one of them leaks some fds, + # the next loop iteration will fail by reaching the max fd limit. + for i in range(15): + p = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write(sys.stdin.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + data = p.communicate(b"lime")[0] + self.assertEqual(data, b"lime") + finally: + for h in handles: + os.close(h) + shutil.rmtree(tmpdir) + + def test_list2cmdline(self): + self.assertEqual(subprocess.list2cmdline(['a b c', 'd', 'e']), + '"a b c" d e') + self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']), + 'ab\\"c \\ d') + self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']), + 'ab\\"c " \\\\" d') + self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']), + 'a\\\\\\b "de fg" h') + self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']), + 'a\\\\\\"b c d') + self.assertEqual(subprocess.list2cmdline(['a\\\\b c', 'd', 'e']), + '"a\\\\b c" d e') + self.assertEqual(subprocess.list2cmdline(['a\\\\b\\ c', 'd', 'e']), + '"a\\\\b\\ c" d e') + self.assertEqual(subprocess.list2cmdline(['ab', '']), + 'ab ""') + + def test_poll(self): + p = subprocess.Popen([sys.executable, "-c", + "import os; os.read(0, 1)"], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + self.assertIsNone(p.poll()) + os.write(p.stdin.fileno(), b'A') + p.wait() + # Subsequent invocations should just return the returncode + self.assertEqual(p.poll(), 0) + + def test_wait(self): + p = subprocess.Popen([sys.executable, "-c", "pass"]) + self.assertEqual(p.wait(), 0) + # Subsequent invocations should just return the returncode + self.assertEqual(p.wait(), 0) + + def test_wait_timeout(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(0.3)"]) + with self.assertRaises(subprocess.TimeoutExpired) as c: + p.wait(timeout=0.0001) + self.assertIn("0.0001", str(c.exception)) # For coverage of __str__. + # Some heavily loaded buildbots (sparc Debian 3.x) require this much + # time to start. + self.assertEqual(p.wait(timeout=3), 0) + + def test_invalid_bufsize(self): + # an invalid type of the bufsize argument should raise + # TypeError. + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], "orange") + + def test_bufsize_is_none(self): + # bufsize=None should be the same as bufsize=0. + p = subprocess.Popen([sys.executable, "-c", "pass"], None) + self.assertEqual(p.wait(), 0) + # Again with keyword arg + p = subprocess.Popen([sys.executable, "-c", "pass"], bufsize=None) + self.assertEqual(p.wait(), 0) + + def _test_bufsize_equal_one(self, line, expected, universal_newlines): + # subprocess may deadlock with bufsize=1, see issue #21332 + with subprocess.Popen([sys.executable, "-c", "import sys;" + "sys.stdout.write(sys.stdin.readline());" + "sys.stdout.flush()"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + bufsize=1, + universal_newlines=universal_newlines) as p: + p.stdin.write(line) # expect that it flushes the line in text mode + os.close(p.stdin.fileno()) # close it without flushing the buffer + read_line = p.stdout.readline() + try: + p.stdin.close() + except OSError: + pass + p.stdin = None + self.assertEqual(p.returncode, 0) + self.assertEqual(read_line, expected) + + def test_bufsize_equal_one_text_mode(self): + # line is flushed in text mode with bufsize=1. + # we should get the full line in return + line = "line\n" + self._test_bufsize_equal_one(line, line, universal_newlines=True) + + def test_bufsize_equal_one_binary_mode(self): + # line is not flushed in binary mode with bufsize=1. + # we should get empty response + line = b'line' + os.linesep.encode() # assume ascii-based locale + self._test_bufsize_equal_one(line, b'', universal_newlines=False) + + def test_leaking_fds_on_error(self): + # see bug #5179: Popen leaks file descriptors to PIPEs if + # the child fails to execute; this will eventually exhaust + # the maximum number of open fds. 1024 seems a very common + # value for that limit, but Windows has 2048, so we loop + # 1024 times (each call leaked two fds). + for i in range(1024): + with self.assertRaises(OSError) as c: + subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # ignore errors that indicate the command was not found + if c.exception.errno not in (errno.ENOENT, errno.EACCES): + raise c.exception + + @unittest.skipIf(threading is None, "threading required") + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(['nonexisting_i_hope'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + + @unittest.skipIf(threading is None, "threading required") + def test_threadsafe_wait(self): + """Issue21291: Popen.wait() needs to be threadsafe for returncode.""" + proc = subprocess.Popen([sys.executable, '-c', + 'import time; time.sleep(12)']) + self.assertEqual(proc.returncode, None) + results = [] + + def kill_proc_timer_thread(): + results.append(('thread-start-poll-result', proc.poll())) + # terminate it from the thread and wait for the result. + proc.kill() + proc.wait() + results.append(('thread-after-kill-and-wait', proc.returncode)) + # this wait should be a no-op given the above. + proc.wait() + results.append(('thread-after-second-wait', proc.returncode)) + + # This is a timing sensitive test, the failure mode is + # triggered when both the main thread and this thread are in + # the wait() call at once. The delay here is to allow the + # main thread to most likely be blocked in its wait() call. + t = threading.Timer(0.2, kill_proc_timer_thread) + t.start() + + if mswindows: + expected_errorcode = 1 + else: + # Should be -9 because of the proc.kill() from the thread. + expected_errorcode = -9 + + # Wait for the process to finish; the thread should kill it + # long before it finishes on its own. Supplying a timeout + # triggers a different code path for better coverage. + proc.wait(timeout=20) + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in wait from main thread") + + # This should be a no-op with no change in returncode. + proc.wait() + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in second main wait.") + + t.join() + # Ensure that all of the thread results are as expected. + # When a race condition occurs in wait(), the returncode could + # be set by the wrong thread that doesn't actually have it + # leading to an incorrect value. + self.assertEqual([('thread-start-poll-result', None), + ('thread-after-kill-and-wait', expected_errorcode), + ('thread-after-second-wait', expected_errorcode)], + results) + + def test_issue8780(self): + # Ensure that stdout is inherited from the parent + # if stdout=PIPE is not used + code = ';'.join(( + 'import subprocess, sys', + 'retcode = subprocess.call(' + "[sys.executable, '-c', 'print(\"Hello World!\")'])", + 'assert retcode == 0')) + output = subprocess.check_output([sys.executable, '-c', code]) + self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + + def test_handles_closed_on_exception(self): + # If CreateProcess exits with an error, ensure the + # duplicate output handles are released + ifhandle, ifname = tempfile.mkstemp() + ofhandle, ofname = tempfile.mkstemp() + efhandle, efname = tempfile.mkstemp() + try: + subprocess.Popen (["*"], stdin=ifhandle, stdout=ofhandle, + stderr=efhandle) + except OSError: + os.close(ifhandle) + os.remove(ifname) + os.close(ofhandle) + os.remove(ofname) + os.close(efhandle) + os.remove(efname) + self.assertFalse(os.path.exists(ifname)) + self.assertFalse(os.path.exists(ofname)) + self.assertFalse(os.path.exists(efname)) + + def test_communicate_epipe(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.communicate(b"x" * 2**20) + + def test_communicate_epipe_only_stdin(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + p.wait() + p.communicate(b"x" * 2**20) + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), + "Requires signal.SIGUSR1") + @unittest.skipUnless(hasattr(os, 'kill'), + "Requires os.kill") + @unittest.skipUnless(hasattr(os, 'getppid'), + "Requires os.getppid") + def test_communicate_eintr(self): + # Issue #12493: communicate() should handle EINTR + def handler(signum, frame): + pass + old_handler = signal.signal(signal.SIGUSR1, handler) + self.addCleanup(signal.signal, signal.SIGUSR1, old_handler) + + args = [sys.executable, "-c", + 'import os, signal;' + 'os.kill(os.getppid(), signal.SIGUSR1)'] + for stream in ('stdout', 'stderr'): + kw = {stream: subprocess.PIPE} + with subprocess.Popen(args, **kw) as process: + # communicate() will be interrupted by SIGUSR1 + process.communicate() + + + # This test is Linux-ish specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + def test_failed_child_execute_fd_leak(self): + """Test for the fork() failure fd leak reported in issue16327.""" + fd_directory = '/proc/%d/fd' % os.getpid() + fds_before_popen = os.listdir(fd_directory) + with self.assertRaises(PopenTestException): + PopenExecuteChildRaises( + [sys.executable, '-c', 'pass'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # NOTE: This test doesn't verify that the real _execute_child + # does not close the file descriptors itself on the way out + # during an exception. Code inspection has confirmed that. + + fds_after_exception = os.listdir(fd_directory) + self.assertEqual(fds_before_popen, fds_after_exception) + + +class RunFuncTestCase(BaseTestCase): + def run_python(self, code, **kwargs): + """Run Python code in a subprocess using subprocess.run""" + argv = [sys.executable, "-c", code] + return subprocess.run(argv, **kwargs) + + def test_returncode(self): + # call() function with sequence argument + cp = self.run_python("import sys; sys.exit(47)") + self.assertEqual(cp.returncode, 47) + with self.assertRaises(subprocess.CalledProcessError): + cp.check_returncode() + + def test_check(self): + with self.assertRaises(subprocess.CalledProcessError) as c: + self.run_python("import sys; sys.exit(47)", check=True) + self.assertEqual(c.exception.returncode, 47) + + def test_check_zero(self): + # check_returncode shouldn't raise when returncode is zero + cp = self.run_python("import sys; sys.exit(0)", check=True) + self.assertEqual(cp.returncode, 0) + + def test_timeout(self): + # run() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.run waits for the + # child. + with self.assertRaises(subprocess.TimeoutExpired): + self.run_python("while True: pass", timeout=0.0001) + + def test_capture_stdout(self): + # capture stdout with zero return code + cp = self.run_python("print('BDFL')", stdout=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stdout) + + def test_capture_stderr(self): + cp = self.run_python("import sys; sys.stderr.write('BDFL')", + stderr=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stderr) + + def test_check_output_stdin_arg(self): + # run() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + stdin=tf, stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + input=b'pear', stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_stdin_with_input_arg(self): + # run() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError, + msg="Expected ValueError when stdin and input args supplied.") as c: + output = self.run_python("print('will not be run')", + stdin=tf, input=b'hare') + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + with self.assertRaises(subprocess.TimeoutExpired) as c: + cp = self.run_python(( + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"), + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3, stdout=subprocess.PIPE) + self.assertEqual(c.exception.output, b'BDFL') + # output is aliased to stdout + self.assertEqual(c.exception.stdout, b'BDFL') + + def test_run_kwargs(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + cp = self.run_python(('import sys, os;' + 'sys.exit(33 if os.getenv("FRUIT")=="banana" else 31)'), + env=newenv) + self.assertEqual(cp.returncode, 33) + + +@unittest.skipIf(mswindows, "POSIX specific tests") +class POSIXProcessTestCase(BaseTestCase): + + def setUp(self): + super().setUp() + self._nonexistent_dir = "/_this/pa.th/does/not/exist" + + def _get_chdir_exception(self): + try: + os.chdir(self._nonexistent_dir) + except OSError as e: + # This avoids hard coding the errno value or the OS perror() + # string and instead capture the exception that we want to see + # below for comparison. + desired_exception = e + desired_exception.strerror += ': ' + repr(self._nonexistent_dir) + else: + self.fail("chdir to nonexistent directory %s succeeded." % + self._nonexistent_dir) + return desired_exception + + def test_exception_cwd(self): + """Test error in the child raised in the parent for a bad cwd.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + cwd=self._nonexistent_dir) + except OSError as e: + # Test that the child process chdir failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_executable(self): + """Test error in the child raised in the parent for a bad executable.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + executable=self._nonexistent_dir) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_args_0(self): + """Test error in the child raised in the parent for a bad args[0].""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([self._nonexistent_dir, "-c", ""]) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_restore_signals(self): + # Code coverage for both values of restore_signals to make sure it + # at least does not blow up. + # A test for behavior would be complex. Contributions welcome. + subprocess.call([sys.executable, "-c", ""], restore_signals=True) + subprocess.call([sys.executable, "-c", ""], restore_signals=False) + + def test_start_new_session(self): + # For code coverage of calling setsid(). We don't care if we get an + # EPERM error from it depending on the test execution environment, that + # still indicates that it was called. + try: + output = subprocess.check_output( + [sys.executable, "-c", + "import os; print(os.getpgid(os.getpid()))"], + start_new_session=True) + except OSError as e: + if e.errno != errno.EPERM: + raise + else: + parent_pgid = os.getpgid(os.getpid()) + child_pgid = int(output) + self.assertNotEqual(parent_pgid, child_pgid) + + def test_run_abort(self): + # returncode handles signal termination + with support.SuppressCrashReport(): + p = subprocess.Popen([sys.executable, "-c", + 'import os; os.abort()']) + p.wait() + self.assertEqual(-p.returncode, signal.SIGABRT) + + def test_preexec(self): + # DISCLAIMER: Setting environment variables is *not* a good use + # of a preexec_fn. This is merely a test. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + preexec_fn=lambda: os.putenv("FRUIT", "apple")) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), b"apple") + + def test_preexec_exception(self): + def raise_it(): + raise ValueError("What if two swallows carried a coconut?") + try: + p = subprocess.Popen([sys.executable, "-c", ""], + preexec_fn=raise_it) + except subprocess.SubprocessError as e: + self.assertTrue( + subprocess._posixsubprocess, + "Expected a ValueError from the preexec_fn") + except ValueError as e: + self.assertIn("coconut", e.args[0]) + else: + self.fail("Exception raised by preexec_fn did not make it " + "to the parent process.") + + class _TestExecuteChildPopen(subprocess.Popen): + """Used to test behavior at the end of _execute_child.""" + def __init__(self, testcase, *args, **kwargs): + self._testcase = testcase + subprocess.Popen.__init__(self, *args, **kwargs) + + def _execute_child(self, *args, **kwargs): + try: + subprocess.Popen._execute_child(self, *args, **kwargs) + finally: + # Open a bunch of file descriptors and verify that + # none of them are the same as the ones the Popen + # instance is using for stdin/stdout/stderr. + devzero_fds = [os.open("/dev/zero", os.O_RDONLY) + for _ in range(8)] + try: + for fd in devzero_fds: + self._testcase.assertNotIn( + fd, (self.stdin.fileno(), self.stdout.fileno(), + self.stderr.fileno()), + msg="At least one fd was closed early.") + finally: + for fd in devzero_fds: + os.close(fd) + + @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") + def test_preexec_errpipe_does_not_double_close_pipes(self): + """Issue16140: Don't double close pipes on preexec error.""" + + def raise_it(): + raise subprocess.SubprocessError( + "force the _execute_child() errpipe_data path.") + + with self.assertRaises(subprocess.SubprocessError): + self._TestExecuteChildPopen( + self, [sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=raise_it) + + def test_preexec_gc_module_failure(self): + # This tests the code that disables garbage collection if the child + # process will execute any Python. + def raise_runtime_error(): + raise RuntimeError("this shouldn't escape") + enabled = gc.isenabled() + orig_gc_disable = gc.disable + orig_gc_isenabled = gc.isenabled + try: + gc.disable() + self.assertFalse(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertFalse(gc.isenabled(), + "Popen enabled gc when it shouldn't.") + + gc.enable() + self.assertTrue(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertTrue(gc.isenabled(), "Popen left gc disabled.") + + gc.disable = raise_runtime_error + self.assertRaises(RuntimeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + + del gc.isenabled # force an AttributeError + self.assertRaises(AttributeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + finally: + gc.disable = orig_gc_disable + gc.isenabled = orig_gc_isenabled + if not enabled: + gc.disable() + + @unittest.skipIf( + sys.platform == 'darwin', 'setrlimit() seems to fail on OS X') + def test_preexec_fork_failure(self): + # The internal code did not preserve the previous exception when + # re-enabling garbage collection + try: + from resource import getrlimit, setrlimit, RLIMIT_NPROC + except ImportError as err: + self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD + limits = getrlimit(RLIMIT_NPROC) + [_, hard] = limits + setrlimit(RLIMIT_NPROC, (0, hard)) + self.addCleanup(setrlimit, RLIMIT_NPROC, limits) + try: + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + except BlockingIOError: + # Forking should raise EAGAIN, translated to BlockingIOError + pass + else: + self.skipTest('RLIMIT_NPROC had no effect; probably superuser') + + def test_args_string(self): + # args is a string + fd, fname = tempfile.mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!/bin/sh\n") + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + p = subprocess.Popen(fname) + p.wait() + os.remove(fname) + self.assertEqual(p.returncode, 47) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + startupinfo=47) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + creationflags=47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen(["echo $FRUIT"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen("echo $FRUIT", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_call_string(self): + # call() function with string argument on UNIX + fd, fname = tempfile.mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!/bin/sh\n") + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + rc = subprocess.call(fname) + os.remove(fname) + self.assertEqual(rc, 47) + + def test_specific_shell(self): + # Issue #9265: Incorrect name passed as arg[0]. + shells = [] + for prefix in ['/bin', '/usr/bin/', '/usr/local/bin']: + for name in ['bash', 'ksh']: + sh = os.path.join(prefix, name) + if os.path.isfile(sh): + shells.append(sh) + if not shells: # Will probably work for any shell but csh. + self.skipTest("bash or ksh required for this test") + sh = '/bin/sh' + if os.path.isfile(sh) and not os.path.islink(sh): + # Test will fail if /bin/sh is a symlink to csh. + shells.append(sh) + for sh in shells: + p = subprocess.Popen("echo $0", executable=sh, shell=True, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) + + def _kill_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + # Also set the SIGINT handler to the default to make sure it's not + # being ignored (some tests rely on that.) + old_handler = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + signal.signal(signal.SIGINT, old_handler) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + return p + + @unittest.skipIf(sys.platform.startswith(('netbsd', 'openbsd')), + "Due to known OS bug (issue #16762)") + def _kill_dead_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + p.communicate() + + def test_send_signal(self): + p = self._kill_process('send_signal', signal.SIGINT) + _, stderr = p.communicate() + self.assertIn(b'KeyboardInterrupt', stderr) + self.assertNotEqual(p.wait(), 0) + + def test_kill(self): + p = self._kill_process('kill') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGKILL) + + def test_terminate(self): + p = self._kill_process('terminate') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGTERM) + + def test_send_signal_dead(self): + # Sending a signal to a dead process + self._kill_dead_process('send_signal', signal.SIGINT) + + def test_kill_dead(self): + # Killing a dead process + self._kill_dead_process('kill') + + def test_terminate_dead(self): + # Terminating a dead process + self._kill_dead_process('terminate') + + def _save_fds(self, save_fds): + fds = [] + for fd in save_fds: + inheritable = os.get_inheritable(fd) + saved = os.dup(fd) + fds.append((fd, saved, inheritable)) + return fds + + def _restore_fds(self, fds): + for fd, saved, inheritable in fds: + os.dup2(saved, fd, inheritable=inheritable) + os.close(saved) + + def check_close_std_fds(self, fds): + # Issue #9905: test that subprocess pipes still work properly with + # some standard fds closed + stdin = 0 + saved_fds = self._save_fds(fds) + for fd, saved, inheritable in saved_fds: + if fd == 0: + stdin = saved + break + try: + for fd in fds: + os.close(fd) + out, err = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + err = support.strip_python_stderr(err) + self.assertEqual((out, err), (b'apple', b'orange')) + finally: + self._restore_fds(saved_fds) + + def test_close_fd_0(self): + self.check_close_std_fds([0]) + + def test_close_fd_1(self): + self.check_close_std_fds([1]) + + def test_close_fd_2(self): + self.check_close_std_fds([2]) + + def test_close_fds_0_1(self): + self.check_close_std_fds([0, 1]) + + def test_close_fds_0_2(self): + self.check_close_std_fds([0, 2]) + + def test_close_fds_1_2(self): + self.check_close_std_fds([1, 2]) + + def test_close_fds_0_1_2(self): + # Issue #10806: test that subprocess pipes still work properly with + # all standard fds closed. + self.check_close_std_fds([0, 1, 2]) + + def test_small_errpipe_write_fd(self): + """Issue #15798: Popen should work when stdio fds are available.""" + new_stdin = os.dup(0) + new_stdout = os.dup(1) + try: + os.close(0) + os.close(1) + + # Side test: if errpipe_write fails to have its CLOEXEC + # flag set this should cause the parent to think the exec + # failed. Extremely unlikely: everyone supports CLOEXEC. + subprocess.Popen([ + sys.executable, "-c", + "print('AssertionError:0:CLOEXEC failure.')"]).wait() + finally: + # Restore original stdin and stdout + os.dup2(new_stdin, 0) + os.dup2(new_stdout, 1) + os.close(new_stdin) + os.close(new_stdout) + + def test_remapping_std_fds(self): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + try: + temp_fds = [fd for fd, fname in temps] + + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # write some data to what will become stdin, and rewind + os.write(temp_fds[1], b"STDIN") + os.lseek(temp_fds[1], 0, 0) + + # move the standard file descriptors out of the way + saved_fds = self._save_fds(range(3)) + try: + # duplicate the file objects over the standard fd's + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # now use those files in the "wrong" order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=temp_fds[1], + stdout=temp_fds[2], + stderr=temp_fds[0]) + p.wait() + finally: + self._restore_fds(saved_fds) + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(temp_fds[2], 1024) + err = support.strip_python_stderr(os.read(temp_fds[0], 1024)) + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + def check_swap_fds(self, stdin_no, stdout_no, stderr_no): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + temp_fds = [fd for fd, fname in temps] + try: + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # save a copy of the standard file descriptors + saved_fds = self._save_fds(range(3)) + try: + # duplicate the temp files over the standard fd's 0, 1, 2 + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # write some data to what will become stdin, and rewind + os.write(stdin_no, b"STDIN") + os.lseek(stdin_no, 0, 0) + + # now use those files in the given order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=stdin_no, + stdout=stdout_no, + stderr=stderr_no) + p.wait() + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(stdout_no, 1024) + err = support.strip_python_stderr(os.read(stderr_no, 1024)) + finally: + self._restore_fds(saved_fds) + + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + # When duping fds, if there arises a situation where one of the fds is + # either 0, 1 or 2, it is possible that it is overwritten (#12607). + # This tests all combinations of this. + def test_swap_fds(self): + self.check_swap_fds(0, 1, 2) + self.check_swap_fds(0, 2, 1) + self.check_swap_fds(1, 0, 2) + self.check_swap_fds(1, 2, 0) + self.check_swap_fds(2, 0, 1) + self.check_swap_fds(2, 1, 0) + + def test_surrogates_error_message(self): + def prepare(): + raise ValueError("surrogate:\uDCff") + + try: + subprocess.call( + [sys.executable, "-c", "pass"], + preexec_fn=prepare) + except ValueError as err: + # Pure Python implementations keeps the message + self.assertIsNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "surrogate:\uDCff") + except subprocess.SubprocessError as err: + # _posixsubprocess uses a default message + self.assertIsNotNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "Exception occurred in preexec_fn.") + else: + self.fail("Expected ValueError or subprocess.SubprocessError") + + def test_undecodable_env(self): + for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): + encoded_value = value.encode("ascii", "surrogateescape") + + # test str with surrogates + script = "import os; print(ascii(os.getenv(%s)))" % repr(key) + env = os.environ.copy() + env[key] = value + # Use C locale to get ASCII for the locale encoding to force + # surrogate-escaping of \xFF in the child process; otherwise it can + # be decoded as-is if the default locale is latin-1. + env['LC_ALL'] = 'C' + if sys.platform.startswith("aix"): + # On AIX, the C locale uses the Latin1 encoding + decoded_value = encoded_value.decode("latin1", "surrogateescape") + else: + # On other UNIXes, the C locale uses the ASCII encoding + decoded_value = value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(decoded_value)) + + # test bytes + key = key.encode("ascii", "surrogateescape") + script = "import os; print(ascii(os.getenvb(%s)))" % repr(key) + env = os.environ.copy() + env[key] = encoded_value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(encoded_value)) + + def test_bytes_program(self): + abs_program = os.fsencode(sys.executable) + path, program = os.path.split(sys.executable) + program = os.fsencode(program) + + # absolute bytes path + exitcode = subprocess.call([abs_program, "-c", "pass"]) + self.assertEqual(exitcode, 0) + + # absolute bytes path as a string + cmd = b"'" + abs_program + b"' -c pass" + exitcode = subprocess.call(cmd, shell=True) + self.assertEqual(exitcode, 0) + + # bytes program, unicode PATH + env = os.environ.copy() + env["PATH"] = path + exitcode = subprocess.call([program, "-c", "pass"], env=env) + self.assertEqual(exitcode, 0) + + # bytes program, bytes PATH + envb = os.environb.copy() + envb[b"PATH"] = os.fsencode(path) + exitcode = subprocess.call([program, "-c", "pass"], env=envb) + self.assertEqual(exitcode, 0) + + def test_pipe_cloexec(self): + sleeper = support.findfile("input_reader.py", subdir="subprocessdata") + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + p1 = subprocess.Popen([sys.executable, sleeper], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=False) + + self.addCleanup(p1.communicate, b'') + + p2 = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + + output, error = p2.communicate() + result_fds = set(map(int, output.split(b','))) + unwanted_fds = set([p1.stdin.fileno(), p1.stdout.fileno(), + p1.stderr.fileno()]) + + self.assertFalse(result_fds & unwanted_fds, + "Expected no fds from %r to be open in child, " + "found %r" % + (unwanted_fds, result_fds & unwanted_fds)) + + def test_pipe_cloexec_real_tools(self): + qcat = support.findfile("qcat.py", subdir="subprocessdata") + qgrep = support.findfile("qgrep.py", subdir="subprocessdata") + + subdata = b'zxcvbn' + data = subdata * 4 + b'\n' + + p1 = subprocess.Popen([sys.executable, qcat], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + close_fds=False) + + p2 = subprocess.Popen([sys.executable, qgrep, subdata], + stdin=p1.stdout, stdout=subprocess.PIPE, + close_fds=False) + + self.addCleanup(p1.wait) + self.addCleanup(p2.wait) + def kill_p1(): + try: + p1.terminate() + except ProcessLookupError: + pass + def kill_p2(): + try: + p2.terminate() + except ProcessLookupError: + pass + self.addCleanup(kill_p1) + self.addCleanup(kill_p2) + + p1.stdin.write(data) + p1.stdin.close() + + readfiles, ignored1, ignored2 = select.select([p2.stdout], [], [], 10) + + self.assertTrue(readfiles, "The child hung") + self.assertEqual(p2.stdout.read(), data) + + p1.stdout.close() + p2.stdout.close() + + def test_close_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + + open_fds = set(fds) + # add a bunch more fds + for _ in range(9): + fd = os.open(os.devnull, os.O_RDONLY) + self.addCleanup(os.close, fd) + open_fds.add(fd) + + for fd in open_fds: + os.set_inheritable(fd, True) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertEqual(remaining_fds & open_fds, open_fds, + "Some fds were closed") + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & open_fds, + "Some fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + # Keep some of the fd's we opened open in the subprocess. + # This tests _posixsubprocess.c's proper handling of fds_to_keep. + fds_to_keep = set(open_fds.pop() for _ in range(8)) + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=()) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & fds_to_keep & open_fds, + "Some fds not in pass_fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD.") + def test_close_fds_when_max_fd_is_lowered(self): + """Confirm that issue21618 is fixed (may fail under valgrind).""" + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # This launches the meat of the test in a child process to + # avoid messing with the larger unittest processes maximum + # number of file descriptors. + # This process launches: + # +--> Process that lowers its RLIMIT_NOFILE aftr setting up + # a bunch of high open fds above the new lower rlimit. + # Those are reported via stdout before launching a new + # process with close_fds=False to run the actual test: + # +--> The TEST: This one launches a fd_status.py + # subprocess with close_fds=True so we can find out if + # any of the fds above the lowered rlimit are still open. + p = subprocess.Popen([sys.executable, '-c', textwrap.dedent( + ''' + import os, resource, subprocess, sys, textwrap + open_fds = set() + # Add a bunch more fds to pass down. + for _ in range(40): + fd = os.open(os.devnull, os.O_RDONLY) + open_fds.add(fd) + + # Leave a two pairs of low ones available for use by the + # internal child error pipe and the stdout pipe. + # We also leave 10 more open as some Python buildbots run into + # "too many open files" errors during the test if we do not. + for fd in sorted(open_fds)[:14]: + os.close(fd) + open_fds.remove(fd) + + for fd in open_fds: + #self.addCleanup(os.close, fd) + os.set_inheritable(fd, True) + + max_fd_open = max(open_fds) + + # Communicate the open_fds to the parent unittest.TestCase process. + print(','.join(map(str, sorted(open_fds)))) + sys.stdout.flush() + + rlim_cur, rlim_max = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + # 29 is lower than the highest fds we are leaving open. + resource.setrlimit(resource.RLIMIT_NOFILE, (29, rlim_max)) + # Launch a new Python interpreter with our low fd rlim_cur that + # inherits open fds above that limit. It then uses subprocess + # with close_fds=True to get a report of open fds in the child. + # An explicit list of fds to check is passed to fd_status.py as + # letting fd_status rely on its default logic would miss the + # fds above rlim_cur as it normally only checks up to that limit. + subprocess.Popen( + [sys.executable, '-c', + textwrap.dedent(""" + import subprocess, sys + subprocess.Popen([sys.executable, %r] + + [str(x) for x in range({max_fd})], + close_fds=True).wait() + """.format(max_fd=max_fd_open+1))], + close_fds=False).wait() + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, (rlim_cur, rlim_max)) + ''' % fd_status)], stdout=subprocess.PIPE) + + output, unused_stderr = p.communicate() + output_lines = output.splitlines() + self.assertEqual(len(output_lines), 2, + msg="expected exactly two lines of output:\n%r" % output) + opened_fds = set(map(int, output_lines[0].strip().split(b','))) + remaining_fds = set(map(int, output_lines[1].strip().split(b','))) + + self.assertFalse(remaining_fds & opened_fds, + msg="Some fds were left open.") + + + # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file + # descriptor of a pipe closed in the parent process is valid in the + # child process according to fstat(), but the mode of the file + # descriptor is invalid, and read or write raise an error. + @support.requires_mac_ver(10, 5) + def test_pass_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + open_fds = set() + + for x in range(5): + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + os.set_inheritable(fds[0], True) + os.set_inheritable(fds[1], True) + open_fds.update(fds) + + for fd in open_fds: + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=(fd, )) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + to_be_closed = open_fds - {fd} + + self.assertIn(fd, remaining_fds, "fd to be passed not passed") + self.assertFalse(remaining_fds & to_be_closed, + "fd to be closed passed") + + # pass_fds overrides close_fds with a warning. + with self.assertWarns(RuntimeWarning) as context: + self.assertFalse(subprocess.call( + [sys.executable, "-c", "import sys; sys.exit(0)"], + close_fds=False, pass_fds=(fd, ))) + self.assertIn('overriding close_fds', str(context.warning)) + + def test_pass_fds_inheritable(self): + script = support.findfile("fd_status.py", subdir="subprocessdata") + + inheritable, non_inheritable = os.pipe() + self.addCleanup(os.close, inheritable) + self.addCleanup(os.close, non_inheritable) + os.set_inheritable(inheritable, True) + os.set_inheritable(non_inheritable, False) + pass_fds = (inheritable, non_inheritable) + args = [sys.executable, script] + args += list(map(str, pass_fds)) + + p = subprocess.Popen(args, + stdout=subprocess.PIPE, close_fds=True, + pass_fds=pass_fds) + output, ignored = p.communicate() + fds = set(map(int, output.split(b','))) + + # the inheritable file descriptor must be inherited, so its inheritable + # flag must be set in the child process after fork() and before exec() + self.assertEqual(fds, set(pass_fds), "output=%a" % output) + + # inheritable flag must not be changed in the parent process + self.assertEqual(os.get_inheritable(inheritable), True) + self.assertEqual(os.get_inheritable(non_inheritable), False) + + def test_stdout_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stdin=inout) + p.wait() + + def test_stdout_stderr_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stderr=inout) + p.wait() + + def test_stderr_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stderr=inout, stdin=inout) + p.wait() + + def test_wait_when_sigchild_ignored(self): + # NOTE: sigchild_ignore.py may not be an effective test on all OSes. + sigchild_ignore = support.findfile("sigchild_ignore.py", + subdir="subprocessdata") + p = subprocess.Popen([sys.executable, sigchild_ignore], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" + " non-zero with this error:\n%s" % + stderr.decode('utf-8')) + + def test_select_unbuffered(self): + # Issue #11459: bufsize=0 should really set the pipes as + # unbuffered (and therefore let select() work properly). + select = support.import_module("select") + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple")'], + stdout=subprocess.PIPE, + bufsize=0) + f = p.stdout + self.addCleanup(f.close) + try: + self.assertEqual(f.read(4), b"appl") + self.assertIn(f, select.select([f], [], [], 0.0)[0]) + finally: + p.wait() + + def test_zombie_fast_process_del(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, it wouldn't be added to subprocess._active, and would + # remain a zombie. + # spawn a Popen, and delete its reference before it exits + p = subprocess.Popen([sys.executable, "-c", + 'import sys, time;' + 'time.sleep(0.2)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + def test_leak_fast_process_del_killed(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, and the process got killed by a signal, it would never + # be removed from subprocess._active, which triggered a FD and memory + # leak. + # spawn a Popen, delete its reference and kill it + p = subprocess.Popen([sys.executable, "-c", + 'import time;' + 'time.sleep(3)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + os.kill(pid, signal.SIGKILL) + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + # let some time for the process to exit, and create a new Popen: this + # should trigger the wait() of p + time.sleep(0.2) + with self.assertRaises(OSError) as c: + with subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + # p should have been wait()ed on, and removed from the _active list + self.assertRaises(OSError, os.waitpid, pid, 0) + self.assertNotIn(ident, [id(o) for o in subprocess._active]) + + def test_close_fds_after_preexec(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # this FD is used as dup2() target by preexec_fn, and should be closed + # in the child process + fd = os.dup(1) + self.addCleanup(os.close, fd) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + preexec_fn=lambda: os.dup2(1, fd)) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + + self.assertNotIn(fd, remaining_fds) + + @support.cpython_only + def test_fork_exec(self): + # Issue #22290: fork_exec() must not crash on memory allocation failure + # or other errors + import _posixsubprocess + gc_enabled = gc.isenabled() + try: + # Use a preexec function and enable the garbage collector + # to force fork_exec() to re-enable the garbage collector + # on error. + func = lambda: None + gc.enable() + + for args, exe_list, cwd, env_list in ( + (123, [b"exe"], None, [b"env"]), + ([b"arg"], 123, None, [b"env"]), + ([b"arg"], [b"exe"], 123, [b"env"]), + ([b"arg"], [b"exe"], None, 123), + ): + with self.assertRaises(TypeError): + _posixsubprocess.fork_exec( + args, exe_list, + True, [], cwd, env_list, + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, func) + finally: + if not gc_enabled: + gc.disable() + + @support.cpython_only + def test_fork_exec_sorted_fd_sanity_check(self): + # Issue #23564: sanity check the fork_exec() fds_to_keep sanity check. + import _posixsubprocess + gc_enabled = gc.isenabled() + try: + gc.enable() + + for fds_to_keep in ( + (-1, 2, 3, 4, 5), # Negative number. + ('str', 4), # Not an int. + (18, 23, 42, 2**63), # Out of range. + (5, 4), # Not sorted. + (6, 7, 7, 8), # Duplicate. + ): + with self.assertRaises( + ValueError, + msg='fds_to_keep={}'.format(fds_to_keep)) as c: + _posixsubprocess.fork_exec( + [b"false"], [b"false"], + True, fds_to_keep, None, [b"env"], + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, None) + self.assertIn('fds_to_keep', str(c.exception)) + finally: + if not gc_enabled: + gc.disable() + + def test_communicate_BrokenPipeError_stdin_close(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + proc.communicate() # Should swallow BrokenPipeError from close. + mock_proc_stdin.close.assert_called_with() + + def test_communicate_BrokenPipeError_stdin_write(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.write.side_effect = BrokenPipeError + proc.communicate(b'stuff') # Should swallow the BrokenPipeError. + mock_proc_stdin.write.assert_called_once_with(b'stuff') + mock_proc_stdin.close.assert_called_once_with() + + def test_communicate_BrokenPipeError_stdin_flush(self): + # Setting stdin and stdout forces the ._communicate() code path. + # python -h exits faster than python -c pass (but spams stdout). + proc = subprocess.Popen([sys.executable, '-h'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin, \ + open(os.devnull, 'wb') as dev_null: + mock_proc_stdin.flush.side_effect = BrokenPipeError + # because _communicate registers a selector using proc.stdin... + mock_proc_stdin.fileno.return_value = dev_null.fileno() + # _communicate() should swallow BrokenPipeError from flush. + proc.communicate(b'stuff') + mock_proc_stdin.flush.assert_called_once_with() + + def test_communicate_BrokenPipeError_stdin_close_with_timeout(self): + # Setting stdin and stdout forces the ._communicate() code path. + # python -h exits faster than python -c pass (but spams stdout). + proc = subprocess.Popen([sys.executable, '-h'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + # _communicate() should swallow BrokenPipeError from close. + proc.communicate(timeout=999) + mock_proc_stdin.close.assert_called_once_with() + + +@unittest.skipUnless(mswindows, "Windows specific tests") +class Win32ProcessTestCase(BaseTestCase): + + def test_startupinfo(self): + # startupinfo argument + # We uses hardcoded constants, because we do not want to + # depend on win32all. + STARTF_USESHOWWINDOW = 1 + SW_MAXIMIZE = 3 + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = STARTF_USESHOWWINDOW + startupinfo.wShowWindow = SW_MAXIMIZE + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_creationflags(self): + # creationflags argument + CREATE_NEW_CONSOLE = 16 + sys.stderr.write(" a DOS box should flash briefly ...\n") + subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + preexec_fn=lambda: 1) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + stdout=subprocess.PIPE, + close_fds=True) + + def test_close_fds(self): + # close file descriptors + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"], + close_fds=True) + self.assertEqual(rc, 47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen(["set"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn(b"physalis", p.stdout.read()) + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn(b"physalis", p.stdout.read()) + + def test_call_string(self): + # call() function with string argument on Windows + rc = subprocess.call(sys.executable + + ' -c "import sys; sys.exit(47)"') + self.assertEqual(rc, 47) + + def _kill_process(self, method, *args): + # Some win32 buildbot raises EOFError if stdin is inherited + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + returncode = p.wait() + self.assertNotEqual(returncode, 0) + + def _kill_dead_process(self, method, *args): + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + sys.exit(42) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() + self.assertEqual(rc, 42) + + def test_send_signal(self): + self._kill_process('send_signal', signal.SIGTERM) + + def test_kill(self): + self._kill_process('kill') + + def test_terminate(self): + self._kill_process('terminate') + + def test_send_signal_dead(self): + self._kill_dead_process('send_signal', signal.SIGTERM) + + def test_kill_dead(self): + self._kill_dead_process('kill') + + def test_terminate_dead(self): + self._kill_dead_process('terminate') + +class MiscTests(unittest.TestCase): + def test_getoutput(self): + self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') + self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), + (0, 'xyzzy')) + + # we use mkdtemp in the next line to create an empty directory + # under our exclusive control; from that, we can invent a pathname + # that we _know_ won't exist. This is guaranteed to fail. + dir = None + try: + dir = tempfile.mkdtemp() + name = os.path.join(dir, "foo") + status, output = subprocess.getstatusoutput( + ("type " if mswindows else "cat ") + name) + self.assertNotEqual(status, 0) + finally: + if dir is not None: + os.rmdir(dir) + + def test__all__(self): + """Ensure that __all__ is populated properly.""" + # STARTUPINFO added to __all__ in 3.6 + intentionally_excluded = {"list2cmdline", "STARTUPINFO", "Handle"} + exported = set(subprocess.__all__) + possible_exports = set() + import types + for name, value in subprocess.__dict__.items(): + if name.startswith('_'): + continue + if isinstance(value, (types.ModuleType,)): + continue + possible_exports.add(name) + self.assertEqual(exported, possible_exports - intentionally_excluded) + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class ProcessTestCaseNoPoll(ProcessTestCase): + def setUp(self): + self.orig_selector = subprocess._PopenSelector + subprocess._PopenSelector = selectors.SelectSelector + ProcessTestCase.setUp(self) + + def tearDown(self): + subprocess._PopenSelector = self.orig_selector + ProcessTestCase.tearDown(self) + + +@unittest.skipUnless(mswindows, "Windows-specific tests") +class CommandsWithSpaces (BaseTestCase): + + def setUp(self): + super().setUp() + f, fname = tempfile.mkstemp(".py", "te st") + self.fname = fname.lower () + os.write(f, b"import sys;" + b"sys.stdout.write('%d %s' % (len(sys.argv), [a.lower () for a in sys.argv]))" + ) + os.close(f) + + def tearDown(self): + os.remove(self.fname) + super().tearDown() + + def with_spaces(self, *args, **kwargs): + kwargs['stdout'] = subprocess.PIPE + p = subprocess.Popen(*args, **kwargs) + self.addCleanup(p.stdout.close) + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) + + def test_shell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd"), shell=1) + + def test_shell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) + + def test_noshell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd")) + + def test_noshell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"]) + + +class ContextManagerTests(BaseTestCase): + + def test_pipe(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write('stdout');" + "sys.stderr.write('stderr');"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + self.assertEqual(proc.stdout.read(), b"stdout") + self.assertStderrEqual(proc.stderr.read(), b"stderr") + + self.assertTrue(proc.stdout.closed) + self.assertTrue(proc.stderr.closed) + + def test_returncode(self): + with subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(100)"]) as proc: + pass + # __exit__ calls wait(), so the returncode should be set + self.assertEqual(proc.returncode, 100) + + def test_communicate_stdin(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.exit(sys.stdin.read() == 'context')"], + stdin=subprocess.PIPE) as proc: + proc.communicate(b"context") + self.assertEqual(proc.returncode, 1) + + def test_invalid_args(self): + with self.assertRaises(FileNotFoundError) as c: + with subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + + def test_broken_pipe_cleanup(self): + """Broken pipe error should not prevent wait() (Issue 21619)""" + proc = subprocess.Popen([sys.executable, '-c', 'pass'], + stdin=subprocess.PIPE, + bufsize=support.PIPE_MAX_SIZE*2) + proc = proc.__enter__() + # Prepare to send enough data to overflow any OS pipe buffering and + # guarantee a broken pipe error. Data is held in BufferedWriter + # buffer until closed. + proc.stdin.write(b'x' * support.PIPE_MAX_SIZE) + self.assertIsNone(proc.returncode) + # EPIPE expected under POSIX; EINVAL under Windows + self.assertRaises(OSError, proc.__exit__, None, None, None) + self.assertEqual(proc.returncode, 0) + self.assertTrue(proc.stdin.closed) + + +def test_main(): + unit_tests = (ProcessTestCase, + POSIXProcessTestCase, + Win32ProcessTestCase, + MiscTests, + ProcessTestCaseNoPoll, + CommandsWithSpaces, + ContextManagerTests, + RunFuncTestCase, + ) + + support.run_unittest(*unit_tests) + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5/test_threading.py b/src/greentest/3.5/test_threading.py new file mode 100644 index 0000000..e9334d3 --- /dev/null +++ b/src/greentest/3.5/test_threading.py @@ -0,0 +1,1133 @@ +""" +Tests for the threading module. +""" + +import test.support +from test.support import (verbose, import_module, cpython_only, + requires_type_collecting) +from test.support.script_helper import assert_python_ok, assert_python_failure + +import random +import re +import sys +_thread = import_module('_thread') +threading = import_module('threading') +import time +import unittest +import weakref +import os +import subprocess + +from test import lock_tests + + +# Between fork() and exec(), only async-safe functions are allowed (issues +# #12316 and #11870), and fork() from a worker thread is known to trigger +# problems with some operating systems (issue #3863): skip problematic tests +# on platforms known to behave badly. +platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', + 'hp-ux11') + + +# A trivial mutable counter. +class Counter(object): + def __init__(self): + self.value = 0 + def inc(self): + self.value += 1 + def dec(self): + self.value -= 1 + def get(self): + return self.value + +class TestThread(threading.Thread): + def __init__(self, name, testcase, sema, mutex, nrunning): + threading.Thread.__init__(self, name=name) + self.testcase = testcase + self.sema = sema + self.mutex = mutex + self.nrunning = nrunning + + def run(self): + delay = random.random() / 10000.0 + if verbose: + print('task %s will run for %.1f usec' % + (self.name, delay * 1e6)) + + with self.sema: + with self.mutex: + self.nrunning.inc() + if verbose: + print(self.nrunning.get(), 'tasks are running') + self.testcase.assertLessEqual(self.nrunning.get(), 3) + + time.sleep(delay) + if verbose: + print('task', self.name, 'done') + + with self.mutex: + self.nrunning.dec() + self.testcase.assertGreaterEqual(self.nrunning.get(), 0) + if verbose: + print('%s is finished. %d tasks are running' % + (self.name, self.nrunning.get())) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test.support.threading_setup() + + def tearDown(self): + test.support.threading_cleanup(*self._threads) + test.support.reap_children() + + +class ThreadTests(BaseTestCase): + + # Create a bunch of threads, let each do some work, wait until all are + # done. + def test_various_ops(self): + # This takes about n/3 seconds to run (about n/3 clumps of tasks, + # times about 1 second per clump). + NUMTASKS = 10 + + # no more than 3 of the 10 can run at once + sema = threading.BoundedSemaphore(value=3) + mutex = threading.RLock() + numrunning = Counter() + + threads = [] + + for i in range(NUMTASKS): + t = TestThread(""%i, self, sema, mutex, numrunning) + threads.append(t) + self.assertIsNone(t.ident) + self.assertRegex(repr(t), r'^$') + t.start() + + if verbose: + print('waiting for all tasks to complete') + for t in threads: + t.join() + self.assertFalse(t.is_alive()) + self.assertNotEqual(t.ident, 0) + self.assertIsNotNone(t.ident) + self.assertRegex(repr(t), r'^$') + if verbose: + print('all tasks done') + self.assertEqual(numrunning.get(), 0) + + def test_ident_of_no_threading_threads(self): + # The ident still must work for the main thread and dummy threads. + self.assertIsNotNone(threading.currentThread().ident) + def f(): + ident.append(threading.currentThread().ident) + done.set() + done = threading.Event() + ident = [] + _thread.start_new_thread(f, ()) + done.wait() + self.assertIsNotNone(ident[0]) + # Kill the "immortal" _DummyThread + del threading._active[ident[0]] + + # run with a small(ish) thread stack size (256kB) + def test_various_ops_small_stack(self): + if verbose: + print('with 256kB thread stack size...') + try: + threading.stack_size(262144) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1MB) + def test_various_ops_large_stack(self): + if verbose: + print('with 1MB thread stack size...') + try: + threading.stack_size(0x100000) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + def test_foreign_thread(self): + # Check that a "foreign" thread can use the threading module. + def f(mutex): + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + threading.current_thread() + mutex.release() + + mutex = threading.Lock() + mutex.acquire() + tid = _thread.start_new_thread(f, (mutex,)) + # Wait for the thread to finish. + mutex.acquire() + self.assertIn(tid, threading._active) + self.assertIsInstance(threading._active[tid], threading._DummyThread) + del threading._active[tid] + + # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) + # exposed at the Python level. This test relies on ctypes to get at it. + def test_PyThreadState_SetAsyncExc(self): + ctypes = import_module("ctypes") + + set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc + + class AsyncExc(Exception): + pass + + exception = ctypes.py_object(AsyncExc) + + # First check it works when setting the exception from the same thread. + tid = threading.get_ident() + + try: + result = set_async_exc(ctypes.c_long(tid), exception) + # The exception is async, so we might have to keep the VM busy until + # it notices. + while True: + pass + except AsyncExc: + pass + else: + # This code is unreachable but it reflects the intent. If we wanted + # to be smarter the above loop wouldn't be infinite. + self.fail("AsyncExc not raised") + try: + self.assertEqual(result, 1) # one thread state modified + except UnboundLocalError: + # The exception was raised too quickly for us to get the result. + pass + + # `worker_started` is set by the thread when it's inside a try/except + # block waiting to catch the asynchronously set AsyncExc exception. + # `worker_saw_exception` is set by the thread upon catching that + # exception. + worker_started = threading.Event() + worker_saw_exception = threading.Event() + + class Worker(threading.Thread): + def run(self): + self.id = threading.get_ident() + self.finished = False + + try: + while True: + worker_started.set() + time.sleep(0.1) + except AsyncExc: + self.finished = True + worker_saw_exception.set() + + t = Worker() + t.daemon = True # so if this fails, we don't hang Python at shutdown + t.start() + if verbose: + print(" started worker thread") + + # Try a thread id that doesn't make sense. + if verbose: + print(" trying nonsensical thread id") + result = set_async_exc(ctypes.c_long(-1), exception) + self.assertEqual(result, 0) # no thread states modified + + # Now raise an exception in the worker thread. + if verbose: + print(" waiting for worker thread to get started") + ret = worker_started.wait() + self.assertTrue(ret) + if verbose: + print(" verifying worker hasn't exited") + self.assertFalse(t.finished) + if verbose: + print(" attempting to raise asynch exception in worker") + result = set_async_exc(ctypes.c_long(t.id), exception) + self.assertEqual(result, 1) # one thread state modified + if verbose: + print(" waiting for worker to say it caught the exception") + worker_saw_exception.wait(timeout=10) + self.assertTrue(t.finished) + if verbose: + print(" all OK -- joining worker") + if t.finished: + t.join() + # else the thread is still running, and we have no way to kill it + + def test_limbo_cleanup(self): + # Issue 7481: Failure to start thread should cleanup the limbo map. + def fail_new_thread(*args): + raise threading.ThreadError() + _start_new_thread = threading._start_new_thread + threading._start_new_thread = fail_new_thread + try: + t = threading.Thread(target=lambda: None) + self.assertRaises(threading.ThreadError, t.start) + self.assertFalse( + t in threading._limbo, + "Failed to cleanup _limbo map on failure of Thread.start().") + finally: + threading._start_new_thread = _start_new_thread + + def test_finalize_runnning_thread(self): + # Issue 1402: the PyGILState_Ensure / _Release functions may be called + # very late on python exit: on deallocation of a running thread for + # example. + import_module("ctypes") + + rc, out, err = assert_python_failure("-c", """if 1: + import ctypes, sys, time, _thread + + # This lock is used as a simple event variable. + ready = _thread.allocate_lock() + ready.acquire() + + # Module globals are cleared before __del__ is run + # So we save the functions in class dict + class C: + ensure = ctypes.pythonapi.PyGILState_Ensure + release = ctypes.pythonapi.PyGILState_Release + def __del__(self): + state = self.ensure() + self.release(state) + + def waitingThread(): + x = C() + ready.release() + time.sleep(100) + + _thread.start_new_thread(waitingThread, ()) + ready.acquire() # Be sure the other thread is waiting. + sys.exit(42) + """) + self.assertEqual(rc, 42) + + def test_finalize_with_trace(self): + # Issue1733757 + # Avoid a deadlock when sys.settrace steps into threading._shutdown + assert_python_ok("-c", """if 1: + import sys, threading + + # A deadlock-killer, to prevent the + # testsuite to hang forever + def killer(): + import os, time + time.sleep(2) + print('program blocked; aborting') + os._exit(2) + t = threading.Thread(target=killer) + t.daemon = True + t.start() + + # This is the trace function + def func(frame, event, arg): + threading.current_thread() + return func + + sys.settrace(func) + """) + + def test_join_nondaemon_on_shutdown(self): + # Issue 1722344 + # Raising SystemExit skipped threading._shutdown + rc, out, err = assert_python_ok("-c", """if 1: + import threading + from time import sleep + + def child(): + sleep(1) + # As a non-daemon thread we SHOULD wake up and nothing + # should be torn down yet + print("Woke up, sleep function is:", sleep) + + threading.Thread(target=child).start() + raise SystemExit + """) + self.assertEqual(out.strip(), + b"Woke up, sleep function is: ") + self.assertEqual(err, b"") + + def test_enumerate_after_join(self): + # Try hard to trigger #1703448: a thread is still returned in + # threading.enumerate() after it has been join()ed. + enum = threading.enumerate + old_interval = sys.getswitchinterval() + try: + for i in range(1, 100): + sys.setswitchinterval(i * 0.0002) + t = threading.Thread(target=lambda: None) + t.start() + t.join() + l = enum() + self.assertNotIn(t, l, + "#1703448 triggered after %d trials: %s" % (i, l)) + finally: + sys.setswitchinterval(old_interval) + + def test_no_refcycle_through_target(self): + class RunSelfFunction(object): + def __init__(self, should_raise): + # The links in this refcycle from Thread back to self + # should be cleaned up when the thread completes. + self.should_raise = should_raise + self.thread = threading.Thread(target=self._run, + args=(self,), + kwargs={'yet_another':self}) + self.thread.start() + + def _run(self, other_ref, yet_another): + if self.should_raise: + raise SystemExit + + cyclic_object = RunSelfFunction(should_raise=False) + weak_cyclic_object = weakref.ref(cyclic_object) + cyclic_object.thread.join() + del cyclic_object + self.assertIsNone(weak_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_cyclic_object()))) + + raising_cyclic_object = RunSelfFunction(should_raise=True) + weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) + raising_cyclic_object.thread.join() + del raising_cyclic_object + self.assertIsNone(weak_raising_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_raising_cyclic_object()))) + + def test_old_threading_api(self): + # Just a quick sanity check to make sure the old method names are + # still present + t = threading.Thread() + t.isDaemon() + t.setDaemon(True) + t.getName() + t.setName("name") + t.isAlive() + e = threading.Event() + e.isSet() + threading.activeCount() + + def test_repr_daemon(self): + t = threading.Thread() + self.assertNotIn('daemon', repr(t)) + t.daemon = True + self.assertIn('daemon', repr(t)) + + def test_deamon_param(self): + t = threading.Thread() + self.assertFalse(t.daemon) + t = threading.Thread(daemon=False) + self.assertFalse(t.daemon) + t = threading.Thread(daemon=True) + self.assertTrue(t.daemon) + + @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + def test_dummy_thread_after_fork(self): + # Issue #14308: a dummy thread in the active list doesn't mess up + # the after-fork mechanism. + code = """if 1: + import _thread, threading, os, time + + def background_thread(evt): + # Creates and registers the _DummyThread instance + threading.current_thread() + evt.set() + time.sleep(10) + + evt = threading.Event() + _thread.start_new_thread(background_thread, (evt,)) + evt.wait() + assert threading.active_count() == 2, threading.active_count() + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + os.wait() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + old_interval = sys.getswitchinterval() + self.addCleanup(sys.setswitchinterval, old_interval) + + # Make the bug more likely to manifest. + sys.setswitchinterval(1e-6) + + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + self.addCleanup(t.join) + pid = os.fork() + if pid == 0: + os._exit(1 if t.is_alive() else 0) + else: + pid, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + def test_main_thread(self): + main = threading.main_thread() + self.assertEqual(main.name, 'MainThread') + self.assertEqual(main.ident, threading.current_thread().ident) + self.assertEqual(main.ident, threading.get_ident()) + + def f(): + self.assertNotEqual(threading.main_thread().ident, + threading.current_thread().ident) + th = threading.Thread(target=f) + th.start() + th.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork(self): + code = """if 1: + import os, threading + + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + else: + os.waitpid(pid, 0) + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "MainThread\nTrue\nTrue\n") + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_nonmain_thread(self): + code = """if 1: + import os, threading, sys + + def f(): + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + os.waitpid(pid, 0) + + th = threading.Thread(target=f) + th.start() + th.join() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertIsNone(t._tstate_lock) + + def test_repr_stopped(self): + # Verify that "stopped" shows up in repr(Thread) appropriately. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + t = threading.Thread(target=f) + t.start() + started.acquire() + self.assertIn("started", repr(t)) + finish.release() + # "stopped" should appear in the repr in a reasonable amount of time. + # Implementation detail: as of this writing, that's trivially true + # if .join() is called, and almost trivially true if .is_alive() is + # called. The detail we're testing here is that "stopped" shows up + # "all on its own". + LOOKING_FOR = "stopped" + for i in range(500): + if LOOKING_FOR in repr(t): + break + time.sleep(0.01) + self.assertIn(LOOKING_FOR, repr(t)) # we waited at least 5 seconds + + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) + + @cpython_only + def test_frame_tstate_tracing(self): + # Issue #14432: Crash when a generator is created in a C thread that is + # destroyed while the generator is still used. The issue was that a + # generator contains a frame, and the frame kept a reference to the + # Python state of the destroyed C thread. The crash occurs when a trace + # function is setup. + + def noop_trace(frame, event, arg): + # no operation + return noop_trace + + def generator(): + while 1: + yield "generator" + + def callback(): + if callback.gen is None: + callback.gen = generator() + return next(callback.gen) + callback.gen = None + + old_trace = sys.gettrace() + sys.settrace(noop_trace) + try: + # Install a trace function + threading.settrace(noop_trace) + + # Create a generator in a C thread which exits after the call + import _testcapi + _testcapi.call_in_temporary_c_thread(callback) + + # Call the generator in a different Python thread, check that the + # generator didn't keep a reference to the destroyed thread state + for test in range(3): + # The trace function is still called here + callback() + finally: + sys.settrace(old_trace) + + +class ThreadJoinOnShutdown(BaseTestCase): + + def _run_and_join(self, script): + script = """if 1: + import sys, os, time, threading + + # a thread, which waits for the main program to terminate + def joiningfunc(mainthread): + mainthread.join() + print('end of thread') + # stdout is fully buffered because not a tty, we have to flush + # before exit. + sys.stdout.flush() + \n""" + script + + rc, out, err = assert_python_ok("-c", script) + data = out.decode().replace('\r', '') + self.assertEqual(data, "end of main\nend of thread\n") + + def test_1_join_on_shutdown(self): + # The usual case: on exit, wait for a non-daemon thread + script = """if 1: + import os + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + time.sleep(0.1) + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_2_join_in_forked_process(self): + # Like the test above, but from a forked interpreter + script = """if 1: + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_3_join_in_forked_from_thread(self): + # Like the test above, but fork() was called from a worker thread + # In the forked process, the main Thread object must be marked as stopped. + + script = """if 1: + main_thread = threading.current_thread() + def worker(): + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(main_thread,)) + print('end of main') + t.start() + t.join() # Should not block: main_thread is already stopped + + w = threading.Thread(target=worker) + w.start() + """ + self._run_and_join(script) + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_4_daemon_threads(self): + # Check that a daemon thread cannot crash the interpreter on shutdown + # by manipulating internal structures that are being disposed of in + # the main thread. + script = """if True: + import os + import random + import sys + import time + import threading + + thread_has_run = set() + + def random_io(): + '''Loop for a while sleeping random tiny amounts and doing some I/O.''' + while True: + in_f = open(os.__file__, 'rb') + stuff = in_f.read(200) + null_f = open(os.devnull, 'wb') + null_f.write(stuff) + time.sleep(random.random() / 1995) + null_f.close() + in_f.close() + thread_has_run.add(threading.current_thread()) + + def main(): + count = 0 + for _ in range(40): + new_thread = threading.Thread(target=random_io) + new_thread.daemon = True + new_thread.start() + count += 1 + while len(thread_has_run) < count: + time.sleep(0.001) + # Trigger process shutdown + sys.exit(0) + + main() + """ + rc, out, err = assert_python_ok('-c', script) + self.assertFalse(err) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_reinit_tls_after_fork(self): + # Issue #13817: fork() would deadlock in a multithreaded program with + # the ad-hoc TLS implementation. + + def do_fork_and_wait(): + # just fork a child process and wait it + pid = os.fork() + if pid > 0: + os.waitpid(pid, 0) + else: + os._exit(0) + + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() + + for t in threads: + t.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_clear_threads_states_after_fork(self): + # Issue #17094: check that threads states are cleared after fork() + + # start a bunch of threads + threads = [] + for i in range(16): + t = threading.Thread(target=lambda : time.sleep(0.3)) + threads.append(t) + t.start() + + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(0) + else: + os._exit(1) + else: + _, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + for t in threads: + t.join() + + +class SubinterpThreadingTests(BaseTestCase): + + def test_threads_join(self): + # Non-daemon threads should be joined at subinterpreter shutdown + # (issue #18808) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + @cpython_only + def test_daemon_threads_fatal_error(self): + subinterp_code = r"""if 1: + import os + import threading + import time + + def f(): + # Make sure the daemon thread is still running when + # Py_EndInterpreter is called. + time.sleep(10) + threading.Thread(target=f, daemon=True).start() + """ + script = r"""if 1: + import _testcapi + + _testcapi.run_in_subinterp(%r) + """ % (subinterp_code,) + with test.support.SuppressCrashReport(): + rc, out, err = assert_python_failure("-c", script) + self.assertIn("Fatal Python error: Py_EndInterpreter: " + "not the last thread", err.decode()) + + +class ThreadingExceptionTests(BaseTestCase): + # A RuntimeError should be raised if Thread.start() is called + # multiple times. + def test_start_thread_again(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, thread.start) + + def test_joining_current_thread(self): + current_thread = threading.current_thread() + self.assertRaises(RuntimeError, current_thread.join); + + def test_joining_inactive_thread(self): + thread = threading.Thread() + self.assertRaises(RuntimeError, thread.join) + + def test_daemonize_active_thread(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + + def test_releasing_unacquired_lock(self): + lock = threading.Lock() + self.assertRaises(RuntimeError, lock.release) + + @unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(), + 'test macosx problem') + def test_recursion_limit(self): + # Issue 9670 + # test that excessive recursion within a non-main thread causes + # an exception rather than crashing the interpreter on platforms + # like Mac OS X or FreeBSD which have small default stack sizes + # for threads + script = """if True: + import threading + + def recurse(): + return recurse() + + def outer(): + try: + recurse() + except RecursionError: + pass + + w = threading.Thread(target=outer) + w.start() + w.join() + print('end of main thread') + """ + expected_output = "end of main thread\n" + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + data = stdout.decode().replace('\r', '') + self.assertEqual(p.returncode, 0, "Unexpected error: " + stderr.decode()) + self.assertEqual(data, expected_output) + + def test_print_exception(self): + script = r"""if True: + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + @requires_type_collecting + def test_print_exception_stderr_is_none_1(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + sys.stderr = None + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_2(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + sys.stderr = None + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + self.assertNotIn("Unhandled exception", err.decode()) + + def test_bare_raise_in_brand_new_thread(self): + def bare_raise(): + raise + + class Issue27558(threading.Thread): + exc = None + + def run(self): + try: + bare_raise() + except Exception as exc: + self.exc = exc + + thread = Issue27558() + thread.start() + thread.join() + self.assertIsNotNone(thread.exc) + self.assertIsInstance(thread.exc, RuntimeError) + +class TimerTests(BaseTestCase): + + def setUp(self): + BaseTestCase.setUp(self) + self.callback_args = [] + self.callback_event = threading.Event() + + def test_init_immutable_default_args(self): + # Issue 17435: constructor defaults were mutable objects, they could be + # mutated via the object attributes and affect other Timer objects. + timer1 = threading.Timer(0.01, self._callback_spy) + timer1.start() + self.callback_event.wait() + timer1.args.append("blah") + timer1.kwargs["foo"] = "bar" + self.callback_event.clear() + timer2 = threading.Timer(0.01, self._callback_spy) + timer2.start() + self.callback_event.wait() + self.assertEqual(len(self.callback_args), 2) + self.assertEqual(self.callback_args, [((), {}), ((), {})]) + + def _callback_spy(self, *args, **kwargs): + self.callback_args.append((args[:], kwargs.copy())) + self.callback_event.set() + +class LockTests(lock_tests.LockTests): + locktype = staticmethod(threading.Lock) + + @unittest.skip("not on gevent") + def test_locked_repr(self): + pass + + @unittest.skip("not on gevent") + def test_repr(self): + pass + +class PyRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._PyRLock) + +@unittest.skipIf(threading._CRLock is None, 'RLock not implemented in C') +class CRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._CRLock) + +class EventTests(lock_tests.EventTests): + eventtype = staticmethod(threading.Event) + + @unittest.skip("not on gevent") + def test_reset_internal_locks(self): + pass + +class ConditionAsRLockTests(lock_tests.RLockTests): + # Condition uses an RLock by default and exports its API. + locktype = staticmethod(threading.Condition) + +class ConditionTests(lock_tests.ConditionTests): + condtype = staticmethod(threading.Condition) + +class SemaphoreTests(lock_tests.SemaphoreTests): + semtype = staticmethod(threading.Semaphore) + +class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): + semtype = staticmethod(threading.BoundedSemaphore) + +class BarrierTests(lock_tests.BarrierTests): + barriertype = staticmethod(threading.Barrier) + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5/test_wsgiref.py b/src/greentest/3.5/test_wsgiref.py new file mode 100644 index 0000000..61a750c --- /dev/null +++ b/src/greentest/3.5/test_wsgiref.py @@ -0,0 +1,785 @@ +from unittest import mock +from test import support +from test.test_httpservers import NoLogRequestHandler +from unittest import TestCase +from wsgiref.util import setup_testing_defaults +from wsgiref.headers import Headers +from wsgiref.handlers import BaseHandler, BaseCGIHandler, SimpleHandler +from wsgiref import util +from wsgiref.validate import validator +from wsgiref.simple_server import WSGIServer, WSGIRequestHandler +from wsgiref.simple_server import make_server +from http.client import HTTPConnection +from io import StringIO, BytesIO, BufferedReader +from socketserver import BaseServer +from platform import python_implementation + +import os +import re +import signal +import sys +import unittest + + +class MockServer(WSGIServer): + """Non-socket HTTP server""" + + def __init__(self, server_address, RequestHandlerClass): + BaseServer.__init__(self, server_address, RequestHandlerClass) + self.server_bind() + + def server_bind(self): + host, port = self.server_address + self.server_name = host + self.server_port = port + self.setup_environ() + + +class MockHandler(WSGIRequestHandler): + """Non-socket HTTP handler""" + def setup(self): + self.connection = self.request + self.rfile, self.wfile = self.connection + + def finish(self): + pass + + +def hello_app(environ,start_response): + start_response("200 OK", [ + ('Content-Type','text/plain'), + ('Date','Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return [b"Hello, world!"] + + +def header_app(environ, start_response): + start_response("200 OK", [ + ('Content-Type', 'text/plain'), + ('Date', 'Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return [';'.join([ + environ['HTTP_X_TEST_HEADER'], environ['QUERY_STRING'], + environ['PATH_INFO'] + ]).encode('iso-8859-1')] + + +def run_amock(app=hello_app, data=b"GET / HTTP/1.0\n\n"): + server = make_server("", 80, app, MockServer, MockHandler) + inp = BufferedReader(BytesIO(data)) + out = BytesIO() + olderr = sys.stderr + err = sys.stderr = StringIO() + + try: + server.finish_request((inp, out), ("127.0.0.1",8888)) + finally: + sys.stderr = olderr + + return out.getvalue(), err.getvalue() + +def compare_generic_iter(make_it,match): + """Utility to compare a generic 2.1/2.2+ iterator with an iterable + + If running under Python 2.2+, this tests the iterator using iter()/next(), + as well as __getitem__. 'make_it' must be a function returning a fresh + iterator to be tested (since this may test the iterator twice).""" + + it = make_it() + n = 0 + for item in match: + if not it[n]==item: raise AssertionError + n+=1 + try: + it[n] + except IndexError: + pass + else: + raise AssertionError("Too many items from __getitem__",it) + + try: + iter, StopIteration + except NameError: + pass + else: + # Only test iter mode under 2.2+ + it = make_it() + if not iter(it) is it: raise AssertionError + for item in match: + if not next(it) == item: raise AssertionError + try: + next(it) + except StopIteration: + pass + else: + raise AssertionError("Too many items from .__next__()", it) + + +class IntegrationTests(TestCase): + + def check_hello(self, out, has_length=True): + pyver = (python_implementation() + "/" + + sys.version.split()[0]) + self.assertEqual(out, + ("HTTP/1.0 200 OK\r\n" + "Server: WSGIServer/0.2 " + pyver +"\r\n" + "Content-Type: text/plain\r\n" + "Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + + (has_length and "Content-Length: 13\r\n" or "") + + "\r\n" + "Hello, world!").encode("iso-8859-1") + ) + + def test_plain_hello(self): + out, err = run_amock() + self.check_hello(out) + + def test_environ(self): + request = ( + b"GET /p%61th/?query=test HTTP/1.0\n" + b"X-Test-Header: Python test \n" + b"X-Test-Header: Python test 2\n" + b"Content-Length: 0\n\n" + ) + out, err = run_amock(header_app, request) + self.assertEqual( + out.splitlines()[-1], + b"Python test,Python test 2;query=test;/path/" + ) + + def test_request_length(self): + out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") + self.assertEqual(out.splitlines()[0], + b"HTTP/1.0 414 Request-URI Too Long") + + def test_validated_hello(self): + out, err = run_amock(validator(hello_app)) + # the middleware doesn't support len(), so content-length isn't there + self.check_hello(out, has_length=False) + + def test_simple_validation_error(self): + def bad_app(environ,start_response): + start_response("200 OK", ('Content-Type','text/plain')) + return ["Hello, world!"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], + "AssertionError: Headers (('Content-Type', 'text/plain')) must" + " be of type list: " + ) + + def test_status_validation_errors(self): + def create_bad_app(status): + def bad_app(environ, start_response): + start_response(status, [("Content-Type", "text/plain; charset=utf-8")]) + return [b"Hello, world!"] + return bad_app + + tests = [ + ('200', 'AssertionError: Status must be at least 4 characters'), + ('20X OK', 'AssertionError: Status message must begin w/3-digit code'), + ('200OK', 'AssertionError: Status message must have a space after code'), + ] + + for status, exc_message in tests: + with self.subTest(status=status): + out, err = run_amock(create_bad_app(status)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual(err.splitlines()[-2], exc_message) + + def test_wsgi_input(self): + def bad_app(e,s): + e["wsgi.input"].read() + s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) + return [b"data"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], "AssertionError" + ) + + def test_bytes_validation(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain; charset=utf-8"), + ("Date", "Wed, 24 Dec 2008 13:29:32 GMT"), + ]) + return [b"data"] + out, err = run_amock(validator(app)) + self.assertTrue(err.endswith('"GET / HTTP/1.0" 200 4\n')) + ver = sys.version.split()[0].encode('ascii') + py = python_implementation().encode('ascii') + pyver = py + b"/" + ver + self.assertEqual( + b"HTTP/1.0 200 OK\r\n" + b"Server: WSGIServer/0.2 "+ pyver + b"\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"Date: Wed, 24 Dec 2008 13:29:32 GMT\r\n" + b"\r\n" + b"data", + out) + + def test_cp1252_url(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain"), + ("Date", "Wed, 24 Dec 2008 13:29:32 GMT"), + ]) + # PEP3333 says environ variables are decoded as latin1. + # Encode as latin1 to get original bytes + return [e["PATH_INFO"].encode("latin1")] + + out, err = run_amock( + validator(app), data=b"GET /\x80%80 HTTP/1.0") + self.assertEqual( + [ + b"HTTP/1.0 200 OK", + mock.ANY, + b"Content-Type: text/plain", + b"Date: Wed, 24 Dec 2008 13:29:32 GMT", + b"", + b"/\x80\x80", + ], + out.splitlines()) + + def test_interrupted_write(self): + # BaseHandler._write() and _flush() have to write all data, even if + # it takes multiple send() calls. Test this by interrupting a send() + # call with a Unix signal. + threading = support.import_module("threading") + pthread_kill = support.get_attribute(signal, "pthread_kill") + + def app(environ, start_response): + start_response("200 OK", []) + return [bytes(support.SOCK_MAX_SIZE)] + + class WsgiHandler(NoLogRequestHandler, WSGIRequestHandler): + pass + + server = make_server(support.HOST, 0, app, handler_class=WsgiHandler) + self.addCleanup(server.server_close) + interrupted = threading.Event() + + def signal_handler(signum, frame): + interrupted.set() + + original = signal.signal(signal.SIGUSR1, signal_handler) + self.addCleanup(signal.signal, signal.SIGUSR1, original) + received = None + main_thread = threading.get_ident() + + def run_client(): + http = HTTPConnection(*server.server_address) + http.request("GET", "/") + with http.getresponse() as response: + response.read(100) + # The main thread should now be blocking in a send() system + # call. But in theory, it could get interrupted by other + # signals, and then retried. So keep sending the signal in a + # loop, in case an earlier signal happens to be delivered at + # an inconvenient moment. + while True: + pthread_kill(main_thread, signal.SIGUSR1) + if interrupted.wait(timeout=float(1)): + break + nonlocal received + received = len(response.read()) + http.close() + + background = threading.Thread(target=run_client) + background.start() + server.handle_request() + background.join() + self.assertEqual(received, support.SOCK_MAX_SIZE - 100) + + +class UtilityTests(TestCase): + + def checkShift(self,sn_in,pi_in,part,sn_out,pi_out): + env = {'SCRIPT_NAME':sn_in,'PATH_INFO':pi_in} + util.setup_testing_defaults(env) + self.assertEqual(util.shift_path_info(env),part) + self.assertEqual(env['PATH_INFO'],pi_out) + self.assertEqual(env['SCRIPT_NAME'],sn_out) + return env + + def checkDefault(self, key, value, alt=None): + # Check defaulting when empty + env = {} + util.setup_testing_defaults(env) + if isinstance(value, StringIO): + self.assertIsInstance(env[key], StringIO) + elif isinstance(value,BytesIO): + self.assertIsInstance(env[key],BytesIO) + else: + self.assertEqual(env[key], value) + + # Check existing value + env = {key:alt} + util.setup_testing_defaults(env) + self.assertIs(env[key], alt) + + def checkCrossDefault(self,key,value,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(kw[key],value) + + def checkAppURI(self,uri,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.application_uri(kw),uri) + + def checkReqURI(self,uri,query=1,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.request_uri(kw,query),uri) + + def checkFW(self,text,size,match): + + def make_it(text=text,size=size): + return util.FileWrapper(StringIO(text),size) + + compare_generic_iter(make_it,match) + + it = make_it() + self.assertFalse(it.filelike.closed) + + for item in it: + pass + + self.assertFalse(it.filelike.closed) + + it.close() + self.assertTrue(it.filelike.closed) + + def testSimpleShifts(self): + self.checkShift('','/', '', '/', '') + self.checkShift('','/x', 'x', '/x', '') + self.checkShift('/','', None, '/', '') + self.checkShift('/a','/x/y', 'x', '/a/x', '/y') + self.checkShift('/a','/x/', 'x', '/a/x', '/') + + def testNormalizedShifts(self): + self.checkShift('/a/b', '/../y', '..', '/a', '/y') + self.checkShift('', '/../y', '..', '', '/y') + self.checkShift('/a/b', '//y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '//y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '/./y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '/./y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '///./..//y/.//', '..', '/a', '/y/') + self.checkShift('/a/b', '///', '', '/a/b/', '') + self.checkShift('/a/b', '/.//', '', '/a/b/', '') + self.checkShift('/a/b', '/x//', 'x', '/a/b/x', '/') + self.checkShift('/a/b', '/.', None, '/a/b', '') + + def testDefaults(self): + for key, value in [ + ('SERVER_NAME','127.0.0.1'), + ('SERVER_PORT', '80'), + ('SERVER_PROTOCOL','HTTP/1.0'), + ('HTTP_HOST','127.0.0.1'), + ('REQUEST_METHOD','GET'), + ('SCRIPT_NAME',''), + ('PATH_INFO','/'), + ('wsgi.version', (1,0)), + ('wsgi.run_once', 0), + ('wsgi.multithread', 0), + ('wsgi.multiprocess', 0), + ('wsgi.input', BytesIO()), + ('wsgi.errors', StringIO()), + ('wsgi.url_scheme','http'), + ]: + self.checkDefault(key,value) + + def testCrossDefaults(self): + self.checkCrossDefault('HTTP_HOST',"foo.bar",SERVER_NAME="foo.bar") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="on") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="1") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="yes") + self.checkCrossDefault('wsgi.url_scheme',"http",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"80",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"443",HTTPS="on") + + def testGuessScheme(self): + self.assertEqual(util.guess_scheme({}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"foo"}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"on"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"yes"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"1"}), "https") + + def testAppURIs(self): + self.checkAppURI("http://127.0.0.1/") + self.checkAppURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkAppURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkAppURI("http://spam.example.com:2071/", + HTTP_HOST="spam.example.com:2071", SERVER_PORT="2071") + self.checkAppURI("http://spam.example.com/", + SERVER_NAME="spam.example.com") + self.checkAppURI("http://127.0.0.1/", + HTTP_HOST="127.0.0.1", SERVER_NAME="spam.example.com") + self.checkAppURI("https://127.0.0.1/", HTTPS="on") + self.checkAppURI("http://127.0.0.1:8000/", SERVER_PORT="8000", + HTTP_HOST=None) + + def testReqURIs(self): + self.checkReqURI("http://127.0.0.1/") + self.checkReqURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkReqURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam", + SCRIPT_NAME="/spammity", PATH_INFO="/spam") + self.checkReqURI("http://127.0.0.1/spammity/sp%E4m", + SCRIPT_NAME="/spammity", PATH_INFO="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam;ham", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;ham") + self.checkReqURI("http://127.0.0.1/spammity/spam;cookie=1234,5678", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;cookie=1234,5678") + self.checkReqURI("http://127.0.0.1/spammity/spam?say=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam?s%E4y=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="s%E4y=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam", 0, + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + + def testFileWrapper(self): + self.checkFW("xyz"*50, 120, ["xyz"*40,"xyz"*10]) + + def testHopByHop(self): + for hop in ( + "Connection Keep-Alive Proxy-Authenticate Proxy-Authorization " + "TE Trailers Transfer-Encoding Upgrade" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertTrue(util.is_hop_by_hop(alt)) + + # Not comprehensive, just a few random header names + for hop in ( + "Accept Cache-Control Date Pragma Trailer Via Warning" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertFalse(util.is_hop_by_hop(alt)) + +class HeaderTests(TestCase): + + def testMappingInterface(self): + test = [('x','y')] + self.assertEqual(len(Headers()), 0) + self.assertEqual(len(Headers([])),0) + self.assertEqual(len(Headers(test[:])),1) + self.assertEqual(Headers(test[:]).keys(), ['x']) + self.assertEqual(Headers(test[:]).values(), ['y']) + self.assertEqual(Headers(test[:]).items(), test) + self.assertIsNot(Headers(test).items(), test) # must be copy! + + h = Headers() + del h['foo'] # should not raise an error + + h['Foo'] = 'bar' + for m in h.__contains__, h.get, h.get_all, h.__getitem__: + self.assertTrue(m('foo')) + self.assertTrue(m('Foo')) + self.assertTrue(m('FOO')) + self.assertFalse(m('bar')) + + self.assertEqual(h['foo'],'bar') + h['foo'] = 'baz' + self.assertEqual(h['FOO'],'baz') + self.assertEqual(h.get_all('foo'),['baz']) + + self.assertEqual(h.get("foo","whee"), "baz") + self.assertEqual(h.get("zoo","whee"), "whee") + self.assertEqual(h.setdefault("foo","whee"), "baz") + self.assertEqual(h.setdefault("zoo","whee"), "whee") + self.assertEqual(h["foo"],"baz") + self.assertEqual(h["zoo"],"whee") + + def testRequireList(self): + self.assertRaises(TypeError, Headers, "foo") + + def testExtras(self): + h = Headers() + self.assertEqual(str(h),'\r\n') + + h.add_header('foo','bar',baz="spam") + self.assertEqual(h['foo'], 'bar; baz="spam"') + self.assertEqual(str(h),'foo: bar; baz="spam"\r\n\r\n') + + h.add_header('Foo','bar',cheese=None) + self.assertEqual(h.get_all('foo'), + ['bar; baz="spam"', 'bar; cheese']) + + self.assertEqual(str(h), + 'foo: bar; baz="spam"\r\n' + 'Foo: bar; cheese\r\n' + '\r\n' + ) + +class ErrorHandler(BaseCGIHandler): + """Simple handler subclass for testing BaseHandler""" + + # BaseHandler records the OS environment at import time, but envvars + # might have been changed later by other tests, which trips up + # HandlerTests.testEnviron(). + os_environ = dict(os.environ.items()) + + def __init__(self,**kw): + setup_testing_defaults(kw) + BaseCGIHandler.__init__( + self, BytesIO(), BytesIO(), StringIO(), kw, + multithread=True, multiprocess=True + ) + +class TestHandler(ErrorHandler): + """Simple handler subclass for testing BaseHandler, w/error passthru""" + + def handle_error(self): + raise # for testing, we want to see what's happening + + +class HandlerTests(TestCase): + + def checkEnvironAttrs(self, handler): + env = handler.environ + for attr in [ + 'version','multithread','multiprocess','run_once','file_wrapper' + ]: + if attr=='file_wrapper' and handler.wsgi_file_wrapper is None: + continue + self.assertEqual(getattr(handler,'wsgi_'+attr),env['wsgi.'+attr]) + + def checkOSEnviron(self,handler): + empty = {}; setup_testing_defaults(empty) + env = handler.environ + from os import environ + for k,v in environ.items(): + if k not in empty: + self.assertEqual(env[k],v) + for k,v in empty.items(): + self.assertIn(k, env) + + def testEnviron(self): + h = TestHandler(X="Y") + h.setup_environ() + self.checkEnvironAttrs(h) + self.checkOSEnviron(h) + self.assertEqual(h.environ["X"],"Y") + + def testCGIEnviron(self): + h = BaseCGIHandler(None,None,None,{}) + h.setup_environ() + for key in 'wsgi.url_scheme', 'wsgi.input', 'wsgi.errors': + self.assertIn(key, h.environ) + + def testScheme(self): + h=TestHandler(HTTPS="on"); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'https') + h=TestHandler(); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'http') + + def testAbstractMethods(self): + h = BaseHandler() + for name in [ + '_flush','get_stdin','get_stderr','add_cgi_vars' + ]: + self.assertRaises(NotImplementedError, getattr(h,name)) + self.assertRaises(NotImplementedError, h._write, "test") + + def testContentLength(self): + # Demo one reason iteration is better than write()... ;) + + def trivial_app1(e,s): + s('200 OK',[]) + return [e['wsgi.url_scheme'].encode('iso-8859-1')] + + def trivial_app2(e,s): + s('200 OK',[])(e['wsgi.url_scheme'].encode('iso-8859-1')) + return [] + + def trivial_app3(e,s): + s('200 OK',[]) + return ['\u0442\u0435\u0441\u0442'.encode("utf-8")] + + def trivial_app4(e,s): + # Simulate a response to a HEAD request + s('200 OK',[('Content-Length', '12345')]) + return [] + + h = TestHandler() + h.run(trivial_app1) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "Content-Length: 4\r\n" + "\r\n" + "http").encode("iso-8859-1")) + + h = TestHandler() + h.run(trivial_app2) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "\r\n" + "http").encode("iso-8859-1")) + + h = TestHandler() + h.run(trivial_app3) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 8\r\n' + b'\r\n' + b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82') + + h = TestHandler() + h.run(trivial_app4) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 12345\r\n' + b'\r\n') + + def testBasicErrorOutput(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + def error_app(e,s): + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(non_error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n").encode("iso-8859-1")) + self.assertEqual(h.stderr.getvalue(),"") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: %s\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: %d\r\n" + "\r\n" % (h.error_status,len(h.error_body))).encode('iso-8859-1') + + h.error_body) + + self.assertIn("AssertionError", h.stderr.getvalue()) + + def testErrorAfterOutput(self): + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "\r\n".encode("iso-8859-1")+MSG)) + self.assertIn("AssertionError", h.stderr.getvalue()) + + def testHeaderFormats(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + stdpat = ( + r"HTTP/%s 200 OK\r\n" + r"Date: \w{3}, [ 0123]\d \w{3} \d{4} \d\d:\d\d:\d\d GMT\r\n" + r"%s" r"Content-Length: 0\r\n" r"\r\n" + ) + shortpat = ( + "Status: 200 OK\r\n" "Content-Length: 0\r\n" "\r\n" + ).encode("iso-8859-1") + + for ssw in "FooBar/1.0", None: + sw = ssw and "Server: %s\r\n" % ssw or "" + + for version in "1.0", "1.1": + for proto in "HTTP/0.9", "HTTP/1.0", "HTTP/1.1": + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = False + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + self.assertEqual(shortpat,h.stdout.getvalue()) + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = True + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + if proto=="HTTP/0.9": + self.assertEqual(h.stdout.getvalue(),b"") + else: + self.assertTrue( + re.match((stdpat%(version,sw)).encode("iso-8859-1"), + h.stdout.getvalue()), + ((stdpat%(version,sw)).encode("iso-8859-1"), + h.stdout.getvalue()) + ) + + def testBytesData(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain; charset=utf-8"), + ]) + return [b"data"] + + h = TestHandler() + h.run(app) + self.assertEqual(b"Status: 200 OK\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"Content-Length: 4\r\n" + b"\r\n" + b"data", + h.stdout.getvalue()) + + def testCloseOnError(self): + side_effects = {'close_called': False} + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + class CrashyIterable(object): + def __iter__(self): + while True: + yield b'blah' + raise AssertionError("This should be caught by handler") + def close(self): + side_effects['close_called'] = True + return CrashyIterable() + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(side_effects['close_called'], True) + + def testPartialWrite(self): + written = bytearray() + + class PartialWriter: + def write(self, b): + partial = b[:7] + written.extend(partial) + return len(partial) + + def flush(self): + pass + + environ = {"SERVER_PROTOCOL": "HTTP/1.0"} + h = SimpleHandler(BytesIO(), PartialWriter(), sys.stderr, environ) + msg = "should not do partial writes" + with self.assertWarnsRegex(DeprecationWarning, msg): + h.run(hello_app) + self.assertEqual(b"HTTP/1.0 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + b"Content-Length: 13\r\n" + b"\r\n" + b"Hello, world!", + written) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5/version b/src/greentest/3.5/version new file mode 100644 index 0000000..444877d --- /dev/null +++ b/src/greentest/3.5/version @@ -0,0 +1 @@ +3.5.3 diff --git a/src/greentest/3.5/wrongcert.pem b/src/greentest/3.5/wrongcert.pem new file mode 100644 index 0000000..5f92f9b --- /dev/null +++ b/src/greentest/3.5/wrongcert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH +FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T +f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB +AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq +1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW +7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg +SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe +19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg +ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666 +lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs +ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv +frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk +2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6 ++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK +24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G +A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9 +OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ +fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E +usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+ +43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw +eSHj5jpC8iZKjCHBn+mAi4cQ514= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/allsans.pem b/src/greentest/3.5pypy/allsans.pem new file mode 100644 index 0000000..3ee4f59 --- /dev/null +++ b/src/greentest/3.5pypy/allsans.pem @@ -0,0 +1,37 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOoy7/QOtTjQ0niE +6uDcTwtkC0R2Tvy1AjVnXohCntZfdzbTGDoYTgXSOLsP8A697jUiJ8VCePGH50xG +Z4DKnAF3a9O3a9nr2pLXb0iY3XOMv+YEBii7CfI+3oxFYgCl0sMgHzDD2ZTVYAsm +DWgLUVsE2gHEccRwrM2tPf2EgR+FAgMBAAECgYEA3qyfyYVSeTrTYxO93x6ZaVMu +A2IZp9zSxMQL9bKiI2GRj+cV2ebSCGbg2btFnD6qBor7FWsmYz+8g6FNN/9sY4az +61rMqMtQvLBe+7L8w70FeTze4qQ4Y1oQri0qD6tBWhDVlpnbI5Py9bkZKD67yVUk +elcEA/5x4PrYXkuqsAECQQD80NjT0mDvaY0JOOaQFSEpMv6QiUA8GGX8Xli7IoKb +tAolPG8rQBa+qSpcWfDMTrWw/aWHuMEEQoP/bVDH9W4FAkEA7SYQbBAKnojZ5A3G +kOHdV7aeivRQxQk/JN8Fb8oKB9Csvpv/BsuGxPKXHdhFa6CBTTsNRtHQw/szPo4l +xMIjgQJAPoMxqibR+0EBM6+TKzteSL6oPXsCnBl4Vk/J5vPgkbmR7KUl4+7j8N8J +b2554TrxKEN/w7CGYZRE6UrRd7ATNQJAWD7Yz41sli+wfPdPU2xo1BHljyl4wMk/ +EPZYbI/PCbdyAH/F935WyQTIjNeEhZc1Zkq6FwdOWw8ns3hrv3rKgQJAHXv1BqUa +czGPIFxX2TNoqtcl6/En4vrxVB1wzsfzkkDAg98kBl7qsF+S3qujSzKikjeaVbI2 +/CyWR2P3yLtOmA== +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDcjCCAtugAwIBAgIJAN5dc9TOWjB7MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTYwODA1 +MTAyMTExWhcNMjYwODAzMTAyMTExWjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO +Q2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0 +aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQDqMu/0DrU40NJ4hOrg3E8LZAtEdk78tQI1Z16IQp7WX3c20xg6GE4F0ji7D/AO +ve41IifFQnjxh+dMRmeAypwBd2vTt2vZ69qS129ImN1zjL/mBAYouwnyPt6MRWIA +pdLDIB8ww9mU1WALJg1oC1FbBNoBxHHEcKzNrT39hIEfhQIDAQABo4IBODCCATQw +ggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBp +ZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMC +AQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUu +b3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMw +IQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGly +bmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAA +AAAAAAAAAAAAAAAAAYgEKgMEBTANBgkqhkiG9w0BAQsFAAOBgQAy16h+F+nOmeiT +VWR0fc8F/j6FcadbLseAUaogcC15OGxCl4UYpLV88HBkABOoGCpP155qwWTwOrdG +iYPGJSusf1OnJEbvzFejZf6u078bPd9/ZL4VWLjv+FPGkjd+N+/OaqMvgj8Lu99f +3Y/C4S7YbHxxwff6C6l2Xli+q6gnuQ== +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/badcert.pem b/src/greentest/3.5pypy/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/greentest/3.5pypy/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/badkey.pem b/src/greentest/3.5pypy/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/greentest/3.5pypy/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/capath/0e4015b9.0 b/src/greentest/3.5pypy/capath/0e4015b9.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.5pypy/capath/0e4015b9.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/capath/4e1295a3.0 b/src/greentest/3.5pypy/capath/4e1295a3.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.5pypy/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/capath/5ed36f99.0 b/src/greentest/3.5pypy/capath/5ed36f99.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.5pypy/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/capath/6e88d7b8.0 b/src/greentest/3.5pypy/capath/6e88d7b8.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.5pypy/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/capath/99d0fa06.0 b/src/greentest/3.5pypy/capath/99d0fa06.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.5pypy/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/capath/ce7b8643.0 b/src/greentest/3.5pypy/capath/ce7b8643.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.5pypy/capath/ce7b8643.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/dh1024.pem b/src/greentest/3.5pypy/dh1024.pem new file mode 100644 index 0000000..a391176 --- /dev/null +++ b/src/greentest/3.5pypy/dh1024.pem @@ -0,0 +1,7 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAIbzw1s9CT8SV5yv6L7esdAdZYZjPi3qWFs61CYTFFQnf2s/d09NYaJt +rrvJhIzWavqnue71qXCf83/J3nz3FEwUU/L0mGyheVbsSHiI64wUo3u50wK5Igo0 +RNs/LD0irs7m0icZ//hijafTU+JOBiuA8zMI+oZfU7BGuc9XrUprAgEC +-----END DH PARAMETERS----- + +Generated with: openssl dhparam -out dh1024.pem 1024 diff --git a/src/greentest/3.5pypy/keycert.passwd.pem b/src/greentest/3.5pypy/keycert.passwd.pem new file mode 100644 index 0000000..e905748 --- /dev/null +++ b/src/greentest/3.5pypy/keycert.passwd.pem @@ -0,0 +1,33 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/keycert.pem b/src/greentest/3.5pypy/keycert.pem new file mode 100644 index 0000000..64318aa --- /dev/null +++ b/src/greentest/3.5pypy/keycert.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/keycert2.pem b/src/greentest/3.5pypy/keycert2.pem new file mode 100644 index 0000000..e8a9e08 --- /dev/null +++ b/src/greentest/3.5pypy/keycert2.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAJnsJZVrppL+W5I9 +zGQrrawWwE5QJpBK9nWw17mXrZ03R1cD9BamLGivVISbPlRlAVnZBEyh1ATpsB7d +CUQ+WHEvALquvx4+Yw5l+fXeiYRjrLRBYZuVy8yNtXzU3iWcGObcYRkUdiXdOyP7 +sLF2YZHRvQZpzgDBKkrraeQ81w21AgMBAAECgYBEm7n07FMHWlE+0kT0sXNsLYfy +YE+QKZnJw9WkaDN+zFEEPELkhZVt5BjsMraJr6v2fIEqF0gGGJPkbenffVq2B5dC +lWUOxvJHufMK4sM3Cp6s/gOp3LP+QkzVnvJSfAyZU6l+4PGX5pLdUsXYjPxgzjzL +S36tF7/2Uv1WePyLUQJBAMsPhYzUXOPRgmbhcJiqi9A9c3GO8kvSDYTCKt3VMnqz +HBn6MQ4VQasCD1F+7jWTI0FU/3vdw8non/Fj8hhYqZcCQQDCDRdvmZqDiZnpMqDq +L6ZSrLTVtMvZXZbgwForaAD9uHj51TME7+eYT7EG2YCgJTXJ4YvRJEnPNyskwdKt +vTSTAkEAtaaN/vyemEJ82BIGStwONNw0ILsSr5cZ9tBHzqiA/tipY+e36HRFiXhP +QcU9zXlxyWkDH8iz9DSAmE2jbfoqwwJANlMJ65E543cjIlitGcKLMnvtCCLcKpb7 +xSG0XJB6Lo11OKPJ66jp0gcFTSCY1Lx2CXVd+gfJrfwI1Pp562+bhwJBAJ9IfDPU +R8OpO9v1SGd8x33Owm7uXOpB9d63/T70AD1QOXjKUC4eXYbt0WWfWuny/RNPRuyh +w7DXSfUF+kPKolU= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICXTCCAcagAwIBAgIJAIO3upAG445fMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTAeFw0x +MDEwMDkxNTAxMDBaFw0yMDEwMDYxNTAxMDBaMGIxCzAJBgNVBAYTAlhZMRcwFQYD +VQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZv +dW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEAmewllWumkv5bkj3MZCutrBbATlAmkEr2dbDXuZetnTdHVwP0 +FqYsaK9UhJs+VGUBWdkETKHUBOmwHt0JRD5YcS8Auq6/Hj5jDmX59d6JhGOstEFh +m5XLzI21fNTeJZwY5txhGRR2Jd07I/uwsXZhkdG9BmnOAMEqSutp5DzXDbUCAwEA +AaMbMBkwFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBBQUAA4GB +AH+iMClLLGSaKWgwXsmdVo4FhTZZHo8Uprrtg3N9FxEeE50btpDVQysgRt5ias3K +m+bME9zbKwvbVWD5zZdjus4pDgzwF/iHyccL8JyYhxOvS/9zmvAtFXj/APIIbZFp +IT75d9f88ScIGEtknZQejnrdhB64tYki/EqluiuKBqKD +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/keycert3.pem b/src/greentest/3.5pypy/keycert3.pem new file mode 100644 index 0000000..5bfa62c --- /dev/null +++ b/src/greentest/3.5pypy/keycert3.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP +jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM +9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ +aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe +yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j +y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ +AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW +5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL +9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 +1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT +DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh +1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m +JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 +RnJdHOMXWem7/w== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: + 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: + c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: + 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: + f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: + 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: + f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: + af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: + 21:82:a5:3c:88:e5:be:1b:b1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: + e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: + f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: + e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: + d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: + 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: + ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: + 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: + 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: + 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: + 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: + f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: + 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: + a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: + fc:a9:94:71 +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C +tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola +N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 +TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR +iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG +xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo +5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv +mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF +YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh +2EJ36/yplHE= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/keycert4.pem b/src/greentest/3.5pypy/keycert4.pem new file mode 100644 index 0000000..53355c8 --- /dev/null +++ b/src/greentest/3.5pypy/keycert4.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv +L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2 +NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1 +L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L +pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de +R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9 +myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT +drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS +Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx +i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK +Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu +JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN ++/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/ +e83Gq6ffLVfKNQ== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443282 (0xb09264b1f2da21d2) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14: + 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9: + cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a: + b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76: + 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7: + 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a: + d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79: + 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd: + 81:7e:bd:1b:ae:0b:5d:c6:39 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0: + 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f: + 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56: + 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15: + 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1: + 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb: + 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e: + e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18: + d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1: + af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53: + 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68: + 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da: + 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92: + 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa: + 49:12:1e:ce +-----BEGIN CERTIFICATE----- +MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z +dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU +aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0 +ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ +hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v +xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338 +Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP +XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0 +UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz +aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb +oF+6ufu6+kkSHs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/lock_tests.py b/src/greentest/3.5pypy/lock_tests.py new file mode 100644 index 0000000..a64aa18 --- /dev/null +++ b/src/greentest/3.5pypy/lock_tests.py @@ -0,0 +1,915 @@ +""" +Various tests for synchronization primitives. +""" + +import sys +import time +from _thread import start_new_thread, TIMEOUT_MAX +import threading +import unittest + +from test import support + + +def _wait(): + # A crude wait/yield function not relying on synchronization primitives. + time.sleep(0.01) + +class Bunch(object): + """ + A bunch of threads. + """ + def __init__(self, f, n, wait_before_exit=False): + """ + Construct a bunch of `n` threads running the same function `f`. + If `wait_before_exit` is True, the threads won't terminate until + do_finish() is called. + """ + self.f = f + self.n = n + self.started = [] + self.finished = [] + self._can_exit = not wait_before_exit + def task(): + tid = threading.get_ident() + self.started.append(tid) + try: + f() + finally: + self.finished.append(tid) + while not self._can_exit: + _wait() + try: + for i in range(n): + start_new_thread(task, ()) + except: + self._can_exit = True + raise + + def wait_for_started(self): + while len(self.started) < self.n: + _wait() + + def wait_for_finished(self): + while len(self.finished) < self.n: + _wait() + + def do_finish(self): + self._can_exit = True + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = support.threading_setup() + + def tearDown(self): + support.threading_cleanup(*self._threads) + support.reap_children() + + def assertTimeout(self, actual, expected): + # The waiting and/or time.time() can be imprecise, which + # is why comparing to the expected value would sometimes fail + # (especially under Windows). + self.assertGreaterEqual(actual, expected * 0.6) + # Test nothing insane happened + self.assertLess(actual, expected * 10.0) + + +class BaseLockTests(BaseTestCase): + """ + Tests for both recursive and non-recursive locks. + """ + + def test_constructor(self): + lock = self.locktype() + del lock + + def test_repr(self): + lock = self.locktype() + self.assertRegex(repr(lock), "") + del lock + + def test_locked_repr(self): + lock = self.locktype() + lock.acquire() + self.assertRegex(repr(lock), "") + del lock + + def test_acquire_destroy(self): + lock = self.locktype() + lock.acquire() + del lock + + def test_acquire_release(self): + lock = self.locktype() + lock.acquire() + lock.release() + del lock + + def test_try_acquire(self): + lock = self.locktype() + self.assertTrue(lock.acquire(False)) + lock.release() + + def test_try_acquire_contended(self): + lock = self.locktype() + lock.acquire() + result = [] + def f(): + result.append(lock.acquire(False)) + Bunch(f, 1).wait_for_finished() + self.assertFalse(result[0]) + lock.release() + + def test_acquire_contended(self): + lock = self.locktype() + lock.acquire() + N = 5 + def f(): + lock.acquire() + lock.release() + + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(len(b.finished), 0) + lock.release() + b.wait_for_finished() + self.assertEqual(len(b.finished), N) + + def test_with(self): + lock = self.locktype() + def f(): + lock.acquire() + lock.release() + def _with(err=None): + with lock: + if err is not None: + raise err + _with() + # Check the lock is unacquired + Bunch(f, 1).wait_for_finished() + self.assertRaises(TypeError, _with, TypeError) + # Check the lock is unacquired + Bunch(f, 1).wait_for_finished() + + def test_thread_leak(self): + # The lock shouldn't leak a Thread instance when used from a foreign + # (non-threading) thread. + lock = self.locktype() + def f(): + lock.acquire() + lock.release() + n = len(threading.enumerate()) + # We run many threads in the hope that existing threads ids won't + # be recycled. + Bunch(f, 15).wait_for_finished() + if len(threading.enumerate()) != n: + # There is a small window during which a Thread instance's + # target function has finished running, but the Thread is still + # alive and registered. Avoid spurious failures by waiting a + # bit more (seen on a buildbot). + time.sleep(0.4) + self.assertEqual(n, len(threading.enumerate())) + + def test_timeout(self): + lock = self.locktype() + # Can't set timeout if not blocking + self.assertRaises(ValueError, lock.acquire, 0, 1) + # Invalid timeout values + self.assertRaises(ValueError, lock.acquire, timeout=-100) + self.assertRaises(OverflowError, lock.acquire, timeout=1e100) + self.assertRaises(OverflowError, lock.acquire, timeout=TIMEOUT_MAX + 1) + # TIMEOUT_MAX is ok + lock.acquire(timeout=TIMEOUT_MAX) + lock.release() + t1 = time.time() + self.assertTrue(lock.acquire(timeout=5)) + t2 = time.time() + # Just a sanity test that it didn't actually wait for the timeout. + self.assertLess(t2 - t1, 5) + results = [] + def f(): + t1 = time.time() + results.append(lock.acquire(timeout=0.5)) + t2 = time.time() + results.append(t2 - t1) + Bunch(f, 1).wait_for_finished() + self.assertFalse(results[0]) + self.assertTimeout(results[1], 0.5) + + +class LockTests(BaseLockTests): + """ + Tests for non-recursive, weak locks + (which can be acquired and released from different threads). + """ + def test_reacquire(self): + # Lock needs to be released before re-acquiring. + lock = self.locktype() + phase = [] + def f(): + lock.acquire() + phase.append(None) + lock.acquire() + phase.append(None) + start_new_thread(f, ()) + while len(phase) == 0: + _wait() + _wait() + self.assertEqual(len(phase), 1) + lock.release() + while len(phase) == 1: + _wait() + self.assertEqual(len(phase), 2) + + def test_different_thread(self): + # Lock can be released from a different thread. + lock = self.locktype() + lock.acquire() + def f(): + lock.release() + b = Bunch(f, 1) + b.wait_for_finished() + lock.acquire() + lock.release() + + def test_state_after_timeout(self): + # Issue #11618: check that lock is in a proper state after a + # (non-zero) timeout. + lock = self.locktype() + lock.acquire() + self.assertFalse(lock.acquire(timeout=0.01)) + lock.release() + self.assertFalse(lock.locked()) + self.assertTrue(lock.acquire(blocking=False)) + + +class RLockTests(BaseLockTests): + """ + Tests for recursive locks. + """ + def test_reacquire(self): + lock = self.locktype() + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + + def test_release_unacquired(self): + # Cannot release an unacquired lock + lock = self.locktype() + self.assertRaises(RuntimeError, lock.release) + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + self.assertRaises(RuntimeError, lock.release) + + def test_release_save_unacquired(self): + # Cannot _release_save an unacquired lock + lock = self.locktype() + self.assertRaises(RuntimeError, lock._release_save) + lock.acquire() + lock.acquire() + lock.release() + lock.acquire() + lock.release() + lock.release() + self.assertRaises(RuntimeError, lock._release_save) + + def test_different_thread(self): + # Cannot release from a different thread + lock = self.locktype() + def f(): + lock.acquire() + b = Bunch(f, 1, True) + try: + self.assertRaises(RuntimeError, lock.release) + finally: + b.do_finish() + + def test__is_owned(self): + lock = self.locktype() + self.assertFalse(lock._is_owned()) + lock.acquire() + self.assertTrue(lock._is_owned()) + lock.acquire() + self.assertTrue(lock._is_owned()) + result = [] + def f(): + result.append(lock._is_owned()) + Bunch(f, 1).wait_for_finished() + self.assertFalse(result[0]) + lock.release() + self.assertTrue(lock._is_owned()) + lock.release() + self.assertFalse(lock._is_owned()) + + +class EventTests(BaseTestCase): + """ + Tests for Event objects. + """ + + def test_is_set(self): + evt = self.eventtype() + self.assertFalse(evt.is_set()) + evt.set() + self.assertTrue(evt.is_set()) + evt.set() + self.assertTrue(evt.is_set()) + evt.clear() + self.assertFalse(evt.is_set()) + evt.clear() + self.assertFalse(evt.is_set()) + + def _check_notify(self, evt): + # All threads get notified + N = 5 + results1 = [] + results2 = [] + def f(): + results1.append(evt.wait()) + results2.append(evt.wait()) + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(len(results1), 0) + evt.set() + b.wait_for_finished() + self.assertEqual(results1, [True] * N) + self.assertEqual(results2, [True] * N) + + def test_notify(self): + evt = self.eventtype() + self._check_notify(evt) + # Another time, after an explicit clear() + evt.set() + evt.clear() + self._check_notify(evt) + + def test_timeout(self): + evt = self.eventtype() + results1 = [] + results2 = [] + N = 5 + def f(): + results1.append(evt.wait(0.0)) + t1 = time.time() + r = evt.wait(0.5) + t2 = time.time() + results2.append((r, t2 - t1)) + Bunch(f, N).wait_for_finished() + self.assertEqual(results1, [False] * N) + for r, dt in results2: + self.assertFalse(r) + self.assertTimeout(dt, 0.5) + # The event is set + results1 = [] + results2 = [] + evt.set() + Bunch(f, N).wait_for_finished() + self.assertEqual(results1, [True] * N) + for r, dt in results2: + self.assertTrue(r) + + def test_set_and_clear(self): + # Issue #13502: check that wait() returns true even when the event is + # cleared before the waiting thread is woken up. + evt = self.eventtype() + results = [] + N = 5 + def f(): + results.append(evt.wait(1)) + b = Bunch(f, N) + b.wait_for_started() + time.sleep(0.5) + evt.set() + evt.clear() + b.wait_for_finished() + self.assertEqual(results, [True] * N) + + def test_reset_internal_locks(self): + # ensure that condition is still using a Lock after reset + evt = self.eventtype() + with evt._cond: + self.assertFalse(evt._cond.acquire(False)) + evt._reset_internal_locks() + with evt._cond: + self.assertFalse(evt._cond.acquire(False)) + + +class ConditionTests(BaseTestCase): + """ + Tests for condition variables. + """ + + def test_acquire(self): + cond = self.condtype() + # Be default we have an RLock: the condition can be acquired multiple + # times. + cond.acquire() + cond.acquire() + cond.release() + cond.release() + lock = threading.Lock() + cond = self.condtype(lock) + cond.acquire() + self.assertFalse(lock.acquire(False)) + cond.release() + self.assertTrue(lock.acquire(False)) + self.assertFalse(cond.acquire(False)) + lock.release() + with cond: + self.assertFalse(lock.acquire(False)) + + def test_unacquired_wait(self): + cond = self.condtype() + self.assertRaises(RuntimeError, cond.wait) + + def test_unacquired_notify(self): + cond = self.condtype() + self.assertRaises(RuntimeError, cond.notify) + + def _check_notify(self, cond): + # Note that this test is sensitive to timing. If the worker threads + # don't execute in a timely fashion, the main thread may think they + # are further along then they are. The main thread therefore issues + # _wait() statements to try to make sure that it doesn't race ahead + # of the workers. + # Secondly, this test assumes that condition variables are not subject + # to spurious wakeups. The absence of spurious wakeups is an implementation + # detail of Condition Cariables in current CPython, but in general, not + # a guaranteed property of condition variables as a programming + # construct. In particular, it is possible that this can no longer + # be conveniently guaranteed should their implementation ever change. + N = 5 + results1 = [] + results2 = [] + phase_num = 0 + def f(): + cond.acquire() + result = cond.wait() + cond.release() + results1.append((result, phase_num)) + cond.acquire() + result = cond.wait() + cond.release() + results2.append((result, phase_num)) + b = Bunch(f, N) + b.wait_for_started() + _wait() + self.assertEqual(results1, []) + # Notify 3 threads at first + cond.acquire() + cond.notify(3) + _wait() + phase_num = 1 + cond.release() + while len(results1) < 3: + _wait() + self.assertEqual(results1, [(True, 1)] * 3) + self.assertEqual(results2, []) + # first wait, to ensure all workers settle into cond.wait() before + # we continue. See issue #8799 + _wait() + # Notify 5 threads: they might be in their first or second wait + cond.acquire() + cond.notify(5) + _wait() + phase_num = 2 + cond.release() + while len(results1) + len(results2) < 8: + _wait() + self.assertEqual(results1, [(True, 1)] * 3 + [(True, 2)] * 2) + self.assertEqual(results2, [(True, 2)] * 3) + _wait() # make sure all workers settle into cond.wait() + # Notify all threads: they are all in their second wait + cond.acquire() + cond.notify_all() + _wait() + phase_num = 3 + cond.release() + while len(results2) < 5: + _wait() + self.assertEqual(results1, [(True, 1)] * 3 + [(True,2)] * 2) + self.assertEqual(results2, [(True, 2)] * 3 + [(True, 3)] * 2) + b.wait_for_finished() + + def test_notify(self): + cond = self.condtype() + self._check_notify(cond) + # A second time, to check internal state is still ok. + self._check_notify(cond) + + def test_timeout(self): + cond = self.condtype() + results = [] + N = 5 + def f(): + cond.acquire() + t1 = time.time() + result = cond.wait(0.5) + t2 = time.time() + cond.release() + results.append((t2 - t1, result)) + Bunch(f, N).wait_for_finished() + self.assertEqual(len(results), N) + for dt, result in results: + self.assertTimeout(dt, 0.5) + # Note that conceptually (that"s the condition variable protocol) + # a wait() may succeed even if no one notifies us and before any + # timeout occurs. Spurious wakeups can occur. + # This makes it hard to verify the result value. + # In practice, this implementation has no spurious wakeups. + self.assertFalse(result) + + def test_waitfor(self): + cond = self.condtype() + state = 0 + def f(): + with cond: + result = cond.wait_for(lambda : state==4) + self.assertTrue(result) + self.assertEqual(state, 4) + b = Bunch(f, 1) + b.wait_for_started() + for i in range(4): + time.sleep(0.01) + with cond: + state += 1 + cond.notify() + b.wait_for_finished() + + def test_waitfor_timeout(self): + cond = self.condtype() + state = 0 + success = [] + def f(): + with cond: + dt = time.time() + result = cond.wait_for(lambda : state==4, timeout=0.1) + dt = time.time() - dt + self.assertFalse(result) + self.assertTimeout(dt, 0.1) + success.append(None) + b = Bunch(f, 1) + b.wait_for_started() + # Only increment 3 times, so state == 4 is never reached. + for i in range(3): + time.sleep(0.01) + with cond: + state += 1 + cond.notify() + b.wait_for_finished() + self.assertEqual(len(success), 1) + + +class BaseSemaphoreTests(BaseTestCase): + """ + Common tests for {bounded, unbounded} semaphore objects. + """ + + def test_constructor(self): + self.assertRaises(ValueError, self.semtype, value = -1) + self.assertRaises(ValueError, self.semtype, value = -sys.maxsize) + + def test_acquire(self): + sem = self.semtype(1) + sem.acquire() + sem.release() + sem = self.semtype(2) + sem.acquire() + sem.acquire() + sem.release() + sem.release() + + def test_acquire_destroy(self): + sem = self.semtype() + sem.acquire() + del sem + + def test_acquire_contended(self): + sem = self.semtype(7) + sem.acquire() + N = 10 + results1 = [] + results2 = [] + phase_num = 0 + def f(): + sem.acquire() + results1.append(phase_num) + sem.acquire() + results2.append(phase_num) + b = Bunch(f, 10) + b.wait_for_started() + while len(results1) + len(results2) < 6: + _wait() + self.assertEqual(results1 + results2, [0] * 6) + phase_num = 1 + for i in range(7): + sem.release() + while len(results1) + len(results2) < 13: + _wait() + self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) + phase_num = 2 + for i in range(6): + sem.release() + while len(results1) + len(results2) < 19: + _wait() + self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) + # The semaphore is still locked + self.assertFalse(sem.acquire(False)) + # Final release, to let the last thread finish + sem.release() + b.wait_for_finished() + + def test_try_acquire(self): + sem = self.semtype(2) + self.assertTrue(sem.acquire(False)) + self.assertTrue(sem.acquire(False)) + self.assertFalse(sem.acquire(False)) + sem.release() + self.assertTrue(sem.acquire(False)) + + def test_try_acquire_contended(self): + sem = self.semtype(4) + sem.acquire() + results = [] + def f(): + results.append(sem.acquire(False)) + results.append(sem.acquire(False)) + Bunch(f, 5).wait_for_finished() + # There can be a thread switch between acquiring the semaphore and + # appending the result, therefore results will not necessarily be + # ordered. + self.assertEqual(sorted(results), [False] * 7 + [True] * 3 ) + + def test_acquire_timeout(self): + sem = self.semtype(2) + self.assertRaises(ValueError, sem.acquire, False, timeout=1.0) + self.assertTrue(sem.acquire(timeout=0.005)) + self.assertTrue(sem.acquire(timeout=0.005)) + self.assertFalse(sem.acquire(timeout=0.005)) + sem.release() + self.assertTrue(sem.acquire(timeout=0.005)) + t = time.time() + self.assertFalse(sem.acquire(timeout=0.5)) + dt = time.time() - t + self.assertTimeout(dt, 0.5) + + def test_default_value(self): + # The default initial value is 1. + sem = self.semtype() + sem.acquire() + def f(): + sem.acquire() + sem.release() + b = Bunch(f, 1) + b.wait_for_started() + _wait() + self.assertFalse(b.finished) + sem.release() + b.wait_for_finished() + + def test_with(self): + sem = self.semtype(2) + def _with(err=None): + with sem: + self.assertTrue(sem.acquire(False)) + sem.release() + with sem: + self.assertFalse(sem.acquire(False)) + if err: + raise err + _with() + self.assertTrue(sem.acquire(False)) + sem.release() + self.assertRaises(TypeError, _with, TypeError) + self.assertTrue(sem.acquire(False)) + sem.release() + +class SemaphoreTests(BaseSemaphoreTests): + """ + Tests for unbounded semaphores. + """ + + def test_release_unacquired(self): + # Unbounded releases are allowed and increment the semaphore's value + sem = self.semtype(1) + sem.release() + sem.acquire() + sem.acquire() + sem.release() + + +class BoundedSemaphoreTests(BaseSemaphoreTests): + """ + Tests for bounded semaphores. + """ + + def test_release_unacquired(self): + # Cannot go past the initial value + sem = self.semtype() + self.assertRaises(ValueError, sem.release) + sem.acquire() + sem.release() + self.assertRaises(ValueError, sem.release) + + +class BarrierTests(BaseTestCase): + """ + Tests for Barrier objects. + """ + N = 5 + defaultTimeout = 2.0 + + def setUp(self): + self.barrier = self.barriertype(self.N, timeout=self.defaultTimeout) + def tearDown(self): + self.barrier.abort() + + def run_threads(self, f): + b = Bunch(f, self.N-1) + f() + b.wait_for_finished() + + def multipass(self, results, n): + m = self.barrier.parties + self.assertEqual(m, self.N) + for i in range(n): + results[0].append(True) + self.assertEqual(len(results[1]), i * m) + self.barrier.wait() + results[1].append(True) + self.assertEqual(len(results[0]), (i + 1) * m) + self.barrier.wait() + self.assertEqual(self.barrier.n_waiting, 0) + self.assertFalse(self.barrier.broken) + + def test_barrier(self, passes=1): + """ + Test that a barrier is passed in lockstep + """ + results = [[],[]] + def f(): + self.multipass(results, passes) + self.run_threads(f) + + def test_barrier_10(self): + """ + Test that a barrier works for 10 consecutive runs + """ + return self.test_barrier(10) + + def test_wait_return(self): + """ + test the return value from barrier.wait + """ + results = [] + def f(): + r = self.barrier.wait() + results.append(r) + + self.run_threads(f) + self.assertEqual(sum(results), sum(range(self.N))) + + def test_action(self): + """ + Test the 'action' callback + """ + results = [] + def action(): + results.append(True) + barrier = self.barriertype(self.N, action) + def f(): + barrier.wait() + self.assertEqual(len(results), 1) + + self.run_threads(f) + + def test_abort(self): + """ + Test that an abort will put the barrier in a broken state + """ + results1 = [] + results2 = [] + def f(): + try: + i = self.barrier.wait() + if i == self.N//2: + raise RuntimeError + self.barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + except RuntimeError: + self.barrier.abort() + pass + + self.run_threads(f) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(self.barrier.broken) + + def test_reset(self): + """ + Test that a 'reset' on a barrier frees the waiting threads + """ + results1 = [] + results2 = [] + results3 = [] + def f(): + i = self.barrier.wait() + if i == self.N//2: + # Wait until the other threads are all in the barrier. + while self.barrier.n_waiting < self.N-1: + time.sleep(0.001) + self.barrier.reset() + else: + try: + self.barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + # Now, pass the barrier again + self.barrier.wait() + results3.append(True) + + self.run_threads(f) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + + def test_abort_and_reset(self): + """ + Test that a barrier can be reset after being broken. + """ + results1 = [] + results2 = [] + results3 = [] + barrier2 = self.barriertype(self.N) + def f(): + try: + i = self.barrier.wait() + if i == self.N//2: + raise RuntimeError + self.barrier.wait() + results1.append(True) + except threading.BrokenBarrierError: + results2.append(True) + except RuntimeError: + self.barrier.abort() + pass + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + if barrier2.wait() == self.N//2: + self.barrier.reset() + barrier2.wait() + self.barrier.wait() + results3.append(True) + + self.run_threads(f) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + def test_timeout(self): + """ + Test wait(timeout) + """ + def f(): + i = self.barrier.wait() + if i == self.N // 2: + # One thread is late! + time.sleep(1.0) + # Default timeout is 2.0, so this is shorter. + self.assertRaises(threading.BrokenBarrierError, + self.barrier.wait, 0.5) + self.run_threads(f) + + def test_default_timeout(self): + """ + Test the barrier's default timeout + """ + # create a barrier with a low default timeout + barrier = self.barriertype(self.N, timeout=0.3) + def f(): + i = barrier.wait() + if i == self.N // 2: + # One thread is later than the default timeout of 0.3s. + time.sleep(1.0) + self.assertRaises(threading.BrokenBarrierError, barrier.wait) + self.run_threads(f) + + def test_single_thread(self): + b = self.barriertype(1) + b.wait() + b.wait() diff --git a/src/greentest/3.5pypy/nokia.pem b/src/greentest/3.5pypy/nokia.pem new file mode 100644 index 0000000..0d044df --- /dev/null +++ b/src/greentest/3.5pypy/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/nullbytecert.pem b/src/greentest/3.5pypy/nullbytecert.pem new file mode 100644 index 0000000..447186c --- /dev/null +++ b/src/greentest/3.5pypy/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/nullcert.pem b/src/greentest/3.5pypy/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/greentest/3.5pypy/pycacert.pem b/src/greentest/3.5pypy/pycacert.pem new file mode 100644 index 0000000..09b1f3e --- /dev/null +++ b/src/greentest/3.5pypy/pycacert.pem @@ -0,0 +1,78 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Jan 2 19:47:07 2023 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: + 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: + e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: + e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: + 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: + 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: + a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: + e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: + 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: + 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: + e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: + c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: + cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: + 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: + 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: + 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: + e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: + c5:4d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + X509v3 Authority Key Identifier: + keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: + 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: + a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: + 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: + 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: + 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: + fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: + 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: + 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: + 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: + ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: + 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: + b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: + 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: + 5e:58:c8:9e +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/pycakey.pem b/src/greentest/3.5pypy/pycakey.pem new file mode 100644 index 0000000..fc6effe --- /dev/null +++ b/src/greentest/3.5pypy/pycakey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDn3unjDJ8AtqH9 +K1uW0m/M4L6GuSBe7AN6VavqpOn5SYXSZtXtx3rqVo4tj+dC4mIoqZ/WG47rtbSc +nxSr3+aUi3YdPm0kYe0MvwCKYQzfXMg2cxYAzUe6baSkdIiDIwoZ/AmnPEpL0+cd +LeTqTFQh8ybbiTcY1AK7QDJfpP8tHPfUu+yOz1yCrOZ8CGxIhWEHfyXgXOC8NF/g +uQRHdchHC4281shoXzODYtIgRDWxrYEais28NbBci0fWGOmcGJfMATwpzOge5OTB +uN7nwhEYh1qTNNimJfcUcevkIaLSDy4u1GIANdPW71xgS0ypFOLdFVhGNzMmt+cu +Xe1C5MVNAgMBAAECggEBAJPM7QuUrPn4cLN/Ysd15lwTWn9oHDFFgkYFvCs66gXE +ju/6Kx2BjWE4wTJby09AHM/MqB0DvguT7Mf1Q2j3tPQ1HZowg8OwRDleuwp6KIls +jBbhL0Jdl/5HC67ktWvZ9wNvO/wFG1rQfT6FVajf9LUbWEaSZbOG2SLhHfsHorzu +xjTJaI3bQ/0+79B1exwk5ruwhzFRd/XpY8hls7D/RfPIuHDlBghkW3N59KFWrf5h +6bNEh2THm0+IyGcGqs0FD+QCOXyvsjwSUswqrr2ctLREOeDcd5ReUjSxYgjcJRrm +J7ceIY/+uwDJxw/OlnmBvF6pQMkKwYW2gFztu+g2t4UCgYEA/9yo01Exz4crxXsy +tAlnDJM++nZcm07rtFjTKHUfKY/cCgNTa8udM0svnfwlid/dpgLsI38gx04HHC1i +EZ4acz+ToIWedLxM0nq73//xeRWEazOvCz1mMTZaMldahTWAyzN8qVK2B/625Yy4 +wNYWyweBBwEB8MzaCs73spksXOsCgYEA5/7wvhiofYGFAfMuANeJIwDL2OtBnoOv +mVNfCmi3GC38fzwyi5ZpskWDiS2woJ+LQfs9Qu4EcZbUFLd7gbeOvb5gmFUtYope +LitUUKunIR18MkQ+mQDBpQPQPhk4QJP5reCbWkrfTu7b5o/iS41s6fBTFmuzhLcT +C71vFdCyeKcCgYAiCCqYeOtELDmBOeLDmaCQRqGQ1N96dOPbCBmF/xYXBCCDYG/f +HaUaJnz96YTgstsbcrYP/p/Qgqtlbw/lQf9IpwMuzbcG1ejt8g89OyDWNyt2ytgU +iaUnFJCos3/Byh0Iah/BsdOueo2/OJl2ZMOBW80orlSgv86cs2y037TL4wKBgQDm +OOyW+MlbowhnIvfoBfwlLEkefnej4nKD6WRLZBcue5Qyf355X06Mhsc9foXlH+6G +D9h/bswiHNdhp6N82rdgPGiHQx/CxiUoE/+b/nvgNO5mw6qLE2EXbG1e8pAMJcyE +bHw+YkawggDfELI036fRj5gki8SeUz8nS1nNgElbyQKBgCRDX9Jh+MwSLu4QBWdt +/fi+lv3K6kun/fI7EOV1vCV/j871tICu7pu5BrOLxAHqoVfU9AUX299/2KjCb5pv +kjogiUK6qWCWBlfuqDNWGCoUGt1rhznUva0nNjSMy5rinBhhjpROZC2pw48lOluP +UuvXsaPph7GTqPuy4Kab12YC +-----END PRIVATE KEY----- diff --git a/src/greentest/3.5pypy/revocation.crl b/src/greentest/3.5pypy/revocation.crl new file mode 100644 index 0000000..6d89b08 --- /dev/null +++ b/src/greentest/3.5pypy/revocation.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTEzMTEyMTE3MDg0N1oXDTIzMDkzMDE3MDg0N1qgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQCNJXC2mVKauEeN3LlQ3ZtM5gkH3ExH ++i4bmJjtJn497WwvvoIeUdrmVXgJQR93RtV37hZwN0SXMLlNmUZPH4rHhihayw4m +unCzVj/OhCCY7/TPjKuJ1O/0XhaLBpBVjQN7R/1ujoRKbSia/CD3vcn7Fqxzw7LK +fSRCKRGTj1CZiuxrphtFchwALXSiFDy9mr2ZKhImcyq1PydfgEzU78APpOkMQsIC +UNJ/cf3c9emzf+dUtcMEcejQ3mynBo4eIGg1EW42bz4q4hSjzQlKcBV0muw5qXhc +HOxH2iTFhQ7SrvVuK/dM14rYM4B5mSX3nRC1kNmXpS9j3wJDhuwmjHed +-----END X509 CRL----- diff --git a/src/greentest/3.5pypy/selfsigned_pythontestdotnet.pem b/src/greentest/3.5pypy/selfsigned_pythontestdotnet.pem new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.5pypy/selfsigned_pythontestdotnet.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/sha256.pem b/src/greentest/3.5pypy/sha256.pem new file mode 100644 index 0000000..d3db4b8 --- /dev/null +++ b/src/greentest/3.5pypy/sha256.pem @@ -0,0 +1,128 @@ +# Certificate chain for https://sha256.tbs-internet.com + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC +-----BEGIN CERTIFICATE----- +MIIGXDCCBUSgAwIBAgIRAKpVmHgg9nfCodAVwcP4siwwDQYJKoZIhvcNAQELBQAw +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMB4XDTEyMDEwNDAwMDAwMFoXDTE0MDIxNzIzNTk1OVowgcsxCzAJBgNV +BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV +BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM +VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS +c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKQIX/zdJcyxty0m +PM1XQSoSSifueS3AVcgqMsaIKS/u+rYzsv4hQ/qA6vLn5m5/ewUcZDj7zdi6rBVf +PaVNXJ6YinLX0tkaW8TEjeVuZG5yksGZlhCt1CJ1Ho9XLiLaP4uJ7MCoNUntpJ+E +LfrOdgsIj91kPmwjDJeztVcQCvKzhjVJA/KxdInc0JvOATn7rpaSmQI5bvIjufgo +qVsTPwVFzuUYULXBk7KxRT7MiEqnd5HvviNh0285QC478zl3v0I0Fb5El4yD3p49 +IthcRnxzMKc0UhU5ogi0SbONyBfm/mzONVfSxpM+MlyvZmJqrbuuLoEDzJD+t8PU +xSuzgbcCAwEAAaOCAj4wggI6MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMB0GA1UdDgQWBBT/qTGYdaj+f61c2IRFL/B1eEsM8DAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG +CisGAQQBgjcKAwMGCWCGSAGG+EIEATBLBgNVHSAERDBCMEAGCisGAQQB5TcCBAEw +MjAwBggrBgEFBQcCARYkaHR0cHM6Ly93d3cudGJzLWludGVybmV0LmNvbS9DQS9D +UFM0MG0GA1UdHwRmMGQwMqAwoC6GLGh0dHA6Ly9jcmwudGJzLWludGVybmV0LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMC6gLKAqhihodHRwOi8vY3JsLnRicy14NTA5LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMIGmBggrBgEFBQcBAQSBmTCBljA4BggrBgEFBQcw +AoYsaHR0cDovL2NydC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQVNHQy5jcnQw +NAYIKwYBBQUHMAKGKGh0dHA6Ly9jcnQudGJzLXg1MDkuY29tL1RCU1g1MDlDQVNH +Qy5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLnRicy14NTA5LmNvbTA/BgNV +HREEODA2ghdzaGEyNTYudGJzLWludGVybmV0LmNvbYIbd3d3LnNoYTI1Ni50YnMt +aW50ZXJuZXQuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQA0pOuL8QvAa5yksTbGShzX +ABApagunUGoEydv4YJT1MXy9tTp7DrWaozZSlsqBxrYAXP1d9r2fuKbEniYHxaQ0 +UYaf1VSIlDo1yuC8wE7wxbHDIpQ/E5KAyxiaJ8obtDhFstWAPAH+UoGXq0kj2teN +21sFQ5dXgA95nldvVFsFhrRUNB6xXAcaj0VZFhttI0ZfQZmQwEI/P+N9Jr40OGun +aa+Dn0TMeUH4U20YntfLbu2nDcJcYfyurm+8/0Tr4HznLnedXu9pCPYj0TaddrgT +XO0oFiyy7qGaY6+qKh71yD64Y3ycCJ/HR9Wm39mjZYc9ezYwT4noP6r7Lk8YO7/q +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6 +rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0 +9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ +ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk +owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G +Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk +9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ +MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3 +AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk +ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k +by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw +cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV +VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B +ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN +AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232 +euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY +1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98 +RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz +8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV +v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E= +-----END CERTIFICATE----- + 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT +AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0 +ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB +IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05 +4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6 +2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh +alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv +u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW +xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p +XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd +tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB +BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX +BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov +L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN +AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO +rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd +FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM ++bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI +3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb ++M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g= +-----END CERTIFICATE----- + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG +EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD +VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu +dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 +E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ +D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK +4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq +lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW +bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB +o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT +MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js +LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr +BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB +AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft +Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj +j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH +KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv +2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 +mfnGV/TJVTl4uix5yaaIK/QI +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/ssl_cert.pem b/src/greentest/3.5pypy/ssl_cert.pem new file mode 100644 index 0000000..47a7d7e --- /dev/null +++ b/src/greentest/3.5pypy/ssl_cert.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.5pypy/ssl_key.passwd.pem b/src/greentest/3.5pypy/ssl_key.passwd.pem new file mode 100644 index 0000000..2524672 --- /dev/null +++ b/src/greentest/3.5pypy/ssl_key.passwd.pem @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- diff --git a/src/greentest/3.5pypy/ssl_key.pem b/src/greentest/3.5pypy/ssl_key.pem new file mode 100644 index 0000000..3fd3bbd --- /dev/null +++ b/src/greentest/3.5pypy/ssl_key.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- diff --git a/src/greentest/3.5pypy/test_asyncore.py b/src/greentest/3.5pypy/test_asyncore.py new file mode 100644 index 0000000..3857916 --- /dev/null +++ b/src/greentest/3.5pypy/test_asyncore.py @@ -0,0 +1,836 @@ +import asyncore +import unittest +import select +import os +import socket +import sys +import time +import errno +import struct + +from test import support +from io import BytesIO + +try: + import threading +except ImportError: + threading = None + +TIMEOUT = 3 +HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') + +class dummysocket: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def fileno(self): + return 42 + +class dummychannel: + def __init__(self): + self.socket = dummysocket() + + def close(self): + self.socket.close() + +class exitingdummy: + def __init__(self): + pass + + def handle_read_event(self): + raise asyncore.ExitNow() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + +class crashingdummy: + def __init__(self): + self.error_handled = False + + def handle_read_event(self): + raise Exception() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + + def handle_error(self): + self.error_handled = True + +# used when testing senders; just collects what it gets until newline is sent +def capture_server(evt, buf, serv): + try: + serv.listen() + conn, addr = serv.accept() + except socket.timeout: + pass + else: + n = 200 + start = time.time() + while n > 0 and time.time() - start < 3.0: + r, w, e = select.select([conn], [], [], 0.1) + if r: + n -= 1 + data = conn.recv(10) + # keep everything except for the newline terminator + buf.write(data.replace(b'\n', b'')) + if b'\n' in data: + break + time.sleep(0.01) + + conn.close() + finally: + serv.close() + evt.set() + +def bind_af_aware(sock, addr): + """Helper function to bind a socket according to its family.""" + if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: + # Make sure the path doesn't exist. + support.unlink(addr) + sock.bind(addr) + + +class HelperFunctionTests(unittest.TestCase): + def test_readwriteexc(self): + # Check exception handling behavior of read, write and _exception + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore read/write/_exception calls + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.read, tr1) + self.assertRaises(asyncore.ExitNow, asyncore.write, tr1) + self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + asyncore.read(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore.write(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore._exception(tr2) + self.assertEqual(tr2.error_handled, True) + + # asyncore.readwrite uses constants in the select module that + # are not present in Windows systems (see this thread: + # http://mail.python.org/pipermail/python-list/2001-October/109973.html) + # These constants should be present as long as poll is available + + @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') + def test_readwrite(self): + # Check that correct methods are called by readwrite() + + attributes = ('read', 'expt', 'write', 'closed', 'error_handled') + + expected = ( + (select.POLLIN, 'read'), + (select.POLLPRI, 'expt'), + (select.POLLOUT, 'write'), + (select.POLLERR, 'closed'), + (select.POLLHUP, 'closed'), + (select.POLLNVAL, 'closed'), + ) + + class testobj: + def __init__(self): + self.read = False + self.write = False + self.closed = False + self.expt = False + self.error_handled = False + + def handle_read_event(self): + self.read = True + + def handle_write_event(self): + self.write = True + + def handle_close(self): + self.closed = True + + def handle_expt_event(self): + self.expt = True + + def handle_error(self): + self.error_handled = True + + for flag, expectedattr in expected: + tobj = testobj() + self.assertEqual(getattr(tobj, expectedattr), False) + asyncore.readwrite(tobj, flag) + + # Only the attribute modified by the routine we expect to be + # called should be True. + for attr in attributes: + self.assertEqual(getattr(tobj, attr), attr==expectedattr) + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore readwrite call + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + self.assertEqual(tr2.error_handled, False) + asyncore.readwrite(tr2, flag) + self.assertEqual(tr2.error_handled, True) + + def test_closeall(self): + self.closeall_check(False) + + def test_closeall_default(self): + self.closeall_check(True) + + def closeall_check(self, usedefault): + # Check that close_all() closes everything in a given map + + l = [] + testmap = {} + for i in range(10): + c = dummychannel() + l.append(c) + self.assertEqual(c.socket.closed, False) + testmap[i] = c + + if usedefault: + socketmap = asyncore.socket_map + try: + asyncore.socket_map = testmap + asyncore.close_all() + finally: + testmap, asyncore.socket_map = asyncore.socket_map, socketmap + else: + asyncore.close_all(testmap) + + self.assertEqual(len(testmap), 0) + + for c in l: + self.assertEqual(c.socket.closed, True) + + def test_compact_traceback(self): + try: + raise Exception("I don't like spam!") + except: + real_t, real_v, real_tb = sys.exc_info() + r = asyncore.compact_traceback() + else: + self.fail("Expected exception") + + (f, function, line), t, v, info = r + self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py') + self.assertEqual(function, 'test_compact_traceback') + self.assertEqual(t, real_t) + self.assertEqual(v, real_v) + self.assertEqual(info, '[%s|%s|%s]' % (f, function, line)) + + +class DispatcherTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + def test_basic(self): + d = asyncore.dispatcher() + self.assertEqual(d.readable(), True) + self.assertEqual(d.writable(), True) + + def test_repr(self): + d = asyncore.dispatcher() + self.assertEqual(repr(d), '' % id(d)) + + def test_log(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log() (to stderr) + l1 = "Lovely spam! Wonderful spam!" + l2 = "I don't like spam!" + with support.captured_stderr() as stderr: + d.log(l1) + d.log(l2) + + lines = stderr.getvalue().splitlines() + self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2]) + + def test_log_info(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log_info() (to stdout via print) + l1 = "Have you got anything without spam?" + l2 = "Why can't she have egg bacon spam and sausage?" + l3 = "THAT'S got spam in it!" + with support.captured_stdout() as stdout: + d.log_info(l1, 'EGGS') + d.log_info(l2) + d.log_info(l3, 'SPAM') + + lines = stdout.getvalue().splitlines() + expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3] + self.assertEqual(lines, expected) + + def test_unhandled(self): + d = asyncore.dispatcher() + d.ignore_log_types = () + + # capture output of dispatcher.log_info() (to stdout via print) + with support.captured_stdout() as stdout: + d.handle_expt() + d.handle_read() + d.handle_write() + d.handle_connect() + + lines = stdout.getvalue().splitlines() + expected = ['warning: unhandled incoming priority event', + 'warning: unhandled read event', + 'warning: unhandled write event', + 'warning: unhandled connect event'] + self.assertEqual(lines, expected) + + def test_strerror(self): + # refers to bug #8573 + err = asyncore._strerror(errno.EPERM) + if hasattr(os, 'strerror'): + self.assertEqual(err, os.strerror(errno.EPERM)) + err = asyncore._strerror(-1) + self.assertTrue(err != "") + + +class dispatcherwithsend_noread(asyncore.dispatcher_with_send): + def readable(self): + return False + + def handle_connect(self): + pass + + +class DispatcherWithSendTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + @unittest.skipUnless(threading, 'Threading required for this test.') + @support.reap_threads + def test_send(self): + evt = threading.Event() + sock = socket.socket() + sock.settimeout(3) + port = support.bind_port(sock) + + cap = BytesIO() + args = (evt, cap, sock) + t = threading.Thread(target=capture_server, args=args) + t.start() + try: + # wait a little longer for the server to initialize (it sometimes + # refuses connections on slow machines without this wait) + time.sleep(0.2) + + data = b"Suppose there isn't a 16-ton weight?" + d = dispatcherwithsend_noread() + d.create_socket() + d.connect((support.HOST, port)) + + # give time for socket to connect + time.sleep(0.1) + + d.send(data) + d.send(data) + d.send(b'\n') + + n = 1000 + while d.out_buffer and n > 0: + asyncore.poll() + n -= 1 + + evt.wait() + + self.assertEqual(cap.getvalue(), data*2) + finally: + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") + + +@unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), + 'asyncore.file_wrapper required') +class FileWrapperTest(unittest.TestCase): + def setUp(self): + self.d = b"It's not dead, it's sleeping!" + with open(support.TESTFN, 'wb') as file: + file.write(self.d) + + def tearDown(self): + support.unlink(support.TESTFN) + + def test_recv(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + w = asyncore.file_wrapper(fd) + os.close(fd) + + self.assertNotEqual(w.fd, fd) + self.assertNotEqual(w.fileno(), fd) + self.assertEqual(w.recv(13), b"It's not dead") + self.assertEqual(w.read(6), b", it's") + w.close() + self.assertRaises(OSError, w.read, 1) + + def test_send(self): + d1 = b"Come again?" + d2 = b"I want to buy some cheese." + fd = os.open(support.TESTFN, os.O_WRONLY | os.O_APPEND) + w = asyncore.file_wrapper(fd) + os.close(fd) + + w.write(d1) + w.send(d2) + w.close() + with open(support.TESTFN, 'rb') as file: + self.assertEqual(file.read(), self.d + d1 + d2) + + @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'), + 'asyncore.file_dispatcher required') + def test_dispatcher(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + data = [] + class FileDispatcher(asyncore.file_dispatcher): + def handle_read(self): + data.append(self.recv(29)) + s = FileDispatcher(fd) + os.close(fd) + asyncore.loop(timeout=0.01, use_poll=True, count=2) + self.assertEqual(b"".join(data), self.d) + + def test_resource_warning(self): + # Issue #11453 + fd = os.open(support.TESTFN, os.O_RDONLY) + f = asyncore.file_wrapper(fd) + + os.close(fd) + with support.check_warnings(('', ResourceWarning)): + f = None + support.gc_collect() + + def test_close_twice(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + f = asyncore.file_wrapper(fd) + os.close(fd) + + f.close() + self.assertEqual(f.fd, -1) + # calling close twice should not fail + f.close() + + +class BaseTestHandler(asyncore.dispatcher): + + def __init__(self, sock=None): + asyncore.dispatcher.__init__(self, sock) + self.flag = False + + def handle_accept(self): + raise Exception("handle_accept not supposed to be called") + + def handle_accepted(self): + raise Exception("handle_accepted not supposed to be called") + + def handle_connect(self): + raise Exception("handle_connect not supposed to be called") + + def handle_expt(self): + raise Exception("handle_expt not supposed to be called") + + def handle_close(self): + raise Exception("handle_close not supposed to be called") + + def handle_error(self): + raise + + +class BaseServer(asyncore.dispatcher): + """A server which listens on an address and dispatches the + connection to a handler. + """ + + def __init__(self, family, addr, handler=BaseTestHandler): + asyncore.dispatcher.__init__(self) + self.create_socket(family) + self.set_reuse_addr() + bind_af_aware(self.socket, addr) + self.listen(5) + self.handler = handler + + @property + def address(self): + return self.socket.getsockname() + + def handle_accepted(self, sock, addr): + self.handler(sock) + + def handle_error(self): + raise + + +class BaseClient(BaseTestHandler): + + def __init__(self, family, address): + BaseTestHandler.__init__(self) + self.create_socket(family) + self.connect(address) + + def handle_connect(self): + pass + + +class BaseTestAPI: + + def tearDown(self): + asyncore.close_all() + + def loop_waiting_for_flag(self, instance, timeout=5): + timeout = float(timeout) / 100 + count = 100 + while asyncore.socket_map and count > 0: + asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) + if instance.flag: + return + count -= 1 + time.sleep(timeout) + self.fail("flag not set") + + def test_handle_connect(self): + # make sure handle_connect is called on connect() + + class TestClient(BaseClient): + def handle_connect(self): + self.flag = True + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_accept(self): + # make sure handle_accept() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self, family, addr): + BaseTestHandler.__init__(self) + self.create_socket(family) + bind_af_aware(self.socket, addr) + self.listen(5) + self.address = self.socket.getsockname() + + def handle_accept(self): + self.flag = True + + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) + self.loop_waiting_for_flag(server) + + def test_handle_accepted(self): + # make sure handle_accepted() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self, family, addr): + BaseTestHandler.__init__(self) + self.create_socket(family) + bind_af_aware(self.socket, addr) + self.listen(5) + self.address = self.socket.getsockname() + + def handle_accept(self): + asyncore.dispatcher.handle_accept(self) + + def handle_accepted(self, sock, addr): + sock.close() + self.flag = True + + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) + self.loop_waiting_for_flag(server) + + + def test_handle_read(self): + # make sure handle_read is called on data received + + class TestClient(BaseClient): + def handle_read(self): + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.send(b'x' * 1024) + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_write(self): + # make sure handle_write is called + + class TestClient(BaseClient): + def handle_write(self): + self.flag = True + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close(self): + # make sure handle_close is called when the other end closes + # the connection + + class TestClient(BaseClient): + + def handle_read(self): + # in order to make handle_close be called we are supposed + # to make at least one recv() call + self.recv(1024) + + def handle_close(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.close() + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close_after_conn_broken(self): + # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and + # #11265). + + data = b'\0' * 128 + + class TestClient(BaseClient): + + def handle_write(self): + self.send(data) + + def handle_close(self): + self.flag = True + self.close() + + def handle_expt(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + + def handle_read(self): + self.recv(len(data)) + self.close() + + def writable(self): + return False + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + @unittest.skipIf(sys.platform.startswith("sunos"), + "OOB support is broken on Solaris") + def test_handle_expt(self): + # Make sure handle_expt is called on OOB data received. + # Note: this might fail on some platforms as OOB data is + # tenuously supported and rarely used. + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + + class TestClient(BaseClient): + def handle_expt(self): + self.socket.recv(1024, socket.MSG_OOB) + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB) + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_error(self): + + class TestClient(BaseClient): + def handle_write(self): + 1.0 / 0 + def handle_error(self): + self.flag = True + try: + raise + except ZeroDivisionError: + pass + else: + raise Exception("exception not raised") + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_connection_attributes(self): + server = BaseServer(self.family, self.addr) + client = BaseClient(self.family, server.address) + + # we start disconnected + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + # this can't be taken for granted across all platforms + #self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # execute some loops so that client connects to server + asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100) + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertTrue(client.connected) + self.assertFalse(client.accepting) + + # disconnect the client + client.close() + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # stop serving + server.close() + self.assertFalse(server.connected) + self.assertFalse(server.accepting) + + def test_create_socket(self): + s = asyncore.dispatcher() + s.create_socket(self.family) + self.assertEqual(s.socket.family, self.family) + SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) + sock_type = socket.SOCK_STREAM | SOCK_NONBLOCK + if hasattr(socket, 'SOCK_CLOEXEC'): + self.assertIn(s.socket.type, + (sock_type | socket.SOCK_CLOEXEC, sock_type)) + else: + self.assertEqual(s.socket.type, sock_type) + + def test_bind(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + s1 = asyncore.dispatcher() + s1.create_socket(self.family) + s1.bind(self.addr) + s1.listen(5) + port = s1.socket.getsockname()[1] + + s2 = asyncore.dispatcher() + s2.create_socket(self.family) + # EADDRINUSE indicates the socket was correctly bound + self.assertRaises(OSError, s2.bind, (self.addr[0], port)) + + def test_set_reuse_addr(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + sock = socket.socket(self.family) + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except OSError: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket(self.family)) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.socket.close() + s.create_socket(self.family) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + finally: + sock.close() + + @unittest.skipUnless(threading, 'Threading required for this test.') + @support.reap_threads + def test_quick_connect(self): + # see: http://bugs.python.org/issue10340 + if self.family in (socket.AF_INET, getattr(socket, "AF_INET6", object())): + server = BaseServer(self.family, self.addr) + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, + count=500)) + t.start() + def cleanup(): + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") + self.addCleanup(cleanup) + + s = socket.socket(self.family, socket.SOCK_STREAM) + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + try: + s.connect(server.address) + except OSError: + pass + finally: + s.close() + +class TestAPI_UseIPv4Sockets(BaseTestAPI): + family = socket.AF_INET + addr = (support.HOST, 0) + +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 support required') +class TestAPI_UseIPv6Sockets(BaseTestAPI): + family = socket.AF_INET6 + addr = (support.HOSTv6, 0) + +@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required') +class TestAPI_UseUnixSockets(BaseTestAPI): + if HAS_UNIX_SOCKETS: + family = socket.AF_UNIX + addr = support.TESTFN + + def tearDown(self): + support.unlink(self.addr) + BaseTestAPI.tearDown(self) + +class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets, unittest.TestCase): + use_poll = True + +class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets, unittest.TestCase): + use_poll = True + +class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets, unittest.TestCase): + use_poll = True + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_httplib.py b/src/greentest/3.5pypy/test_httplib.py new file mode 100644 index 0000000..61ed6bb --- /dev/null +++ b/src/greentest/3.5pypy/test_httplib.py @@ -0,0 +1,1753 @@ +import errno +from http import client +import io +import itertools +import os +import array +import socket + +import unittest +TestCase = unittest.TestCase + +from test import support + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') +# Self-signed cert file for self-signed.pythontest.net +CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') + +# constants for testing chunked encoding +chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd! \r\n' + '8\r\n' + 'and now \r\n' + '22\r\n' + 'for something completely different\r\n' +) +chunked_expected = b'hello world! and now for something completely different' +chunk_extension = ";foo=bar" +last_chunk = "0\r\n" +last_chunk_extended = "0" + chunk_extension + "\r\n" +trailers = "X-Dummy: foo\r\nX-Dumm2: bar\r\n" +chunked_end = "\r\n" + +HOST = support.HOST + +class FakeSocket: + def __init__(self, text, fileclass=io.BytesIO, host=None, port=None): + if isinstance(text, str): + text = text.encode("ascii") + self.text = text + self.fileclass = fileclass + self.data = b'' + self.sendall_calls = 0 + self.file_closed = False + self.host = host + self.port = port + + def sendall(self, data): + self.sendall_calls += 1 + self.data += data + + def makefile(self, mode, bufsize=None): + if mode != 'r' and mode != 'rb': + raise client.UnimplementedFileMode() + # keep the file around so we can check how much was read from it + self.file = self.fileclass(self.text) + self.file.close = self.file_close #nerf close () + return self.file + + def file_close(self): + self.file_closed = True + + def close(self): + pass + + def setsockopt(self, level, optname, value): + pass + +class EPipeSocket(FakeSocket): + + def __init__(self, text, pipe_trigger): + # When sendall() is called with pipe_trigger, raise EPIPE. + FakeSocket.__init__(self, text) + self.pipe_trigger = pipe_trigger + + def sendall(self, data): + if self.pipe_trigger in data: + raise OSError(errno.EPIPE, "gotcha") + self.data += data + + def close(self): + pass + +class NoEOFBytesIO(io.BytesIO): + """Like BytesIO, but raises AssertionError on EOF. + + This is used below to test that http.client doesn't try to read + more from the underlying file than it should. + """ + def read(self, n=-1): + data = io.BytesIO.read(self, n) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + + def readline(self, length=None): + data = io.BytesIO.readline(self, length) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + +class FakeSocketHTTPConnection(client.HTTPConnection): + """HTTPConnection subclass using FakeSocket; counts connect() calls""" + + def __init__(self, *args): + self.connections = 0 + super().__init__('example.com') + self.fake_socket_args = args + self._create_connection = self.create_connection + + def connect(self): + """Count the number of times connect() is invoked""" + self.connections += 1 + return super().connect() + + def create_connection(self, *pos, **kw): + return FakeSocket(*self.fake_socket_args) + +class HeaderTests(TestCase): + def test_auto_headers(self): + # Some headers are added automatically, but should not be added by + # .request() if they are explicitly set. + + class HeaderCountingBuffer(list): + def __init__(self): + self.count = {} + def append(self, item): + kv = item.split(b':') + if len(kv) > 1: + # item is a 'Key: Value' header string + lcKey = kv[0].decode('ascii').lower() + self.count.setdefault(lcKey, 0) + self.count[lcKey] += 1 + list.append(self, item) + + for explicit_header in True, False: + for header in 'Content-length', 'Host', 'Accept-encoding': + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('blahblahblah') + conn._buffer = HeaderCountingBuffer() + + body = 'spamspamspam' + headers = {} + if explicit_header: + headers[header] = str(len(body)) + conn.request('POST', '/', body, headers) + self.assertEqual(conn._buffer.count[header.lower()], 1) + + def test_content_length_0(self): + + class ContentLengthChecker(list): + def __init__(self): + list.__init__(self) + self.content_length = None + def append(self, item): + kv = item.split(b':', 1) + if len(kv) > 1 and kv[0].lower() == b'content-length': + self.content_length = kv[1].strip() + list.append(self, item) + + # Here, we're testing that methods expecting a body get a + # content-length set to zero if the body is empty (either None or '') + bodies = (None, '') + methods_with_body = ('PUT', 'POST', 'PATCH') + for method, body in itertools.product(methods_with_body, bodies): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', body) + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # For these methods, we make sure that content-length is not set when + # the body is None because it might cause unexpected behaviour on the + # server. + methods_without_body = ( + 'GET', 'CONNECT', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', + ) + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', None) + self.assertEqual( + conn._buffer.content_length, None, + 'Header Content-Length set for empty body on {}'.format(method) + ) + + # If the body is set to '', that's considered to be "present but + # empty" rather than "missing", so content length would be set, even + # for methods that don't expect a body. + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', '') + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # If the body is set, make sure Content-Length is set. + for method in itertools.chain(methods_without_body, methods_with_body): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', ' ') + self.assertEqual( + conn._buffer.content_length, b'1', + 'Header Content-Length incorrect on {}'.format(method) + ) + + def test_putheader(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.putrequest('GET','/') + conn.putheader('Content-length', 42) + self.assertIn(b'Content-length: 42', conn._buffer) + + conn.putheader('Foo', ' bar ') + self.assertIn(b'Foo: bar ', conn._buffer) + conn.putheader('Bar', '\tbaz\t') + self.assertIn(b'Bar: \tbaz\t', conn._buffer) + conn.putheader('Authorization', 'Bearer mytoken') + self.assertIn(b'Authorization: Bearer mytoken', conn._buffer) + conn.putheader('IterHeader', 'IterA', 'IterB') + self.assertIn(b'IterHeader: IterA\r\n\tIterB', conn._buffer) + conn.putheader('LatinHeader', b'\xFF') + self.assertIn(b'LatinHeader: \xFF', conn._buffer) + conn.putheader('Utf8Header', b'\xc3\x80') + self.assertIn(b'Utf8Header: \xc3\x80', conn._buffer) + conn.putheader('C1-Control', b'next\x85line') + self.assertIn(b'C1-Control: next\x85line', conn._buffer) + conn.putheader('Embedded-Fold-Space', 'is\r\n allowed') + self.assertIn(b'Embedded-Fold-Space: is\r\n allowed', conn._buffer) + conn.putheader('Embedded-Fold-Tab', 'is\r\n\tallowed') + self.assertIn(b'Embedded-Fold-Tab: is\r\n\tallowed', conn._buffer) + conn.putheader('Key Space', 'value') + self.assertIn(b'Key Space: value', conn._buffer) + conn.putheader('KeySpace ', 'value') + self.assertIn(b'KeySpace : value', conn._buffer) + conn.putheader(b'Nonbreak\xa0Space', 'value') + self.assertIn(b'Nonbreak\xa0Space: value', conn._buffer) + conn.putheader(b'\xa0NonbreakSpace', 'value') + self.assertIn(b'\xa0NonbreakSpace: value', conn._buffer) + + def test_ipv6host_header(self): + # Default host header on IPv6 transaction should be wrapped by [] if + # it is an IPv6 address + expected = b'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001::]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + expected = b'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001:102A::]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + def test_malformed_headers_coped_with(self): + # Issue 19996 + body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + + self.assertEqual(resp.getheader('First'), 'val') + self.assertEqual(resp.getheader('Second'), 'val') + + def test_parse_all_octets(self): + # Ensure no valid header field octet breaks the parser + body = ( + b'HTTP/1.1 200 OK\r\n' + b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters + b'VCHAR: ' + bytes(range(0x21, 0x7E + 1)) + b'\r\n' + b'obs-text: ' + bytes(range(0x80, 0xFF + 1)) + b'\r\n' + b'obs-fold: text\r\n' + b' folded with space\r\n' + b'\tfolded with tab\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.getheader('Content-Length'), '0') + self.assertEqual(resp.msg['Content-Length'], '0') + self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value') + self.assertEqual(resp.msg["!#$%&'*+-.^_`|~"], 'value') + vchar = ''.join(map(chr, range(0x21, 0x7E + 1))) + self.assertEqual(resp.getheader('VCHAR'), vchar) + self.assertEqual(resp.msg['VCHAR'], vchar) + self.assertIsNotNone(resp.getheader('obs-text')) + self.assertIn('obs-text', resp.msg) + for folded in (resp.getheader('obs-fold'), resp.msg['obs-fold']): + self.assertTrue(folded.startswith('text')) + self.assertIn(' folded with space', folded) + self.assertTrue(folded.endswith('folded with tab')) + + def test_invalid_headers(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/') + + # http://tools.ietf.org/html/rfc7230#section-3.2.4, whitespace is no + # longer allowed in header names + cases = ( + (b'Invalid\r\nName', b'ValidValue'), + (b'Invalid\rName', b'ValidValue'), + (b'Invalid\nName', b'ValidValue'), + (b'\r\nInvalidName', b'ValidValue'), + (b'\rInvalidName', b'ValidValue'), + (b'\nInvalidName', b'ValidValue'), + (b' InvalidName', b'ValidValue'), + (b'\tInvalidName', b'ValidValue'), + (b'Invalid:Name', b'ValidValue'), + (b':InvalidName', b'ValidValue'), + (b'ValidName', b'Invalid\r\nValue'), + (b'ValidName', b'Invalid\rValue'), + (b'ValidName', b'Invalid\nValue'), + (b'ValidName', b'InvalidValue\r\n'), + (b'ValidName', b'InvalidValue\r'), + (b'ValidName', b'InvalidValue\n'), + ) + for name, value in cases: + with self.subTest((name, value)): + with self.assertRaisesRegex(ValueError, 'Invalid header'): + conn.putheader(name, value) + + +class BasicTest(TestCase): + def test_status_lines(self): + # Test HTTP status lines + + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(0), b'') # Issue #20007 + self.assertFalse(resp.isclosed()) + self.assertFalse(resp.closed) + self.assertEqual(resp.read(), b"Text") + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + self.assertRaises(client.BadStatusLine, resp.begin) + + def test_bad_status_repr(self): + exc = client.BadStatusLine('') + self.assertEqual(repr(exc), '''BadStatusLine("\'\'",)''') + + def test_partial_reads(self): + # if we have Content-Length, HTTPResponse knows when to close itself, + # the same behaviour as when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_mixed_reads(self): + # readline() should update the remaining length, so that read() knows + # how much data is left and does not raise IncompleteRead + body = "HTTP/1.1 200 Ok\r\nContent-Length: 13\r\n\r\nText\r\nAnother" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.readline(), b'Text\r\n') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(), b'Another') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos(self): + # if we have Content-Length, HTTPResponse knows when to close itself, + # the same behaviour as when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_reads_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + + def test_partial_reads_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + + def test_partial_readintos_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:80", "www.python.org", 80), + ("www.python.org:", "www.python.org", 80), + ("www.python.org", "www.python.org", 80), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", 80)): + c = client.HTTPConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + def test_response_headers(self): + # test response with multiple message headers with the same field name. + text = ('HTTP/1.1 200 OK\r\n' + 'Set-Cookie: Customer="WILE_E_COYOTE"; ' + 'Version="1"; Path="/acme"\r\n' + 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' + ' Path="/acme"\r\n' + '\r\n' + 'No body\r\n') + hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"' + ', ' + 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"') + s = FakeSocket(text) + r = client.HTTPResponse(s) + r.begin() + cookies = r.getheader("Set-Cookie") + self.assertEqual(cookies, hdr) + + def test_read_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + if resp.read(): + self.fail("Did not expect response from HEAD request") + + def test_readinto_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + if resp.readinto(b) != 0: + self.fail("Did not expect response from HEAD request") + self.assertEqual(bytes(b), b'\x00'*5) + + def test_too_many_headers(self): + headers = '\r\n'.join('Header%d: foo' % i + for i in range(client._MAXHEADERS + 1)) + '\r\n' + text = ('HTTP/1.1 200 OK\r\n' + headers) + s = FakeSocket(text) + r = client.HTTPResponse(s) + self.assertRaisesRegex(client.HTTPException, + r"got more than \d+ headers", r.begin) + + def test_send_file(self): + expected = (b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' + b'Accept-Encoding: identity\r\nContent-Length:') + + with open(__file__, 'rb') as body: + conn = client.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertTrue(sock.data.startswith(expected), '%r != %r' % + (sock.data[:len(expected)], expected)) + + def test_send(self): + expected = b'this is a test this is only a test' + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + conn.send(expected) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(array.array('b', expected)) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(io.BytesIO(expected)) + self.assertEqual(expected, sock.data) + + def test_send_updating_file(self): + def data(): + yield 'data' + yield None + yield 'data_two' + + class UpdatingFile(): + mode = 'r' + d = data() + def read(self, blocksize=-1): + return self.d.__next__() + + expected = b'data' + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.send(UpdatingFile()) + self.assertEqual(sock.data, expected) + + + def test_send_iter(self): + expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \ + b'\r\nonetwothree' + + def body(): + yield b"one" + yield b"two" + yield b"three" + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.request('GET', '/foo', body(), {'Content-Length': '11'}) + self.assertEqual(sock.data, expected) + + def test_send_type_error(self): + # See: Issue #12676 + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + with self.assertRaises(TypeError): + conn.request('POST', 'test', conn) + + def test_chunked(self): + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(n) + resp.read(n) + resp.read(), expected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_readinto_chunked(self): + + expected = chunked_expected + nexpected = len(expected) + b = bytearray(128) + + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + n = resp.readinto(b) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(n, nexpected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + m = memoryview(b) + i = resp.readinto(m[0:n]) + i += resp.readinto(m[i:n + i]) + i += resp.readinto(m[i:]) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(i, nexpected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + n = resp.readinto(b) + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_readinto_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertEqual(bytes(b), b'\x00'*5) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_negative_content_length(self): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), b'Hello\r\n') + self.assertTrue(resp.isclosed()) + + def test_incomplete_read(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, b'Hello\r\n') + self.assertEqual(repr(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertEqual(str(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertTrue(resp.isclosed()) + else: + self.fail('IncompleteRead expected') + + def test_epipe(self): + sock = EPipeSocket( + "HTTP/1.0 401 Authorization Required\r\n" + "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"example\"\r\n", + b"Content-Length") + conn = client.HTTPConnection("example.com") + conn.sock = sock + self.assertRaises(OSError, + lambda: conn.request("PUT", "/url", "body")) + resp = conn.getresponse() + self.assertEqual(401, resp.status) + self.assertEqual("Basic realm=\"example\"", + resp.getheader("www-authenticate")) + + # Test lines overflowing the max line size (_MAXLINE in http.client) + + def test_overflowing_status_line(self): + body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n" + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises((client.LineTooLong, client.BadStatusLine), resp.begin) + + def test_overflowing_header_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises(client.LineTooLong, resp.begin) + + def test_overflowing_chunked_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + + '0' * 65536 + 'a\r\n' + 'hello world\r\n' + '0\r\n' + '\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertRaises(client.LineTooLong, resp.read) + + def test_early_eof(self): + # Test httpresponse with no \r\n termination, + body = "HTTP/1.1 200 Ok" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_error_leak(self): + # Test that the socket is not leaked if getresponse() fails + conn = client.HTTPConnection('example.com') + response = None + class Response(client.HTTPResponse): + def __init__(self, *pos, **kw): + nonlocal response + response = self # Avoid garbage collector closing the socket + client.HTTPResponse.__init__(self, *pos, **kw) + conn.response_class = Response + conn.sock = FakeSocket('Invalid status line') + conn.request('GET', '/') + self.assertRaises(client.BadStatusLine, conn.getresponse) + self.assertTrue(response.closed) + self.assertTrue(conn.sock.file_closed) + + def test_chunked_extension(self): + extra = '3;foo=bar\r\n' + 'abc\r\n' + expected = chunked_expected + b'abc' + + sock = FakeSocket(chunked_start + extra + last_chunk_extended + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + def test_chunked_missing_end(self): + """some servers may serve up a short chunked encoding stream""" + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk) #no terminating crlf + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + def test_chunked_trailers(self): + """See that trailers are read and ignored""" + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # we should have reached the end of the file + self.assertEqual(sock.file.read(), b"") #we read to the end + resp.close() + + def test_chunked_sync(self): + """Check that we don't read past the end of the chunked-encoding stream""" + expected = chunked_expected + extradata = "extradata" + sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata.encode("ascii")) #we read to the end + resp.close() + + def test_content_length_sync(self): + """Check that we don't read past the end of the Content-Length stream""" + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_readlines_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.readlines(2000), [expected]) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_read1_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read1(2000), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_readline_bound_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.readline(10), expected) + self.assertEqual(resp.readline(10), b"") + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_read1_bound_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 30\r\n\r\n' + expected*3 + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read1(20), expected*2) + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_response_fileno(self): + # Make sure fd returned by fileno is valid. + threading = support.import_module("threading") + + serv = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + self.addCleanup(serv.close) + serv.bind((HOST, 0)) + serv.listen() + + result = None + def run_server(): + [conn, address] = serv.accept() + with conn, conn.makefile("rb") as reader: + # Read the request header until a blank line + while True: + line = reader.readline() + if not line.rstrip(b"\r\n"): + break + conn.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n") + nonlocal result + result = reader.read() + + thread = threading.Thread(target=run_server) + thread.start() + conn = client.HTTPConnection(*serv.getsockname()) + conn.request("CONNECT", "dummy:1234") + response = conn.getresponse() + try: + self.assertEqual(response.status, client.OK) + s = socket.socket(fileno=response.fileno()) + try: + s.sendall(b"proxied data\n") + finally: + s.detach() + finally: + response.close() + conn.close() + thread.join() + self.assertEqual(result, b"proxied data\n") + +class ExtendedReadTest(TestCase): + """ + Test peek(), read1(), readline() + """ + lines = ( + 'HTTP/1.1 200 OK\r\n' + '\r\n' + 'hello world!\n' + 'and now \n' + 'for something completely different\n' + 'foo' + ) + lines_expected = lines[lines.find('hello'):].encode("ascii") + lines_chunked = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd!\n\r\n' + '9\r\n' + 'and now \n\r\n' + '23\r\n' + 'for something completely different\n\r\n' + '3\r\n' + 'foo\r\n' + '0\r\n' # terminating chunk + '\r\n' # end of trailers + ) + + def setUp(self): + sock = FakeSocket(self.lines) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + resp.fp = io.BufferedReader(resp.fp) + self.resp = resp + + + + def test_peek(self): + resp = self.resp + # patch up the buffered peek so that it returns not too much stuff + oldpeek = resp.fp.peek + def mypeek(n=-1): + p = oldpeek(n) + if n >= 0: + return p[:n] + return p[:10] + resp.fp.peek = mypeek + + all = [] + while True: + # try a short peek + p = resp.peek(3) + if p: + self.assertGreater(len(p), 0) + # then unbounded peek + p2 = resp.peek() + self.assertGreaterEqual(len(p2), len(p)) + self.assertTrue(p2.startswith(p)) + next = resp.read(len(p2)) + self.assertEqual(next, p2) + else: + next = resp.read() + self.assertFalse(next) + all.append(next) + if not next: + break + self.assertEqual(b"".join(all), self.lines_expected) + + def test_readline(self): + resp = self.resp + self._verify_readline(self.resp.readline, self.lines_expected) + + def _verify_readline(self, readline, expected): + all = [] + while True: + # short readlines + line = readline(5) + if line and line != b"foo": + if len(line) < 5: + self.assertTrue(line.endswith(b"\n")) + all.append(line) + if not line: + break + self.assertEqual(b"".join(all), expected) + + def test_read1(self): + resp = self.resp + def r(): + res = resp.read1(4) + self.assertLessEqual(len(res), 4) + return res + readliner = Readliner(r) + self._verify_readline(readliner.readline, self.lines_expected) + + def test_read1_unbounded(self): + resp = self.resp + all = [] + while True: + data = resp.read1() + if not data: + break + all.append(data) + self.assertEqual(b"".join(all), self.lines_expected) + + def test_read1_bounded(self): + resp = self.resp + all = [] + while True: + data = resp.read1(10) + if not data: + break + self.assertLessEqual(len(data), 10) + all.append(data) + self.assertEqual(b"".join(all), self.lines_expected) + + def test_read1_0(self): + self.assertEqual(self.resp.read1(0), b"") + + def test_peek_0(self): + p = self.resp.peek(0) + self.assertLessEqual(0, len(p)) + +class ExtendedReadTestChunked(ExtendedReadTest): + """ + Test peek(), read1(), readline() in chunked mode + """ + lines = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd!\n\r\n' + '9\r\n' + 'and now \n\r\n' + '23\r\n' + 'for something completely different\n\r\n' + '3\r\n' + 'foo\r\n' + '0\r\n' # terminating chunk + '\r\n' # end of trailers + ) + + +class Readliner: + """ + a simple readline class that uses an arbitrary read function and buffering + """ + def __init__(self, readfunc): + self.readfunc = readfunc + self.remainder = b"" + + def readline(self, limit): + data = [] + datalen = 0 + read = self.remainder + try: + while True: + idx = read.find(b'\n') + if idx != -1: + break + if datalen + len(read) >= limit: + idx = limit - datalen - 1 + # read more data + data.append(read) + read = self.readfunc() + if not read: + idx = 0 #eof condition + break + idx += 1 + data.append(read[:idx]) + self.remainder = read[idx:] + return b"".join(data) + except: + self.remainder = b"".join(data) + raise + + +class OfflineTest(TestCase): + def test_all(self): + # Documented objects defined in the module should be in __all__ + expected = {"responses"} # White-list documented dict() object + # HTTPMessage, parse_headers(), and the HTTP status code constants are + # intentionally omitted for simplicity + blacklist = {"HTTPMessage", "parse_headers"} + for name in dir(client): + if name.startswith("_") or name in blacklist: + continue + module_object = getattr(client, name) + if getattr(module_object, "__module__", None) == "http.client": + expected.add(name) + self.assertCountEqual(client.__all__, expected) + + def test_responses(self): + self.assertEqual(client.responses[client.NOT_FOUND], "Not Found") + + def test_client_constants(self): + # Make sure we don't break backward compatibility with 3.4 + expected = [ + 'CONTINUE', + 'SWITCHING_PROTOCOLS', + 'PROCESSING', + 'OK', + 'CREATED', + 'ACCEPTED', + 'NON_AUTHORITATIVE_INFORMATION', + 'NO_CONTENT', + 'RESET_CONTENT', + 'PARTIAL_CONTENT', + 'MULTI_STATUS', + 'IM_USED', + 'MULTIPLE_CHOICES', + 'MOVED_PERMANENTLY', + 'FOUND', + 'SEE_OTHER', + 'NOT_MODIFIED', + 'USE_PROXY', + 'TEMPORARY_REDIRECT', + 'BAD_REQUEST', + 'UNAUTHORIZED', + 'PAYMENT_REQUIRED', + 'FORBIDDEN', + 'NOT_FOUND', + 'METHOD_NOT_ALLOWED', + 'NOT_ACCEPTABLE', + 'PROXY_AUTHENTICATION_REQUIRED', + 'REQUEST_TIMEOUT', + 'CONFLICT', + 'GONE', + 'LENGTH_REQUIRED', + 'PRECONDITION_FAILED', + 'REQUEST_ENTITY_TOO_LARGE', + 'REQUEST_URI_TOO_LONG', + 'UNSUPPORTED_MEDIA_TYPE', + 'REQUESTED_RANGE_NOT_SATISFIABLE', + 'EXPECTATION_FAILED', + 'UNPROCESSABLE_ENTITY', + 'LOCKED', + 'FAILED_DEPENDENCY', + 'UPGRADE_REQUIRED', + 'PRECONDITION_REQUIRED', + 'TOO_MANY_REQUESTS', + 'REQUEST_HEADER_FIELDS_TOO_LARGE', + 'INTERNAL_SERVER_ERROR', + 'NOT_IMPLEMENTED', + 'BAD_GATEWAY', + 'SERVICE_UNAVAILABLE', + 'GATEWAY_TIMEOUT', + 'HTTP_VERSION_NOT_SUPPORTED', + 'INSUFFICIENT_STORAGE', + 'NOT_EXTENDED', + 'NETWORK_AUTHENTICATION_REQUIRED', + ] + for const in expected: + with self.subTest(constant=const): + self.assertTrue(hasattr(client, const)) + + +class SourceAddressTest(TestCase): + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.source_port = support.find_unused_port() + self.serv.listen() + self.conn = None + + def tearDown(self): + if self.conn: + self.conn.close() + self.conn = None + self.serv.close() + self.serv = None + + def testHTTPConnectionSourceAddress(self): + self.conn = client.HTTPConnection(HOST, self.port, + source_address=('', self.source_port)) + self.conn.connect() + self.assertEqual(self.conn.sock.getsockname()[1], self.source_port) + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not defined') + def testHTTPSConnectionSourceAddress(self): + self.conn = client.HTTPSConnection(HOST, self.port, + source_address=('', self.source_port)) + # We don't test anything here other than the constructor not barfing as + # this code doesn't deal with setting up an active running SSL server + # for an ssl_wrapped connect() to actually return from. + + +class TimeoutTest(TestCase): + PORT = None + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + TimeoutTest.PORT = support.bind_port(self.serv) + self.serv.listen() + + def tearDown(self): + self.serv.close() + self.serv = None + + def testTimeoutAttribute(self): + # This will prove that the timeout gets through HTTPConnection + # and into the socket. + + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + # no timeout -- do not use global socket default + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, + timeout=None) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), None) + httpConn.close() + + # a value + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30) + httpConn.connect() + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + +class PersistenceTest(TestCase): + + def test_reuse_reconnect(self): + # Should reuse or reconnect depending on header from server + tests = ( + ('1.0', '', False), + ('1.0', 'Connection: keep-alive\r\n', True), + ('1.1', '', True), + ('1.1', 'Connection: close\r\n', False), + ('1.0', 'Connection: keep-ALIVE\r\n', True), + ('1.1', 'Connection: cloSE\r\n', False), + ) + for version, header, reuse in tests: + with self.subTest(version=version, header=header): + msg = ( + 'HTTP/{} 200 OK\r\n' + '{}' + 'Content-Length: 12\r\n' + '\r\n' + 'Dummy body\r\n' + ).format(version, header) + conn = FakeSocketHTTPConnection(msg) + self.assertIsNone(conn.sock) + conn.request('GET', '/open-connection') + with conn.getresponse() as response: + self.assertEqual(conn.sock is None, not reuse) + response.read() + self.assertEqual(conn.sock is None, not reuse) + self.assertEqual(conn.connections, 1) + conn.request('GET', '/subsequent-request') + self.assertEqual(conn.connections, 1 if reuse else 2) + + def test_disconnected(self): + + def make_reset_reader(text): + """Return BufferedReader that raises ECONNRESET at EOF""" + stream = io.BytesIO(text) + def readinto(buffer): + size = io.BytesIO.readinto(stream, buffer) + if size == 0: + raise ConnectionResetError() + return size + stream.readinto = readinto + return io.BufferedReader(stream) + + tests = ( + (io.BytesIO, client.RemoteDisconnected), + (make_reset_reader, ConnectionResetError), + ) + for stream_factory, exception in tests: + with self.subTest(exception=exception): + conn = FakeSocketHTTPConnection(b'', stream_factory) + conn.request('GET', '/eof-response') + self.assertRaises(exception, conn.getresponse) + self.assertIsNone(conn.sock) + # HTTPConnection.connect() should be automatically invoked + conn.request('GET', '/reconnect') + self.assertEqual(conn.connections, 2) + + def test_100_close(self): + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + # Missing final response + ) + conn.request('GET', '/', headers={'Expect': '100-continue'}) + self.assertRaises(client.RemoteDisconnected, conn.getresponse) + self.assertIsNone(conn.sock) + conn.request('GET', '/reconnect') + self.assertEqual(conn.connections, 2) + + +class HTTPSTest(TestCase): + + def setUp(self): + if not hasattr(client, 'HTTPSConnection'): + self.skipTest('ssl support required') + + def make_server(self, certfile): + from test.ssl_servers import make_https_server + return make_https_server(self, certfile=certfile) + + def test_attributes(self): + # simple test to check it's storing the timeout + h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) + self.assertEqual(h.timeout, 30) + + def test_networked(self): + # Default settings: requires a valid cert from a trusted CA + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + h = client.HTTPSConnection('self-signed.pythontest.net', 443) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_networked_noverification(self): + # Switch off cert verification + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl._create_unverified_context() + h = client.HTTPSConnection('self-signed.pythontest.net', 443, + context=context) + h.request('GET', '/') + resp = h.getresponse() + h.close() + self.assertIn('nginx', resp.getheader('server')) + resp.close() + + @support.system_must_validate_cert + def test_networked_trusted_by_default_cert(self): + # Default settings: requires a valid cert from a trusted CA + support.requires('network') + with support.transient_internet('www.python.org'): + h = client.HTTPSConnection('www.python.org', 443) + h.request('GET', '/') + resp = h.getresponse() + content_type = resp.getheader('content-type') + resp.close() + h.close() + self.assertIn('text/html', content_type) + + def test_networked_good_cert(self): + # We feed the server's cert as a validating cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_selfsigned_pythontestdotnet) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h.request('GET', '/') + resp = h.getresponse() + server_string = resp.getheader('server') + resp.close() + h.close() + self.assertIn('nginx', server_string) + + def test_networked_bad_cert(self): + # We feed a "CA" cert that is unrelated to the server's cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_unknown_cert(self): + # The custom cert isn't known to the default trust bundle + import ssl + server = self.make_server(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_good_hostname(self): + # The (valid) cert validates the HTTP hostname + import ssl + server = self.make_server(CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port, context=context) + self.addCleanup(h.close) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.addCleanup(resp.close) + self.assertEqual(resp.status, 404) + + def test_local_bad_hostname(self): + # The (valid) cert doesn't validate the HTTP hostname + import ssl + server = self.make_server(CERT_fakehostname) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(CERT_fakehostname) + h = client.HTTPSConnection('localhost', server.port, context=context) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # Same with explicit check_hostname=True + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # With check_hostname=False, the mismatching is ignored + context.check_hostname = False + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=False) + h.request('GET', '/nonexistent') + resp = h.getresponse() + resp.close() + h.close() + self.assertEqual(resp.status, 404) + # The context's check_hostname setting is used if one isn't passed to + # HTTPSConnection. + context.check_hostname = False + h = client.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + resp.close() + h.close() + # Passing check_hostname to HTTPSConnection should override the + # context's setting. + h = client.HTTPSConnection('localhost', server.port, context=context, + check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not available') + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPSConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:443", "www.python.org", 443), + ("www.python.org:", "www.python.org", 443), + ("www.python.org", "www.python.org", 443), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 443), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", + 443)): + c = client.HTTPSConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + +class RequestBodyTest(TestCase): + """Test cases where a request includes a message body.""" + + def setUp(self): + self.conn = client.HTTPConnection('example.com') + self.conn.sock = self.sock = FakeSocket("") + self.conn.sock = self.sock + + def get_headers_and_fp(self): + f = io.BytesIO(self.sock.data) + f.readline() # read the request line + message = client.parse_headers(f) + return message, f + + def test_manual_content_length(self): + # Set an incorrect content-length so that we can verify that + # it will not be over-ridden by the library. + self.conn.request("PUT", "/url", "body", + {"Content-Length": "42"}) + message, f = self.get_headers_and_fp() + self.assertEqual("42", message.get("content-length")) + self.assertEqual(4, len(f.read())) + + def test_ascii_body(self): + self.conn.request("PUT", "/url", "body") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("4", message.get("content-length")) + self.assertEqual(b'body', f.read()) + + def test_latin1_body(self): + self.conn.request("PUT", "/url", "body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_bytes_body(self): + self.conn.request("PUT", "/url", b"body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as f: + f.write("body") + with open(support.TESTFN) as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("4", message.get("content-length")) + self.assertEqual(b'body', f.read()) + + def test_binary_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "wb") as f: + f.write(b"body\xc1") + with open(support.TESTFN, "rb") as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + +class HTTPResponseTest(TestCase): + + def setUp(self): + body = "HTTP/1.1 200 Ok\r\nMy-Header: first-value\r\nMy-Header: \ + second-value\r\n\r\nText" + sock = FakeSocket(body) + self.resp = client.HTTPResponse(sock) + self.resp.begin() + + def test_getting_header(self): + header = self.resp.getheader('My-Header') + self.assertEqual(header, 'first-value, second-value') + + header = self.resp.getheader('My-Header', 'some default') + self.assertEqual(header, 'first-value, second-value') + + def test_getting_nonexistent_header_with_string_default(self): + header = self.resp.getheader('No-Such-Header', 'default-value') + self.assertEqual(header, 'default-value') + + def test_getting_nonexistent_header_with_iterable_default(self): + header = self.resp.getheader('No-Such-Header', ['default', 'values']) + self.assertEqual(header, 'default, values') + + header = self.resp.getheader('No-Such-Header', ('default', 'values')) + self.assertEqual(header, 'default, values') + + def test_getting_nonexistent_header_without_default(self): + header = self.resp.getheader('No-Such-Header') + self.assertEqual(header, None) + + def test_getting_header_defaultint(self): + header = self.resp.getheader('No-Such-Header',default=42) + self.assertEqual(header, 42) + +class TunnelTests(TestCase): + def setUp(self): + response_text = ( + 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n' # Reply to HEAD + 'Content-Length: 42\r\n\r\n' + ) + self.host = 'proxy.com' + self.conn = client.HTTPConnection(self.host) + self.conn._create_connection = self._create_connection(response_text) + + def tearDown(self): + self.conn.close() + + def _create_connection(self, response_text): + def create_connection(address, timeout=None, source_address=None): + return FakeSocket(response_text, host=address[0], port=address[1]) + return create_connection + + def test_set_tunnel_host_port_headers(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers) + + def test_disallow_set_tunnel_after_connect(self): + # Once connected, we shouldn't be able to tunnel anymore + self.conn.connect() + self.assertRaises(RuntimeError, self.conn.set_tunnel, + 'destination.com') + + def test_connect_with_tunnel(self): + self.conn.set_tunnel('destination.com') + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + # issue22095 + self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + # This test should be removed when CONNECT gets the HTTP/1.1 blessing + self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) + + def test_connect_put_request(self): + self.conn.set_tunnel('destination.com') + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + def test_tunnel_debuglog(self): + expected_header = 'X-Dummy: 1' + response_text = 'HTTP/1.0 200 OK\r\n{}\r\n\r\n'.format(expected_header) + + self.conn.set_debuglevel(1) + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + with support.captured_stdout() as output: + self.conn.request('PUT', '/', '') + lines = output.getvalue().splitlines() + self.assertIn('header: {}'.format(expected_header), lines) + + +@support.reap_threads +def test_main(verbose=None): + support.run_unittest(HeaderTests, OfflineTest, BasicTest, TimeoutTest, + PersistenceTest, + HTTPSTest, RequestBodyTest, SourceAddressTest, + HTTPResponseTest, ExtendedReadTest, + ExtendedReadTestChunked, TunnelTests) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/3.5pypy/test_httpservers.py b/src/greentest/3.5pypy/test_httpservers.py new file mode 100644 index 0000000..a21aa11 --- /dev/null +++ b/src/greentest/3.5pypy/test_httpservers.py @@ -0,0 +1,1016 @@ +"""Unittests for the various HTTPServer modules. + +Written by Cody A.W. Somerville , +Josip Dzolonga, and Michael Otteneder for the 2007/08 GHOP contest. +""" + +from http.server import BaseHTTPRequestHandler, HTTPServer, \ + SimpleHTTPRequestHandler, CGIHTTPRequestHandler +from http import server, HTTPStatus + +import os +import sys +import re +import base64 +import ntpath +import shutil +import urllib.parse +import html +import http.client +import tempfile +from io import BytesIO + +import unittest +from test import support +threading = support.import_module('threading') + +class NoLogRequestHandler: + def log_message(self, *args): + # don't write log messages to stderr + pass + + def read(self, n=None): + return '' + + +class TestServerThread(threading.Thread): + def __init__(self, test_object, request_handler): + threading.Thread.__init__(self) + self.request_handler = request_handler + self.test_object = test_object + + def run(self): + self.server = HTTPServer(('localhost', 0), self.request_handler) + self.test_object.HOST, self.test_object.PORT = self.server.socket.getsockname() + self.test_object.server_started.set() + self.test_object = None + try: + self.server.serve_forever(0.05) + finally: + self.server.server_close() + + def stop(self): + self.server.shutdown() + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = support.threading_setup() + os.environ = support.EnvironmentVarGuard() + self.server_started = threading.Event() + self.thread = TestServerThread(self, self.request_handler) + self.thread.start() + self.server_started.wait() + + def tearDown(self): + self.thread.stop() + self.thread = None + os.environ.__exit__() + support.threading_cleanup(*self._threads) + + def request(self, uri, method='GET', body=None, headers={}): + self.connection = http.client.HTTPConnection(self.HOST, self.PORT) + self.connection.request(method, uri, body, headers) + return self.connection.getresponse() + + +class BaseHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + default_request_version = 'HTTP/1.1' + + def do_TEST(self): + self.send_response(HTTPStatus.NO_CONTENT) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'close') + self.end_headers() + + def do_KEEP(self): + self.send_response(HTTPStatus.NO_CONTENT) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'keep-alive') + self.end_headers() + + def do_KEYERROR(self): + self.send_error(999) + + def do_NOTFOUND(self): + self.send_error(HTTPStatus.NOT_FOUND) + + def do_EXPLAINERROR(self): + self.send_error(999, "Short Message", + "This is a long \n explanation") + + def do_CUSTOM(self): + self.send_response(999) + self.send_header('Content-Type', 'text/html') + self.send_header('Connection', 'close') + self.end_headers() + + def do_LATINONEHEADER(self): + self.send_response(999) + self.send_header('X-Special', 'Dängerous Mind') + self.send_header('Connection', 'close') + self.end_headers() + body = self.headers['x-special-incoming'].encode('utf-8') + self.wfile.write(body) + + def do_SEND_ERROR(self): + self.send_error(int(self.path[1:])) + + def do_HEAD(self): + self.send_error(int(self.path[1:])) + + def setUp(self): + BaseTestCase.setUp(self) + self.con = http.client.HTTPConnection(self.HOST, self.PORT) + self.con.connect() + + def test_command(self): + self.con.request('GET', '/') + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.NOT_IMPLEMENTED) + + def test_request_line_trimming(self): + self.con._http_vsn_str = 'HTTP/1.1\n' + self.con.putrequest('XYZBOGUS', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.NOT_IMPLEMENTED) + + def test_version_bogus(self): + self.con._http_vsn_str = 'FUBAR' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_version_digits(self): + self.con._http_vsn_str = 'HTTP/9.9.9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_version_none_get(self): + self.con._http_vsn_str = '' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.NOT_IMPLEMENTED) + + def test_version_none(self): + # Test that a valid method is rejected when not HTTP/1.x + self.con._http_vsn_str = '' + self.con.putrequest('CUSTOM', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_version_invalid(self): + self.con._http_vsn = 99 + self.con._http_vsn_str = 'HTTP/9.9' + self.con.putrequest('GET', '/') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.HTTP_VERSION_NOT_SUPPORTED) + + def test_send_blank(self): + self.con._http_vsn_str = '' + self.con.putrequest('', '') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.BAD_REQUEST) + + def test_header_close(self): + self.con.putrequest('GET', '/') + self.con.putheader('Connection', 'close') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.NOT_IMPLEMENTED) + + def test_head_keep_alive(self): + self.con._http_vsn_str = 'HTTP/1.1' + self.con.putrequest('GET', '/') + self.con.putheader('Connection', 'keep-alive') + self.con.endheaders() + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.NOT_IMPLEMENTED) + + def test_handler(self): + self.con.request('TEST', '/') + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.NO_CONTENT) + + def test_return_header_keep_alive(self): + self.con.request('KEEP', '/') + res = self.con.getresponse() + self.assertEqual(res.getheader('Connection'), 'keep-alive') + self.con.request('TEST', '/') + self.addCleanup(self.con.close) + + def test_internal_key_error(self): + self.con.request('KEYERROR', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 999) + + def test_return_custom_status(self): + self.con.request('CUSTOM', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 999) + + def test_return_explain_error(self): + self.con.request('EXPLAINERROR', '/') + res = self.con.getresponse() + self.assertEqual(res.status, 999) + self.assertTrue(int(res.getheader('Content-Length'))) + + def test_latin1_header(self): + self.con.request('LATINONEHEADER', '/', headers={ + 'X-Special-Incoming': 'Ärger mit Unicode' + }) + res = self.con.getresponse() + self.assertEqual(res.getheader('X-Special'), 'Dängerous Mind') + self.assertEqual(res.read(), 'Ärger mit Unicode'.encode('utf-8')) + + def test_error_content_length(self): + # Issue #16088: standard error responses should have a content-length + self.con.request('NOTFOUND', '/') + res = self.con.getresponse() + self.assertEqual(res.status, HTTPStatus.NOT_FOUND) + + data = res.read() + self.assertEqual(int(res.getheader('Content-Length')), len(data)) + + def test_send_error(self): + allow_transfer_encoding_codes = (HTTPStatus.NOT_MODIFIED, + HTTPStatus.RESET_CONTENT) + for code in (HTTPStatus.NO_CONTENT, HTTPStatus.NOT_MODIFIED, + HTTPStatus.PROCESSING, HTTPStatus.RESET_CONTENT, + HTTPStatus.SWITCHING_PROTOCOLS): + self.con.request('SEND_ERROR', '/{}'.format(code)) + res = self.con.getresponse() + self.assertEqual(code, res.status) + self.assertEqual(None, res.getheader('Content-Length')) + self.assertEqual(None, res.getheader('Content-Type')) + if code not in allow_transfer_encoding_codes: + self.assertEqual(None, res.getheader('Transfer-Encoding')) + + data = res.read() + self.assertEqual(b'', data) + + def test_head_via_send_error(self): + allow_transfer_encoding_codes = (HTTPStatus.NOT_MODIFIED, + HTTPStatus.RESET_CONTENT) + for code in (HTTPStatus.OK, HTTPStatus.NO_CONTENT, + HTTPStatus.NOT_MODIFIED, HTTPStatus.RESET_CONTENT, + HTTPStatus.SWITCHING_PROTOCOLS): + self.con.request('HEAD', '/{}'.format(code)) + res = self.con.getresponse() + self.assertEqual(code, res.status) + if code == HTTPStatus.OK: + self.assertTrue(int(res.getheader('Content-Length')) > 0) + self.assertIn('text/html', res.getheader('Content-Type')) + else: + self.assertEqual(None, res.getheader('Content-Length')) + self.assertEqual(None, res.getheader('Content-Type')) + if code not in allow_transfer_encoding_codes: + self.assertEqual(None, res.getheader('Transfer-Encoding')) + + data = res.read() + self.assertEqual(b'', data) + + +class RequestHandlerLoggingTestCase(BaseTestCase): + class request_handler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + default_request_version = 'HTTP/1.1' + + def do_GET(self): + self.send_response(HTTPStatus.OK) + self.end_headers() + + def do_ERROR(self): + self.send_error(HTTPStatus.NOT_FOUND, 'File not found') + + def test_get(self): + self.con = http.client.HTTPConnection(self.HOST, self.PORT) + self.con.connect() + + with support.captured_stderr() as err: + self.con.request('GET', '/') + with self.con.getresponse(): + pass + + self.assertTrue( + err.getvalue().endswith('"GET / HTTP/1.1" 200 -\n')) + + def test_err(self): + self.con = http.client.HTTPConnection(self.HOST, self.PORT) + self.con.connect() + + with support.captured_stderr() as err: + self.con.request('ERROR', '/') + with self.con.getresponse(): + pass + + lines = err.getvalue().split('\n') + self.assertTrue(lines[0].endswith('code 404, message File not found')) + self.assertTrue(lines[1].endswith('"ERROR / HTTP/1.1" 404 -')) + + +class SimpleHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, SimpleHTTPRequestHandler): + pass + + def setUp(self): + BaseTestCase.setUp(self) + self.cwd = os.getcwd() + basetempdir = tempfile.gettempdir() + os.chdir(basetempdir) + self.data = b'We are the knights who say Ni!' + self.tempdir = tempfile.mkdtemp(dir=basetempdir) + self.tempdir_name = os.path.basename(self.tempdir) + self.base_url = '/' + self.tempdir_name + with open(os.path.join(self.tempdir, 'test'), 'wb') as temp: + temp.write(self.data) + + def tearDown(self): + try: + os.chdir(self.cwd) + try: + shutil.rmtree(self.tempdir) + except: + pass + finally: + BaseTestCase.tearDown(self) + + def check_status_and_reason(self, response, status, data=None): + def close_conn(): + """Don't close reader yet so we can check if there was leftover + buffered input""" + nonlocal reader + reader = response.fp + response.fp = None + reader = None + response._close_conn = close_conn + + body = response.read() + self.assertTrue(response) + self.assertEqual(response.status, status) + self.assertIsNotNone(response.reason) + if data: + self.assertEqual(data, body) + # Ensure the server has not set up a persistent connection, and has + # not sent any extra data + self.assertEqual(response.version, 10) + self.assertEqual(response.msg.get("Connection", "close"), "close") + self.assertEqual(reader.read(30), b'', 'Connection should be closed') + + reader.close() + return body + + @support.requires_mac_ver(10, 5) + @unittest.skipUnless(support.TESTFN_UNDECODABLE, + 'need support.TESTFN_UNDECODABLE') + def test_undecodable_filename(self): + enc = sys.getfilesystemencoding() + filename = os.fsdecode(support.TESTFN_UNDECODABLE) + '.txt' + with open(os.path.join(self.tempdir, filename), 'wb') as f: + f.write(support.TESTFN_UNDECODABLE) + response = self.request(self.base_url + '/') + if sys.platform == 'darwin': + # On Mac OS the HFS+ filesystem replaces bytes that aren't valid + # UTF-8 into a percent-encoded value. + for name in os.listdir(self.tempdir): + if name != 'test': # Ignore a filename created in setUp(). + filename = name + break + body = self.check_status_and_reason(response, HTTPStatus.OK) + quotedname = urllib.parse.quote(filename, errors='surrogatepass') + self.assertIn(('href="%s"' % quotedname) + .encode(enc, 'surrogateescape'), body) + self.assertIn(('>%s<' % html.escape(filename)) + .encode(enc, 'surrogateescape'), body) + response = self.request(self.base_url + '/' + quotedname) + self.check_status_and_reason(response, HTTPStatus.OK, + data=support.TESTFN_UNDECODABLE) + + def test_get(self): + #constructs the path relative to the root directory of the HTTPServer + response = self.request(self.base_url + '/test') + self.check_status_and_reason(response, HTTPStatus.OK, data=self.data) + # check for trailing "/" which should return 404. See Issue17324 + response = self.request(self.base_url + '/test/') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + response = self.request(self.base_url + '/?hi=2') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '?hi=1') + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), + self.base_url + "/?hi=1") + response = self.request('/ThisDoesNotExist') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request('/' + 'ThisDoesNotExist' + '/') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + + data = b"Dummy index file\r\n" + with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: + f.write(data) + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, HTTPStatus.OK, data) + + # chmod() doesn't work as expected on Windows, and filesystem + # permissions are ignored by root on Unix. + if os.name == 'posix' and os.geteuid() != 0: + os.chmod(self.tempdir, 0) + try: + response = self.request(self.base_url + '/') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + finally: + os.chmod(self.tempdir, 0o755) + + def test_head(self): + response = self.request( + self.base_url + '/test', method='HEAD') + self.check_status_and_reason(response, HTTPStatus.OK) + self.assertEqual(response.getheader('content-length'), + str(len(self.data))) + self.assertEqual(response.getheader('content-type'), + 'application/octet-stream') + + def test_invalid_requests(self): + response = self.request('/', method='FOO') + self.check_status_and_reason(response, HTTPStatus.NOT_IMPLEMENTED) + # requests must be case sensitive,so this should fail too + response = self.request('/', method='custom') + self.check_status_and_reason(response, HTTPStatus.NOT_IMPLEMENTED) + response = self.request('/', method='GETs') + self.check_status_and_reason(response, HTTPStatus.NOT_IMPLEMENTED) + + def test_path_without_leading_slash(self): + response = self.request(self.tempdir_name + '/test') + self.check_status_and_reason(response, HTTPStatus.OK, data=self.data) + response = self.request(self.tempdir_name + '/test/') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.tempdir_name + '/') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.tempdir_name) + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + response = self.request(self.tempdir_name + '/?hi=2') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.tempdir_name + '?hi=1') + self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), + self.tempdir_name + "/?hi=1") + + +cgi_file1 = """\ +#!%s + +print("Content-type: text/html") +print() +print("Hello World") +""" + +cgi_file2 = """\ +#!%s +import cgi + +print("Content-type: text/html") +print() + +form = cgi.FieldStorage() +print("%%s, %%s, %%s" %% (form.getfirst("spam"), form.getfirst("eggs"), + form.getfirst("bacon"))) +""" + +cgi_file4 = """\ +#!%s +import os + +print("Content-type: text/html") +print() + +print(os.environ["%s"]) +""" + + +@unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, + "This test can't be run reliably as root (issue #13308).") +class CGIHTTPServerTestCase(BaseTestCase): + class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): + pass + + linesep = os.linesep.encode('ascii') + + def setUp(self): + BaseTestCase.setUp(self) + self.cwd = os.getcwd() + self.parent_dir = tempfile.mkdtemp() + self.cgi_dir = os.path.join(self.parent_dir, 'cgi-bin') + self.cgi_child_dir = os.path.join(self.cgi_dir, 'child-dir') + os.mkdir(self.cgi_dir) + os.mkdir(self.cgi_child_dir) + self.nocgi_path = None + self.file1_path = None + self.file2_path = None + self.file3_path = None + self.file4_path = None + + # The shebang line should be pure ASCII: use symlink if possible. + # See issue #7668. + if support.can_symlink(): + self.pythonexe = os.path.join(self.parent_dir, 'python') + os.symlink(sys.executable, self.pythonexe) + else: + self.pythonexe = sys.executable + + try: + # The python executable path is written as the first line of the + # CGI Python script. The encoding cookie cannot be used, and so the + # path should be encodable to the default script encoding (utf-8) + self.pythonexe.encode('utf-8') + except UnicodeEncodeError: + self.tearDown() + self.skipTest("Python executable path is not encodable to utf-8") + + self.nocgi_path = os.path.join(self.parent_dir, 'nocgi.py') + with open(self.nocgi_path, 'w') as fp: + fp.write(cgi_file1 % self.pythonexe) + os.chmod(self.nocgi_path, 0o777) + + self.file1_path = os.path.join(self.cgi_dir, 'file1.py') + with open(self.file1_path, 'w', encoding='utf-8') as file1: + file1.write(cgi_file1 % self.pythonexe) + os.chmod(self.file1_path, 0o777) + + self.file2_path = os.path.join(self.cgi_dir, 'file2.py') + with open(self.file2_path, 'w', encoding='utf-8') as file2: + file2.write(cgi_file2 % self.pythonexe) + os.chmod(self.file2_path, 0o777) + + self.file3_path = os.path.join(self.cgi_child_dir, 'file3.py') + with open(self.file3_path, 'w', encoding='utf-8') as file3: + file3.write(cgi_file1 % self.pythonexe) + os.chmod(self.file3_path, 0o777) + + self.file4_path = os.path.join(self.cgi_dir, 'file4.py') + with open(self.file4_path, 'w', encoding='utf-8') as file4: + file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING')) + os.chmod(self.file4_path, 0o777) + + os.chdir(self.parent_dir) + + def tearDown(self): + try: + os.chdir(self.cwd) + if self.pythonexe != sys.executable: + os.remove(self.pythonexe) + if self.nocgi_path: + os.remove(self.nocgi_path) + if self.file1_path: + os.remove(self.file1_path) + if self.file2_path: + os.remove(self.file2_path) + if self.file3_path: + os.remove(self.file3_path) + if self.file4_path: + os.remove(self.file4_path) + os.rmdir(self.cgi_child_dir) + os.rmdir(self.cgi_dir) + os.rmdir(self.parent_dir) + finally: + BaseTestCase.tearDown(self) + + def test_url_collapse_path(self): + # verify tail is the last portion and head is the rest on proper urls + test_vectors = { + '': '//', + '..': IndexError, + '/.//..': IndexError, + '/': '//', + '//': '//', + '/\\': '//\\', + '/.//': '//', + 'cgi-bin/file1.py': '/cgi-bin/file1.py', + '/cgi-bin/file1.py': '/cgi-bin/file1.py', + 'a': '//a', + '/a': '//a', + '//a': '//a', + './a': '//a', + './C:/': '/C:/', + '/a/b': '/a/b', + '/a/b/': '/a/b/', + '/a/b/.': '/a/b/', + '/a/b/c/..': '/a/b/', + '/a/b/c/../d': '/a/b/d', + '/a/b/c/../d/e/../f': '/a/b/d/f', + '/a/b/c/../d/e/../../f': '/a/b/f', + '/a/b/c/../d/e/.././././..//f': '/a/b/f', + '../a/b/c/../d/e/.././././..//f': IndexError, + '/a/b/c/../d/e/../../../f': '/a/f', + '/a/b/c/../d/e/../../../../f': '//f', + '/a/b/c/../d/e/../../../../../f': IndexError, + '/a/b/c/../d/e/../../../../f/..': '//', + '/a/b/c/../d/e/../../../../f/../.': '//', + } + for path, expected in test_vectors.items(): + if isinstance(expected, type) and issubclass(expected, Exception): + self.assertRaises(expected, + server._url_collapse_path, path) + else: + actual = server._url_collapse_path(path) + self.assertEqual(expected, actual, + msg='path = %r\nGot: %r\nWanted: %r' % + (path, actual, expected)) + + def test_headers_and_content(self): + res = self.request('/cgi-bin/file1.py') + self.assertEqual( + (res.read(), res.getheader('Content-type'), res.status), + (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK)) + + def test_issue19435(self): + res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh') + self.assertEqual(res.status, HTTPStatus.NOT_FOUND) + + def test_post(self): + params = urllib.parse.urlencode( + {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) + headers = {'Content-type' : 'application/x-www-form-urlencoded'} + res = self.request('/cgi-bin/file2.py', 'POST', params, headers) + + self.assertEqual(res.read(), b'1, python, 123456' + self.linesep) + + def test_invaliduri(self): + res = self.request('/cgi-bin/invalid') + res.read() + self.assertEqual(res.status, HTTPStatus.NOT_FOUND) + + def test_authorization(self): + headers = {b'Authorization' : b'Basic ' + + base64.b64encode(b'username:pass')} + res = self.request('/cgi-bin/file1.py', 'GET', headers=headers) + self.assertEqual( + (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_no_leading_slash(self): + # http://bugs.python.org/issue2254 + res = self.request('cgi-bin/file1.py') + self.assertEqual( + (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_os_environ_is_not_altered(self): + signature = "Test CGI Server" + os.environ['SERVER_SOFTWARE'] = signature + res = self.request('/cgi-bin/file1.py') + self.assertEqual( + (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), + (res.read(), res.getheader('Content-type'), res.status)) + self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) + + def test_urlquote_decoding_in_cgi_check(self): + res = self.request('/cgi-bin%2ffile1.py') + self.assertEqual( + (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_nested_cgi_path_issue21323(self): + res = self.request('/cgi-bin/child-dir/file3.py') + self.assertEqual( + (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_query_with_multiple_question_mark(self): + res = self.request('/cgi-bin/file4.py?a=b?c=d') + self.assertEqual( + (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK), + (res.read(), res.getheader('Content-type'), res.status)) + + def test_query_with_continuous_slashes(self): + res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//') + self.assertEqual( + (b'k=aa%2F%2Fbb&//q//p//=//a//b//' + self.linesep, + 'text/html', HTTPStatus.OK), + (res.read(), res.getheader('Content-type'), res.status)) + + +class SocketlessRequestHandler(SimpleHTTPRequestHandler): + def __init__(self): + self.get_called = False + self.protocol_version = "HTTP/1.1" + + def do_GET(self): + self.get_called = True + self.send_response(HTTPStatus.OK) + self.send_header('Content-Type', 'text/html') + self.end_headers() + self.wfile.write(b'Data\r\n') + + def log_message(self, format, *args): + pass + +class RejectingSocketlessRequestHandler(SocketlessRequestHandler): + def handle_expect_100(self): + self.send_error(HTTPStatus.EXPECTATION_FAILED) + return False + + +class AuditableBytesIO: + + def __init__(self): + self.datas = [] + + def write(self, data): + self.datas.append(data) + + def getData(self): + return b''.join(self.datas) + + @property + def numWrites(self): + return len(self.datas) + + +class BaseHTTPRequestHandlerTestCase(unittest.TestCase): + """Test the functionality of the BaseHTTPServer. + + Test the support for the Expect 100-continue header. + """ + + HTTPResponseMatch = re.compile(b'HTTP/1.[0-9]+ 200 OK') + + def setUp (self): + self.handler = SocketlessRequestHandler() + + def send_typical_request(self, message): + input = BytesIO(message) + output = BytesIO() + self.handler.rfile = input + self.handler.wfile = output + self.handler.handle_one_request() + output.seek(0) + return output.readlines() + + def verify_get_called(self): + self.assertTrue(self.handler.get_called) + + def verify_expected_headers(self, headers): + for fieldName in b'Server: ', b'Date: ', b'Content-Type: ': + self.assertEqual(sum(h.startswith(fieldName) for h in headers), 1) + + def verify_http_server_response(self, response): + match = self.HTTPResponseMatch.search(response) + self.assertIsNotNone(match) + + def test_http_1_1(self): + result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], b'Data\r\n') + self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') + self.assertEqual(self.handler.command, 'GET') + self.assertEqual(self.handler.path, '/') + self.assertEqual(self.handler.request_version, 'HTTP/1.1') + self.assertSequenceEqual(self.handler.headers.items(), ()) + + def test_http_1_0(self): + result = self.send_typical_request(b'GET / HTTP/1.0\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], b'Data\r\n') + self.assertEqual(self.handler.requestline, 'GET / HTTP/1.0') + self.assertEqual(self.handler.command, 'GET') + self.assertEqual(self.handler.path, '/') + self.assertEqual(self.handler.request_version, 'HTTP/1.0') + self.assertSequenceEqual(self.handler.headers.items(), ()) + + def test_http_0_9(self): + result = self.send_typical_request(b'GET / HTTP/0.9\r\n\r\n') + self.assertEqual(len(result), 1) + self.assertEqual(result[0], b'Data\r\n') + self.verify_get_called() + + def test_with_continue_1_0(self): + result = self.send_typical_request(b'GET / HTTP/1.0\r\nExpect: 100-continue\r\n\r\n') + self.verify_http_server_response(result[0]) + self.verify_expected_headers(result[1:-1]) + self.verify_get_called() + self.assertEqual(result[-1], b'Data\r\n') + self.assertEqual(self.handler.requestline, 'GET / HTTP/1.0') + self.assertEqual(self.handler.command, 'GET') + self.assertEqual(self.handler.path, '/') + self.assertEqual(self.handler.request_version, 'HTTP/1.0') + headers = (("Expect", "100-continue"),) + self.assertSequenceEqual(self.handler.headers.items(), headers) + + def test_with_continue_1_1(self): + result = self.send_typical_request(b'GET / HTTP/1.1\r\nExpect: 100-continue\r\n\r\n') + self.assertEqual(result[0], b'HTTP/1.1 100 Continue\r\n') + self.assertEqual(result[1], b'\r\n') + self.assertEqual(result[2], b'HTTP/1.1 200 OK\r\n') + self.verify_expected_headers(result[2:-1]) + self.verify_get_called() + self.assertEqual(result[-1], b'Data\r\n') + self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') + self.assertEqual(self.handler.command, 'GET') + self.assertEqual(self.handler.path, '/') + self.assertEqual(self.handler.request_version, 'HTTP/1.1') + headers = (("Expect", "100-continue"),) + self.assertSequenceEqual(self.handler.headers.items(), headers) + + def test_header_buffering_of_send_error(self): + + input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' + handler.requestline = '' + handler.command = None + + handler.send_error(418) + self.assertEqual(output.numWrites, 2) + + def test_header_buffering_of_send_response_only(self): + + input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' + + handler.send_response_only(418) + self.assertEqual(output.numWrites, 0) + handler.end_headers() + self.assertEqual(output.numWrites, 1) + + def test_header_buffering_of_send_header(self): + + input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' + + handler.send_header('Foo', 'foo') + handler.send_header('bar', 'bar') + self.assertEqual(output.numWrites, 0) + handler.end_headers() + self.assertEqual(output.getData(), b'Foo: foo\r\nbar: bar\r\n\r\n') + self.assertEqual(output.numWrites, 1) + + def test_header_unbuffered_when_continue(self): + + def _readAndReseek(f): + pos = f.tell() + f.seek(0) + data = f.read() + f.seek(pos) + return data + + input = BytesIO(b'GET / HTTP/1.1\r\nExpect: 100-continue\r\n\r\n') + output = BytesIO() + self.handler.rfile = input + self.handler.wfile = output + self.handler.request_version = 'HTTP/1.1' + + self.handler.handle_one_request() + self.assertNotEqual(_readAndReseek(output), b'') + result = _readAndReseek(output).split(b'\r\n') + self.assertEqual(result[0], b'HTTP/1.1 100 Continue') + self.assertEqual(result[1], b'') + self.assertEqual(result[2], b'HTTP/1.1 200 OK') + + def test_with_continue_rejected(self): + usual_handler = self.handler # Save to avoid breaking any subsequent tests. + self.handler = RejectingSocketlessRequestHandler() + result = self.send_typical_request(b'GET / HTTP/1.1\r\nExpect: 100-continue\r\n\r\n') + self.assertEqual(result[0], b'HTTP/1.1 417 Expectation Failed\r\n') + self.verify_expected_headers(result[1:-1]) + # The expect handler should short circuit the usual get method by + # returning false here, so get_called should be false + self.assertFalse(self.handler.get_called) + self.assertEqual(sum(r == b'Connection: close\r\n' for r in result[1:-1]), 1) + self.handler = usual_handler # Restore to avoid breaking any subsequent tests. + + def test_request_length(self): + # Issue #10714: huge request lines are discarded, to avoid Denial + # of Service attacks. + result = self.send_typical_request(b'GET ' + b'x' * 65537) + self.assertEqual(result[0], b'HTTP/1.1 414 Request-URI Too Long\r\n') + self.assertFalse(self.handler.get_called) + self.assertIsInstance(self.handler.requestline, str) + + def test_header_length(self): + # Issue #6791: same for headers + result = self.send_typical_request( + b'GET / HTTP/1.1\r\nX-Foo: bar' + b'r' * 65537 + b'\r\n\r\n') + self.assertEqual(result[0], b'HTTP/1.1 400 Line too long\r\n') + self.assertFalse(self.handler.get_called) + self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') + + def test_too_many_headers(self): + result = self.send_typical_request( + b'GET / HTTP/1.1\r\n' + b'X-Foo: bar\r\n' * 101 + b'\r\n') + self.assertEqual(result[0], b'HTTP/1.1 431 Too many headers\r\n') + self.assertFalse(self.handler.get_called) + self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') + + def test_close_connection(self): + # handle_one_request() should be repeatedly called until + # it sets close_connection + def handle_one_request(): + self.handler.close_connection = next(close_values) + self.handler.handle_one_request = handle_one_request + + close_values = iter((True,)) + self.handler.handle() + self.assertRaises(StopIteration, next, close_values) + + close_values = iter((False, False, True)) + self.handler.handle() + self.assertRaises(StopIteration, next, close_values) + +class SimpleHTTPRequestHandlerTestCase(unittest.TestCase): + """ Test url parsing """ + def setUp(self): + self.translated = os.getcwd() + self.translated = os.path.join(self.translated, 'filename') + self.handler = SocketlessRequestHandler() + + def test_query_arguments(self): + path = self.handler.translate_path('/filename') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('/filename?foo=bar') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('/filename?a=b&spam=eggs#zot') + self.assertEqual(path, self.translated) + + def test_start_with_double_slash(self): + path = self.handler.translate_path('//filename') + self.assertEqual(path, self.translated) + path = self.handler.translate_path('//filename?foo=bar') + self.assertEqual(path, self.translated) + + def test_windows_colon(self): + with support.swap_attr(server.os, 'path', ntpath): + path = self.handler.translate_path('c:c:c:foo/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('\\c:../filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('c:\\c:..\\foo/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + path = self.handler.translate_path('c:c:foo\\c:c:bar/filename') + path = path.replace(ntpath.sep, os.sep) + self.assertEqual(path, self.translated) + + +class MiscTestCase(unittest.TestCase): + def test_all(self): + expected = [] + blacklist = {'executable', 'nobody_uid', 'test'} + for name in dir(server): + if name.startswith('_') or name in blacklist: + continue + module_object = getattr(server, name) + if getattr(module_object, '__module__', None) == 'http.server': + expected.append(name) + self.assertCountEqual(server.__all__, expected) + + +def test_main(verbose=None): + cwd = os.getcwd() + try: + support.run_unittest( + RequestHandlerLoggingTestCase, + BaseHTTPRequestHandlerTestCase, + BaseHTTPServerTestCase, + SimpleHTTPServerTestCase, + CGIHTTPServerTestCase, + SimpleHTTPRequestHandlerTestCase, + MiscTestCase, + ) + finally: + os.chdir(cwd) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/3.5pypy/test_queue.py b/src/greentest/3.5pypy/test_queue.py new file mode 100644 index 0000000..4ccaa39 --- /dev/null +++ b/src/greentest/3.5pypy/test_queue.py @@ -0,0 +1,358 @@ +# Some simple queue module tests, plus some failure conditions +# to ensure the Queue locks remain stable. +import queue +import time +import unittest +from test import support +threading = support.import_module('threading') + +QUEUE_SIZE = 5 + +def qfull(q): + return q.maxsize > 0 and q.qsize() == q.maxsize + +# A thread to run a function that unclogs a blocked Queue. +class _TriggerThread(threading.Thread): + def __init__(self, fn, args): + self.fn = fn + self.args = args + self.startedEvent = threading.Event() + threading.Thread.__init__(self) + + def run(self): + # The sleep isn't necessary, but is intended to give the blocking + # function in the main thread a chance at actually blocking before + # we unclog it. But if the sleep is longer than the timeout-based + # tests wait in their blocking functions, those tests will fail. + # So we give them much longer timeout values compared to the + # sleep here (I aimed at 10 seconds for blocking functions -- + # they should never actually wait that long - they should make + # progress as soon as we call self.fn()). + time.sleep(0.1) + self.startedEvent.set() + self.fn(*self.args) + + +# Execute a function that blocks, and in a separate thread, a function that +# triggers the release. Returns the result of the blocking function. Caution: +# block_func must guarantee to block until trigger_func is called, and +# trigger_func must guarantee to change queue state so that block_func can make +# enough progress to return. In particular, a block_func that just raises an +# exception regardless of whether trigger_func is called will lead to +# timing-dependent sporadic failures, and one of those went rarely seen but +# undiagnosed for years. Now block_func must be unexceptional. If block_func +# is supposed to raise an exception, call do_exceptional_blocking_test() +# instead. + +class BlockingTestMixin: + + def tearDown(self): + self.t = None + + def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): + self.t = _TriggerThread(trigger_func, trigger_args) + self.t.start() + self.result = block_func(*block_args) + # If block_func returned before our thread made the call, we failed! + if not self.t.startedEvent.is_set(): + self.fail("blocking function '%r' appeared not to block" % + block_func) + self.t.join(10) # make sure the thread terminates + if self.t.is_alive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) + return self.result + + # Call this instead if block_func is supposed to raise an exception. + def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, + trigger_args, expected_exception_class): + self.t = _TriggerThread(trigger_func, trigger_args) + self.t.start() + try: + try: + block_func(*block_args) + except expected_exception_class: + raise + else: + self.fail("expected exception of kind %r" % + expected_exception_class) + finally: + self.t.join(10) # make sure the thread terminates + if self.t.is_alive(): + self.fail("trigger function '%r' appeared to not return" % + trigger_func) + if not self.t.startedEvent.is_set(): + self.fail("trigger thread ended but event never set") + + +class BaseQueueTestMixin(BlockingTestMixin): + def setUp(self): + self.cum = 0 + self.cumlock = threading.Lock() + + def simple_queue_test(self, q): + if q.qsize(): + raise RuntimeError("Call this function with an empty queue") + self.assertTrue(q.empty()) + self.assertFalse(q.full()) + # I guess we better check things actually queue correctly a little :) + q.put(111) + q.put(333) + q.put(222) + target_order = dict(Queue = [111, 333, 222], + LifoQueue = [222, 333, 111], + PriorityQueue = [111, 222, 333]) + actual_order = [q.get(), q.get(), q.get()] + self.assertEqual(actual_order, target_order[q.__class__.__name__], + "Didn't seem to queue the correct data!") + for i in range(QUEUE_SIZE-1): + q.put(i) + self.assertTrue(q.qsize(), "Queue should not be empty") + self.assertTrue(not qfull(q), "Queue should not be full") + last = 2 * QUEUE_SIZE + full = 3 * 2 * QUEUE_SIZE + q.put(last) + self.assertTrue(qfull(q), "Queue should be full") + self.assertFalse(q.empty()) + self.assertTrue(q.full()) + try: + q.put(full, block=0) + self.fail("Didn't appear to block with a full queue") + except queue.Full: + pass + try: + q.put(full, timeout=0.01) + self.fail("Didn't appear to time-out with a full queue") + except queue.Full: + pass + # Test a blocking put + self.do_blocking_test(q.put, (full,), q.get, ()) + self.do_blocking_test(q.put, (full, True, 10), q.get, ()) + # Empty it + for i in range(QUEUE_SIZE): + q.get() + self.assertTrue(not q.qsize(), "Queue should be empty") + try: + q.get(block=0) + self.fail("Didn't appear to block with an empty queue") + except queue.Empty: + pass + try: + q.get(timeout=0.01) + self.fail("Didn't appear to time-out with an empty queue") + except queue.Empty: + pass + # Test a blocking get + self.do_blocking_test(q.get, (), q.put, ('empty',)) + self.do_blocking_test(q.get, (True, 10), q.put, ('empty',)) + + + def worker(self, q): + while True: + x = q.get() + if x < 0: + q.task_done() + return + with self.cumlock: + self.cum += x + q.task_done() + + def queue_join_test(self, q): + self.cum = 0 + for i in (0,1): + threading.Thread(target=self.worker, args=(q,)).start() + for i in range(100): + q.put(i) + q.join() + self.assertEqual(self.cum, sum(range(100)), + "q.join() did not block until all tasks were done") + for i in (0,1): + q.put(-1) # instruct the threads to close + q.join() # verify that you can join twice + + def test_queue_task_done(self): + # Test to make sure a queue task completed successfully. + q = self.type2test() + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + + def test_queue_join(self): + # Test that a queue join()s successfully, and before anything else + # (done twice for insurance). + q = self.type2test() + self.queue_join_test(q) + self.queue_join_test(q) + try: + q.task_done() + except ValueError: + pass + else: + self.fail("Did not detect task count going negative") + + def test_simple_queue(self): + # Do it a couple of times on the same queue. + # Done twice to make sure works with same instance reused. + q = self.type2test(QUEUE_SIZE) + self.simple_queue_test(q) + self.simple_queue_test(q) + + def test_negative_timeout_raises_exception(self): + q = self.type2test(QUEUE_SIZE) + with self.assertRaises(ValueError): + q.put(1, timeout=-1) + with self.assertRaises(ValueError): + q.get(1, timeout=-1) + + def test_nowait(self): + q = self.type2test(QUEUE_SIZE) + for i in range(QUEUE_SIZE): + q.put_nowait(1) + with self.assertRaises(queue.Full): + q.put_nowait(1) + + for i in range(QUEUE_SIZE): + q.get_nowait() + with self.assertRaises(queue.Empty): + q.get_nowait() + + def test_shrinking_queue(self): + # issue 10110 + q = self.type2test(3) + q.put(1) + q.put(2) + q.put(3) + with self.assertRaises(queue.Full): + q.put_nowait(4) + self.assertEqual(q.qsize(), 3) + q.maxsize = 2 # shrink the queue + with self.assertRaises(queue.Full): + q.put_nowait(4) + +class QueueTest(BaseQueueTestMixin, unittest.TestCase): + type2test = queue.Queue + +class LifoQueueTest(BaseQueueTestMixin, unittest.TestCase): + type2test = queue.LifoQueue + +class PriorityQueueTest(BaseQueueTestMixin, unittest.TestCase): + type2test = queue.PriorityQueue + + + +# A Queue subclass that can provoke failure at a moment's notice :) +class FailingQueueException(Exception): + pass + +class FailingQueue(queue.Queue): + def __init__(self, *args): + self.fail_next_put = False + self.fail_next_get = False + queue.Queue.__init__(self, *args) + def _put(self, item): + if self.fail_next_put: + self.fail_next_put = False + raise FailingQueueException("You Lose") + return queue.Queue._put(self, item) + def _get(self): + if self.fail_next_get: + self.fail_next_get = False + raise FailingQueueException("You Lose") + return queue.Queue._get(self) + +class FailingQueueTest(BlockingTestMixin, unittest.TestCase): + + def failing_queue_test(self, q): + if q.qsize(): + raise RuntimeError("Call this function with an empty queue") + for i in range(QUEUE_SIZE-1): + q.put(i) + # Test a failing non-blocking put. + q.fail_next_put = True + try: + q.put("oops", block=0) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + q.fail_next_put = True + try: + q.put("oops", timeout=0.1) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + q.put("last") + self.assertTrue(qfull(q), "Queue should be full") + # Test a failing blocking put + q.fail_next_put = True + try: + self.do_blocking_test(q.put, ("full",), q.get, ()) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # Check the Queue isn't damaged. + # put failed, but get succeeded - re-add + q.put("last") + # Test a failing timeout put + q.fail_next_put = True + try: + self.do_exceptional_blocking_test(q.put, ("full", True, 10), q.get, (), + FailingQueueException) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # Check the Queue isn't damaged. + # put failed, but get succeeded - re-add + q.put("last") + self.assertTrue(qfull(q), "Queue should be full") + q.get() + self.assertTrue(not qfull(q), "Queue should not be full") + q.put("last") + self.assertTrue(qfull(q), "Queue should be full") + # Test a blocking put + self.do_blocking_test(q.put, ("full",), q.get, ()) + # Empty it + for i in range(QUEUE_SIZE): + q.get() + self.assertTrue(not q.qsize(), "Queue should be empty") + q.put("first") + q.fail_next_get = True + try: + q.get() + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + self.assertTrue(q.qsize(), "Queue should not be empty") + q.fail_next_get = True + try: + q.get(timeout=0.1) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + self.assertTrue(q.qsize(), "Queue should not be empty") + q.get() + self.assertTrue(not q.qsize(), "Queue should be empty") + q.fail_next_get = True + try: + self.do_exceptional_blocking_test(q.get, (), q.put, ('empty',), + FailingQueueException) + self.fail("The queue didn't fail when it should have") + except FailingQueueException: + pass + # put succeeded, but get failed. + self.assertTrue(q.qsize(), "Queue should not be empty") + q.get() + self.assertTrue(not q.qsize(), "Queue should be empty") + + def test_failing_queue(self): + # Test to make sure a queue is functioning correctly. + # Done twice to the same instance. + q = FailingQueue(QUEUE_SIZE) + self.failing_queue_test(q) + self.failing_queue_test(q) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_select.py b/src/greentest/3.5pypy/test_select.py new file mode 100644 index 0000000..d2579e9 --- /dev/null +++ b/src/greentest/3.5pypy/test_select.py @@ -0,0 +1,92 @@ +import errno +import os +import select +import sys +import unittest +from test import support + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + try: + select.select([fd], [], [], 0) + except OSError as err: + self.assertEqual(err.errno, errno.EBADF) + else: + self.fail("exception not raised") + + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + def test_select(self): + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print('timeout =', tout) + rfd, wfd, xfd = select.select([p], [], [], tout) + if (rfd, wfd, xfd) == ([], [], []): + continue + if (rfd, wfd, xfd) == ([p], [], []): + line = p.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select():', rfd, wfd, xfd) + p.close() + + # Issue 16230: Crash on select resized list + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + result = select.select([], a, []) + # CPython: 'a' ends up with 5 items, because each fileno() + # removes an item and at the middle the iteration stops. + # PyPy: 'a' ends up empty, because the iteration is done on + # a copy of the original list: fileno() is called 10 times. + if support.check_impl_detail(cpython=True): + self.assertEqual(len(result[1]), 5) + self.assertEqual(len(a), 5) + if support.check_impl_detail(pypy=True): + self.assertEqual(len(result[1]), 10) + self.assertEqual(len(a), 0) + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_signal.py b/src/greentest/3.5pypy/test_signal.py new file mode 100644 index 0000000..e397a28 --- /dev/null +++ b/src/greentest/3.5pypy/test_signal.py @@ -0,0 +1,1110 @@ +import unittest +from test import support +from contextlib import closing +import enum +import gc +import pickle +import select +import signal +import socket +import struct +import subprocess +import traceback +import sys, os, time, errno +from test.support.script_helper import assert_python_ok, spawn_python +try: + import threading +except ImportError: + threading = None +try: + import _testcapi +except ImportError: + _testcapi = None + + +class HandlerBCalled(Exception): + pass + + +def exit_subprocess(): + """Use os._exit(0) to exit the current subprocess. + + Otherwise, the test catches the SystemExit and continues executing + in parallel with the original test, so you wind up with an + exponential number of tests running concurrently. + """ + os._exit(0) + + +def ignoring_eintr(__func, *args, **kwargs): + try: + return __func(*args, **kwargs) + except OSError as e: + if e.errno != errno.EINTR: + raise + return None + + +class GenericTests(unittest.TestCase): + + @unittest.skipIf(threading is None, "test needs threading module") + def test_enums(self): + for name in dir(signal): + sig = getattr(signal, name) + if name in {'SIG_DFL', 'SIG_IGN'}: + self.assertIsInstance(sig, signal.Handlers) + elif name in {'SIG_BLOCK', 'SIG_UNBLOCK', 'SIG_SETMASK'}: + self.assertIsInstance(sig, signal.Sigmasks) + elif name.startswith('SIG') and not name.startswith('SIG_'): + self.assertIsInstance(sig, signal.Signals) + elif name.startswith('CTRL_'): + self.assertIsInstance(sig, signal.Signals) + self.assertEqual(sys.platform, "win32") + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class InterProcessSignalTests(unittest.TestCase): + MAX_DURATION = 20 # Entire test should last at most 20 sec. + + def setUp(self): + self.using_gc = gc.isenabled() + gc.disable() + + def tearDown(self): + if self.using_gc: + gc.enable() + + def format_frame(self, frame, limit=None): + return ''.join(traceback.format_stack(frame, limit=limit)) + + def handlerA(self, signum, frame): + self.a_called = True + + def handlerB(self, signum, frame): + self.b_called = True + raise HandlerBCalled(signum, self.format_frame(frame)) + + def wait(self, child): + """Wait for child to finish, ignoring EINTR.""" + while True: + try: + child.wait() + return + except OSError as e: + if e.errno != errno.EINTR: + raise + + def run_test(self): + # Install handlers. This function runs in a sub-process, so we + # don't worry about re-setting the default handlers. + signal.signal(signal.SIGHUP, self.handlerA) + signal.signal(signal.SIGUSR1, self.handlerB) + signal.signal(signal.SIGUSR2, signal.SIG_IGN) + signal.signal(signal.SIGALRM, signal.default_int_handler) + + # Variables the signals will modify: + self.a_called = False + self.b_called = False + + # Let the sub-processes know who to send signals to. + pid = os.getpid() + + child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)]) + if child: + self.wait(child) + if not self.a_called: + time.sleep(1) # Give the signal time to be delivered. + self.assertTrue(self.a_called) + self.assertFalse(self.b_called) + self.a_called = False + + # Make sure the signal isn't delivered while the previous + # Popen object is being destroyed, because __del__ swallows + # exceptions. + del child + try: + child = subprocess.Popen(['kill', '-USR1', str(pid)]) + # This wait should be interrupted by the signal's exception. + self.wait(child) + time.sleep(1) # Give the signal time to be delivered. + self.fail('HandlerBCalled exception not raised') + except HandlerBCalled: + self.assertTrue(self.b_called) + self.assertFalse(self.a_called) + + child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)]) + if child: + self.wait(child) # Nothing should happen. + + try: + signal.alarm(1) + # The race condition in pause doesn't matter in this case, + # since alarm is going to raise a KeyboardException, which + # will skip the call. + signal.pause() + # But if another signal arrives before the alarm, pause + # may return early. + time.sleep(1) + except KeyboardInterrupt: + pass + except: + self.fail("Some other exception woke us from pause: %s" % + traceback.format_exc()) + else: + self.fail("pause returned of its own accord, and the signal" + " didn't arrive after another second.") + + # Issue 3864, unknown if this affects earlier versions of freebsd also + @unittest.skipIf(sys.platform=='freebsd6', + 'inter process signals not reliable (do not mix well with threading) ' + 'on freebsd6') + def test_main(self): + # This function spawns a child process to insulate the main + # test-running process from all the signals. It then + # communicates with that child process over a pipe and + # re-raises information about any exceptions the child + # raises. The real work happens in self.run_test(). + os_done_r, os_done_w = os.pipe() + with closing(os.fdopen(os_done_r, 'rb')) as done_r, \ + closing(os.fdopen(os_done_w, 'wb')) as done_w: + child = os.fork() + if child == 0: + # In the child process; run the test and report results + # through the pipe. + try: + done_r.close() + # Have to close done_w again here because + # exit_subprocess() will skip the enclosing with block. + with closing(done_w): + try: + self.run_test() + except: + pickle.dump(traceback.format_exc(), done_w) + else: + pickle.dump(None, done_w) + except: + print('Uh oh, raised from pickle.') + traceback.print_exc() + finally: + exit_subprocess() + + done_w.close() + # Block for up to MAX_DURATION seconds for the test to finish. + r, w, x = select.select([done_r], [], [], self.MAX_DURATION) + if done_r in r: + tb = pickle.load(done_r) + if tb: + self.fail(tb) + else: + os.kill(child, signal.SIGKILL) + self.fail('Test deadlocked after %d seconds.' % + self.MAX_DURATION) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class PosixTests(unittest.TestCase): + def trivial_signal_handler(self, *args): + pass + + def test_out_of_range_signal_number_raises_error(self): + self.assertRaises(ValueError, signal.getsignal, 4242) + + self.assertRaises(ValueError, signal.signal, 4242, + self.trivial_signal_handler) + + def test_setting_signal_handler_to_none_raises_error(self): + self.assertRaises(TypeError, signal.signal, + signal.SIGUSR1, None) + + def test_getsignal(self): + hup = signal.signal(signal.SIGHUP, self.trivial_signal_handler) + self.assertIsInstance(hup, signal.Handlers) + self.assertEqual(signal.getsignal(signal.SIGHUP), + self.trivial_signal_handler) + signal.signal(signal.SIGHUP, hup) + self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + + +@unittest.skipUnless(sys.platform == "win32", "Windows specific") +class WindowsSignalTests(unittest.TestCase): + def test_issue9324(self): + # Updated for issue #10003, adding SIGBREAK + handler = lambda x, y: None + checked = set() + for sig in (signal.SIGABRT, signal.SIGBREAK, signal.SIGFPE, + signal.SIGILL, signal.SIGINT, signal.SIGSEGV, + signal.SIGTERM): + # Set and then reset a handler for signals that work on windows. + # Issue #18396, only for signals without a C-level handler. + if signal.getsignal(sig) is not None: + signal.signal(sig, signal.signal(sig, handler)) + checked.add(sig) + # Issue #18396: Ensure the above loop at least tested *something* + self.assertTrue(checked) + + with self.assertRaises(ValueError): + signal.signal(-1, handler) + + with self.assertRaises(ValueError): + signal.signal(7, handler) + + +class WakeupFDTests(unittest.TestCase): + + def test_invalid_fd(self): + fd = support.make_bad_fd() + self.assertRaises((ValueError, OSError), + signal.set_wakeup_fd, fd) + + def test_invalid_socket(self): + sock = socket.socket() + fd = sock.fileno() + sock.close() + self.assertRaises((ValueError, OSError), + signal.set_wakeup_fd, fd) + + def test_set_wakeup_fd_result(self): + r1, w1 = os.pipe() + self.addCleanup(os.close, r1) + self.addCleanup(os.close, w1) + r2, w2 = os.pipe() + self.addCleanup(os.close, r2) + self.addCleanup(os.close, w2) + + if hasattr(os, 'set_blocking'): + os.set_blocking(w1, False) + os.set_blocking(w2, False) + + signal.set_wakeup_fd(w1) + self.assertEqual(signal.set_wakeup_fd(w2), w1) + self.assertEqual(signal.set_wakeup_fd(-1), w2) + self.assertEqual(signal.set_wakeup_fd(-1), -1) + + def test_set_wakeup_fd_socket_result(self): + sock1 = socket.socket() + self.addCleanup(sock1.close) + sock1.setblocking(False) + fd1 = sock1.fileno() + + sock2 = socket.socket() + self.addCleanup(sock2.close) + sock2.setblocking(False) + fd2 = sock2.fileno() + + signal.set_wakeup_fd(fd1) + self.assertEqual(signal.set_wakeup_fd(fd2), fd1) + self.assertEqual(signal.set_wakeup_fd(-1), fd2) + self.assertEqual(signal.set_wakeup_fd(-1), -1) + + # On Windows, files are always blocking and Windows does not provide a + # function to test if a socket is in non-blocking mode. + @unittest.skipIf(sys.platform == "win32", "tests specific to POSIX") + def test_set_wakeup_fd_blocking(self): + rfd, wfd = os.pipe() + self.addCleanup(os.close, rfd) + self.addCleanup(os.close, wfd) + + # fd must be non-blocking + os.set_blocking(wfd, True) + with self.assertRaises(ValueError) as cm: + signal.set_wakeup_fd(wfd) + self.assertEqual(str(cm.exception), + "the fd %s must be in non-blocking mode" % wfd) + + # non-blocking is ok + os.set_blocking(wfd, False) + signal.set_wakeup_fd(wfd) + signal.set_wakeup_fd(-1) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class WakeupSignalTests(unittest.TestCase): + @unittest.skipIf(_testcapi is None, 'need _testcapi') + def check_wakeup(self, test_body, *signals, ordered=True): + # use a subprocess to have only one thread + code = """if 1: + import _testcapi + import os + import signal + import struct + + signals = {!r} + + def handler(signum, frame): + pass + + def check_signum(signals): + data = os.read(read, len(signals)+1) + raised = struct.unpack('%uB' % len(data), data) + if not {!r}: + raised = set(raised) + signals = set(signals) + if raised != signals: + raise Exception("%r != %r" % (raised, signals)) + + {} + + signal.signal(signal.SIGALRM, handler) + read, write = os.pipe() + os.set_blocking(write, False) + signal.set_wakeup_fd(write) + + test() + check_signum(signals) + + os.close(read) + os.close(write) + """.format(tuple(map(int, signals)), ordered, test_body) + + assert_python_ok('-c', code) + + @support.impl_detail("pypy writes the message to fd 2, not to sys.stderr", + pypy=False) + @unittest.skipIf(_testcapi is None, 'need _testcapi') + def test_wakeup_write_error(self): + # Issue #16105: write() errors in the C signal handler should not + # pass silently. + # Use a subprocess to have only one thread. + code = """if 1: + import _testcapi + import errno + import os + import signal + import sys + from test.support import captured_stderr + + def handler(signum, frame): + 1/0 + + signal.signal(signal.SIGALRM, handler) + r, w = os.pipe() + os.set_blocking(r, False) + + # Set wakeup_fd a read-only file descriptor to trigger the error + signal.set_wakeup_fd(r) + try: + with captured_stderr() as err: + _testcapi.raise_signal(signal.SIGALRM) + except ZeroDivisionError: + # An ignored exception should have been printed out on stderr + err = err.getvalue() + if ('Exception ignored when trying to write to the signal wakeup fd' + not in err): + raise AssertionError(err) + if ('OSError: [Errno %d]' % errno.EBADF) not in err: + raise AssertionError(err) + else: + raise AssertionError("ZeroDivisionError not raised") + + os.close(r) + os.close(w) + """ + r, w = os.pipe() + try: + os.write(r, b'x') + except OSError: + pass + else: + self.skipTest("OS doesn't report write() error on the read end of a pipe") + finally: + os.close(r) + os.close(w) + + assert_python_ok('-c', code) + + def test_wakeup_fd_early(self): + self.check_wakeup("""def test(): + import select + import time + + TIMEOUT_FULL = 10 + TIMEOUT_HALF = 5 + + class InterruptSelect(Exception): + pass + + def handler(signum, frame): + raise InterruptSelect + signal.signal(signal.SIGALRM, handler) + + signal.alarm(1) + + # We attempt to get a signal during the sleep, + # before select is called + try: + select.select([], [], [], TIMEOUT_FULL) + except InterruptSelect: + pass + else: + raise Exception("select() was not interrupted") + + before_time = time.monotonic() + select.select([read], [], [], TIMEOUT_FULL) + after_time = time.monotonic() + dt = after_time - before_time + if dt >= TIMEOUT_HALF: + raise Exception("%s >= %s" % (dt, TIMEOUT_HALF)) + """, signal.SIGALRM) + + def test_wakeup_fd_during(self): + self.check_wakeup("""def test(): + import select + import time + + TIMEOUT_FULL = 10 + TIMEOUT_HALF = 5 + + class InterruptSelect(Exception): + pass + + def handler(signum, frame): + raise InterruptSelect + signal.signal(signal.SIGALRM, handler) + + signal.alarm(1) + before_time = time.monotonic() + # We attempt to get a signal during the select call + try: + select.select([read], [], [], TIMEOUT_FULL) + except InterruptSelect: + pass + else: + raise Exception("select() was not interrupted") + after_time = time.monotonic() + dt = after_time - before_time + if dt >= TIMEOUT_HALF: + raise Exception("%s >= %s" % (dt, TIMEOUT_HALF)) + """, signal.SIGALRM) + + def test_signum(self): + self.check_wakeup("""def test(): + import _testcapi + signal.signal(signal.SIGUSR1, handler) + _testcapi.raise_signal(signal.SIGUSR1) + _testcapi.raise_signal(signal.SIGALRM) + """, signal.SIGUSR1, signal.SIGALRM) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pending(self): + self.check_wakeup("""def test(): + signum1 = signal.SIGUSR1 + signum2 = signal.SIGUSR2 + + signal.signal(signum1, handler) + signal.signal(signum2, handler) + + signal.pthread_sigmask(signal.SIG_BLOCK, (signum1, signum2)) + _testcapi.raise_signal(signum1) + _testcapi.raise_signal(signum2) + # Unblocking the 2 signals calls the C signal handler twice + signal.pthread_sigmask(signal.SIG_UNBLOCK, (signum1, signum2)) + """, signal.SIGUSR1, signal.SIGUSR2, ordered=False) + + +@unittest.skipUnless(hasattr(socket, 'socketpair'), 'need socket.socketpair') +class WakeupSocketSignalTests(unittest.TestCase): + + @unittest.skipIf(_testcapi is None, 'need _testcapi') + def test_socket(self): + # use a subprocess to have only one thread + code = """if 1: + import signal + import socket + import struct + import _testcapi + + signum = signal.SIGINT + signals = (signum,) + + def handler(signum, frame): + pass + + signal.signal(signum, handler) + + read, write = socket.socketpair() + read.setblocking(False) + write.setblocking(False) + signal.set_wakeup_fd(write.fileno()) + + _testcapi.raise_signal(signum) + + data = read.recv(1) + if not data: + raise Exception("no signum written") + raised = struct.unpack('B', data) + if raised != signals: + raise Exception("%r != %r" % (raised, signals)) + + read.close() + write.close() + """ + + assert_python_ok('-c', code) + + @unittest.skipIf(_testcapi is None, 'need _testcapi') + def test_send_error(self): + # Use a subprocess to have only one thread. + if os.name == 'nt': + action = 'send' + else: + action = 'write' + code = """if 1: + import errno + import signal + import socket + import sys + import time + import _testcapi + from test.support import captured_stderr + + signum = signal.SIGINT + + def handler(signum, frame): + pass + + signal.signal(signum, handler) + + read, write = socket.socketpair() + read.setblocking(False) + write.setblocking(False) + + signal.set_wakeup_fd(write.fileno()) + + # Close sockets: send() will fail + read.close() + write.close() + + with captured_stderr() as err: + _testcapi.raise_signal(signum) + + err = err.getvalue() + if ('Exception ignored when trying to {action} to the signal wakeup fd' + not in err) and {cpython_only}: + raise AssertionError(err) + """.format(action=action, cpython_only=support.check_impl_detail()) + # note that PyPy produces the same error message, but sent to + # the real stderr instead of to sys.stderr. + assert_python_ok('-c', code) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class SiginterruptTest(unittest.TestCase): + + def readpipe_interrupted(self, interrupt): + """Perform a read during which a signal will arrive. Return True if the + read is interrupted by the signal and raises an exception. Return False + if it returns normally. + """ + # use a subprocess to have only one thread, to have a timeout on the + # blocking read and to not touch signal handling in this process + code = """if 1: + import errno + import os + import signal + import sys + + interrupt = %r + r, w = os.pipe() + + def handler(signum, frame): + 1 / 0 + + signal.signal(signal.SIGALRM, handler) + if interrupt is not None: + signal.siginterrupt(signal.SIGALRM, interrupt) + + print("ready") + sys.stdout.flush() + + # run the test twice + try: + for loop in range(2): + # send a SIGALRM in a second (during the read) + signal.alarm(1) + try: + # blocking call: read from a pipe without data + os.read(r, 1) + except ZeroDivisionError: + pass + else: + sys.exit(2) + sys.exit(3) + finally: + os.close(r) + os.close(w) + """ % (interrupt,) + with spawn_python('-c', code) as process: + try: + # wait until the child process is loaded and has started + first_line = process.stdout.readline() + + stdout, stderr = process.communicate(timeout=5.0) + except subprocess.TimeoutExpired: + process.kill() + return False + else: + stdout = first_line + stdout + exitcode = process.wait() + if exitcode not in (2, 3): + raise Exception("Child error (exit code %s): %r" + % (exitcode, stdout)) + return (exitcode == 3) + + def test_without_siginterrupt(self): + # If a signal handler is installed and siginterrupt is not called + # at all, when that signal arrives, it interrupts a syscall that's in + # progress. + interrupted = self.readpipe_interrupted(None) + self.assertTrue(interrupted) + + def test_siginterrupt_on(self): + # If a signal handler is installed and siginterrupt is called with + # a true value for the second argument, when that signal arrives, it + # interrupts a syscall that's in progress. + interrupted = self.readpipe_interrupted(True) + self.assertTrue(interrupted) + + def test_siginterrupt_off(self): + # If a signal handler is installed and siginterrupt is called with + # a false value for the second argument, when that signal arrives, it + # does not interrupt a syscall that's in progress. + interrupted = self.readpipe_interrupted(False) + self.assertFalse(interrupted) + + +@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") +class ItimerTest(unittest.TestCase): + def setUp(self): + self.hndl_called = False + self.hndl_count = 0 + self.itimer = None + self.old_alarm = signal.signal(signal.SIGALRM, self.sig_alrm) + + def tearDown(self): + signal.signal(signal.SIGALRM, self.old_alarm) + if self.itimer is not None: # test_itimer_exc doesn't change this attr + # just ensure that itimer is stopped + signal.setitimer(self.itimer, 0) + + def sig_alrm(self, *args): + self.hndl_called = True + + def sig_vtalrm(self, *args): + self.hndl_called = True + + if self.hndl_count > 3: + # it shouldn't be here, because it should have been disabled. + raise signal.ItimerError("setitimer didn't disable ITIMER_VIRTUAL " + "timer.") + elif self.hndl_count == 3: + # disable ITIMER_VIRTUAL, this function shouldn't be called anymore + signal.setitimer(signal.ITIMER_VIRTUAL, 0) + + self.hndl_count += 1 + + def sig_prof(self, *args): + self.hndl_called = True + signal.setitimer(signal.ITIMER_PROF, 0) + + def test_itimer_exc(self): + # XXX I'm assuming -1 is an invalid itimer, but maybe some platform + # defines it ? + self.assertRaises(signal.ItimerError, signal.setitimer, -1, 0) + # Negative times are treated as zero on some platforms. + if 0: + self.assertRaises(signal.ItimerError, + signal.setitimer, signal.ITIMER_REAL, -1) + + def test_itimer_real(self): + self.itimer = signal.ITIMER_REAL + signal.setitimer(self.itimer, 1.0) + signal.pause() + self.assertEqual(self.hndl_called, True) + + # Issue 3864, unknown if this affects earlier versions of freebsd also + @unittest.skipIf(sys.platform in ('freebsd6', 'netbsd5'), + 'itimer not reliable (does not mix well with threading) on some BSDs.') + def test_itimer_virtual(self): + self.itimer = signal.ITIMER_VIRTUAL + signal.signal(signal.SIGVTALRM, self.sig_vtalrm) + signal.setitimer(self.itimer, 0.3, 0.2) + + start_time = time.monotonic() + while time.monotonic() - start_time < 60.0: + # use up some virtual time by doing real work + _ = pow(12345, 67890, 10000019) + if signal.getitimer(self.itimer) == (0.0, 0.0): + break # sig_vtalrm handler stopped this itimer + else: # Issue 8424 + self.skipTest("timeout: likely cause: machine too slow or load too " + "high") + + # virtual itimer should be (0.0, 0.0) now + self.assertEqual(signal.getitimer(self.itimer), (0.0, 0.0)) + # and the handler should have been called + self.assertEqual(self.hndl_called, True) + + # Issue 3864, unknown if this affects earlier versions of freebsd also + @unittest.skipIf(sys.platform=='freebsd6', + 'itimer not reliable (does not mix well with threading) on freebsd6') + def test_itimer_prof(self): + self.itimer = signal.ITIMER_PROF + signal.signal(signal.SIGPROF, self.sig_prof) + signal.setitimer(self.itimer, 0.2, 0.2) + + start_time = time.monotonic() + while time.monotonic() - start_time < 60.0: + # do some work + _ = pow(12345, 67890, 10000019) + if signal.getitimer(self.itimer) == (0.0, 0.0): + break # sig_prof handler stopped this itimer + else: # Issue 8424 + self.skipTest("timeout: likely cause: machine too slow or load too " + "high") + + # profiling itimer should be (0.0, 0.0) now + self.assertEqual(signal.getitimer(self.itimer), (0.0, 0.0)) + # and the handler should have been called + self.assertEqual(self.hndl_called, True) + + +class PendingSignalsTests(unittest.TestCase): + """ + Test pthread_sigmask(), pthread_kill(), sigpending() and sigwait() + functions. + """ + @unittest.skipUnless(hasattr(signal, 'sigpending'), + 'need signal.sigpending()') + def test_sigpending_empty(self): + self.assertEqual(signal.sigpending(), set()) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + @unittest.skipUnless(hasattr(signal, 'sigpending'), + 'need signal.sigpending()') + def test_sigpending(self): + code = """if 1: + import os + import signal + + def handler(signum, frame): + 1/0 + + signum = signal.SIGUSR1 + signal.signal(signum, handler) + + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + os.kill(os.getpid(), signum) + pending = signal.sigpending() + for sig in pending: + assert isinstance(sig, signal.Signals), repr(pending) + if pending != {signum}: + raise Exception('%s != {%s}' % (pending, signum)) + try: + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + """ + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'pthread_kill'), + 'need signal.pthread_kill()') + def test_pthread_kill(self): + code = """if 1: + import signal + import threading + import sys + + signum = signal.SIGUSR1 + + def handler(signum, frame): + 1/0 + + signal.signal(signum, handler) + + if sys.platform == 'freebsd6': + # Issue #12392 and #12469: send a signal to the main thread + # doesn't work before the creation of the first thread on + # FreeBSD 6 + def noop(): + pass + thread = threading.Thread(target=noop) + thread.start() + thread.join() + + tid = threading.get_ident() + try: + signal.pthread_kill(tid, signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + """ + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def wait_helper(self, blocked, test): + """ + test: body of the "def test(signum):" function. + blocked: number of the blocked signal + """ + code = '''if 1: + import signal + import sys + from signal import Signals + + def handler(signum, frame): + 1/0 + + %s + + blocked = %s + signum = signal.SIGALRM + + # child: block and wait the signal + try: + signal.signal(signum, handler) + signal.pthread_sigmask(signal.SIG_BLOCK, [blocked]) + + # Do the tests + test(signum) + + # The handler must not be called on unblock + try: + signal.pthread_sigmask(signal.SIG_UNBLOCK, [blocked]) + except ZeroDivisionError: + print("the signal handler has been called", + file=sys.stderr) + sys.exit(1) + except BaseException as err: + print("error: {}".format(err), file=sys.stderr) + sys.stderr.flush() + sys.exit(1) + ''' % (test.strip(), blocked) + + # sig*wait* must be called with the signal blocked: since the current + # process might have several threads running, use a subprocess to have + # a single thread. + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'sigwait'), + 'need signal.sigwait()') + def test_sigwait(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + received = signal.sigwait([signum]) + assert isinstance(received, signal.Signals), received + if received != signum: + raise Exception('received %s, not %s' % (received, signum)) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + info = signal.sigwaitinfo([signum]) + if info.si_signo != signum: + raise Exception("info.si_signo != %s" % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + info = signal.sigtimedwait([signum], 10.1000) + if info.si_signo != signum: + raise Exception('info.si_signo != %s' % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_poll(self): + # check that polling with sigtimedwait works + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + import os + os.kill(os.getpid(), signum) + info = signal.sigtimedwait([signum], 0) + if info.si_signo != signum: + raise Exception('info.si_signo != %s' % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_timeout(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + received = signal.sigtimedwait([signum], 1.0) + if received is not None: + raise Exception("received=%r" % (received,)) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_negative_timeout(self): + signum = signal.SIGALRM + self.assertRaises(ValueError, signal.sigtimedwait, [signum], -1.0) + + @unittest.skipUnless(hasattr(signal, 'sigwait'), + 'need signal.sigwait()') + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + @unittest.skipIf(threading is None, "test needs threading module") + def test_sigwait_thread(self): + # Check that calling sigwait() from a thread doesn't suspend the whole + # process. A new interpreter is spawned to avoid problems when mixing + # threads and fork(): only async-safe functions are allowed between + # fork() and exec(). + assert_python_ok("-c", """if True: + import os, threading, sys, time, signal + + # the default handler terminates the process + signum = signal.SIGUSR1 + + def kill_later(): + # wait until the main thread is waiting in sigwait() + time.sleep(1) + os.kill(os.getpid(), signum) + + # the signal must be blocked by all the threads + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + killer = threading.Thread(target=kill_later) + killer.start() + received = signal.sigwait([signum]) + if received != signum: + print("sigwait() received %s, not %s" % (received, signum), + file=sys.stderr) + sys.exit(1) + killer.join() + # unblock the signal, which should have been cleared by sigwait() + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + """) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pthread_sigmask_arguments(self): + self.assertRaises(TypeError, signal.pthread_sigmask) + self.assertRaises(TypeError, signal.pthread_sigmask, 1) + self.assertRaises(TypeError, signal.pthread_sigmask, 1, 2, 3) + self.assertRaises(OSError, signal.pthread_sigmask, 1700, []) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pthread_sigmask(self): + code = """if 1: + import signal + import os; import threading + + def handler(signum, frame): + 1/0 + + def kill(signum): + os.kill(os.getpid(), signum) + + def check_mask(mask): + for sig in mask: + assert isinstance(sig, signal.Signals), repr(sig) + + def read_sigmask(): + sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, []) + check_mask(sigmask) + return sigmask + + signum = signal.SIGUSR1 + + # Install our signal handler + old_handler = signal.signal(signum, handler) + + # Unblock SIGUSR1 (and copy the old mask) to test our signal handler + old_mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + check_mask(old_mask) + try: + kill(signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + + # Block and then raise SIGUSR1. The signal is blocked: the signal + # handler is not called, and the signal is now pending + mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + check_mask(mask) + kill(signum) + + # Check the new mask + blocked = read_sigmask() + check_mask(blocked) + if signum not in blocked: + raise Exception("%s not in %s" % (signum, blocked)) + if old_mask ^ blocked != {signum}: + raise Exception("%s ^ %s != {%s}" % (old_mask, blocked, signum)) + + # Unblock SIGUSR1 + try: + # unblock the pending signal calls immediately the signal handler + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + try: + kill(signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + + # Check the new mask + unblocked = read_sigmask() + if signum in unblocked: + raise Exception("%s in %s" % (signum, unblocked)) + if blocked ^ unblocked != {signum}: + raise Exception("%s ^ %s != {%s}" % (blocked, unblocked, signum)) + if old_mask != unblocked: + raise Exception("%s != %s" % (old_mask, unblocked)) + """ + assert_python_ok('-c', code) + + @unittest.skipIf(sys.platform == 'freebsd6', + "issue #12392: send a signal to the main thread doesn't work " + "before the creation of the first thread on FreeBSD 6") + @unittest.skipUnless(hasattr(signal, 'pthread_kill'), + 'need signal.pthread_kill()') + def test_pthread_kill_main_thread(self): + # Test that a signal can be sent to the main thread with pthread_kill() + # before any other thread has been created (see issue #12392). + code = """if True: + import threading + import signal + import sys + + def handler(signum, frame): + sys.exit(3) + + signal.signal(signal.SIGUSR1, handler) + signal.pthread_kill(threading.get_ident(), signal.SIGUSR1) + sys.exit(2) + """ + + with spawn_python('-c', code) as process: + stdout, stderr = process.communicate() + exitcode = process.wait() + if exitcode != 3: + raise Exception("Child error (exit code %s): %s" % + (exitcode, stdout)) + + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_smtplib.py b/src/greentest/3.5pypy/test_smtplib.py new file mode 100644 index 0000000..28539f3 --- /dev/null +++ b/src/greentest/3.5pypy/test_smtplib.py @@ -0,0 +1,1290 @@ +import asyncore +import base64 +import email.mime.text +from email.message import EmailMessage +from email.base64mime import body_encode as encode_base64 +import email.utils +import hmac +import socket +import smtpd +import smtplib +import io +import re +import sys +import time +import select +import errno +import textwrap + +import unittest +from test import support, mock_socket + +try: + import threading +except ImportError: + threading = None + +HOST = support.HOST + +if sys.platform == 'darwin': + # select.poll returns a select.POLLHUP at the end of the tests + # on darwin, so just ignore it + def handle_expt(self): + pass + smtpd.SMTPChannel.handle_expt = handle_expt + + +def server(evt, buf, serv): + serv.listen() + evt.set() + try: + conn, addr = serv.accept() + except socket.timeout: + pass + else: + n = 500 + while buf and n > 0: + r, w, e = select.select([], [conn], []) + if w: + sent = conn.send(buf) + buf = buf[sent:] + + n -= 1 + + conn.close() + finally: + serv.close() + evt.set() + +class GeneralTests(unittest.TestCase): + + def setUp(self): + smtplib.socket = mock_socket + self.port = 25 + + def tearDown(self): + smtplib.socket = socket + + # This method is no longer used but is retained for backward compatibility, + # so test to make sure it still works. + def testQuoteData(self): + teststr = "abc\n.jkl\rfoo\r\n..blue" + expected = "abc\r\n..jkl\r\nfoo\r\n...blue" + self.assertEqual(expected, smtplib.quotedata(teststr)) + + def testBasic1(self): + mock_socket.reply_with(b"220 Hola mundo") + # connects + smtp = smtplib.SMTP(HOST, self.port) + smtp.close() + + def testSourceAddress(self): + mock_socket.reply_with(b"220 Hola mundo") + # connects + smtp = smtplib.SMTP(HOST, self.port, + source_address=('127.0.0.1',19876)) + self.assertEqual(smtp.source_address, ('127.0.0.1', 19876)) + smtp.close() + + def testBasic2(self): + mock_socket.reply_with(b"220 Hola mundo") + # connects, include port in host name + smtp = smtplib.SMTP("%s:%s" % (HOST, self.port)) + smtp.close() + + def testLocalHostName(self): + mock_socket.reply_with(b"220 Hola mundo") + # check that supplied local_hostname is used + smtp = smtplib.SMTP(HOST, self.port, local_hostname="testhost") + self.assertEqual(smtp.local_hostname, "testhost") + smtp.close() + + def testTimeoutDefault(self): + mock_socket.reply_with(b"220 Hola mundo") + self.assertIsNone(mock_socket.getdefaulttimeout()) + mock_socket.setdefaulttimeout(30) + self.assertEqual(mock_socket.getdefaulttimeout(), 30) + try: + smtp = smtplib.SMTP(HOST, self.port) + finally: + mock_socket.setdefaulttimeout(None) + self.assertEqual(smtp.sock.gettimeout(), 30) + smtp.close() + + def testTimeoutNone(self): + mock_socket.reply_with(b"220 Hola mundo") + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + smtp = smtplib.SMTP(HOST, self.port, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(smtp.sock.gettimeout()) + smtp.close() + + def testTimeoutValue(self): + mock_socket.reply_with(b"220 Hola mundo") + smtp = smtplib.SMTP(HOST, self.port, timeout=30) + self.assertEqual(smtp.sock.gettimeout(), 30) + smtp.close() + + def test_debuglevel(self): + mock_socket.reply_with(b"220 Hello world") + smtp = smtplib.SMTP() + smtp.set_debuglevel(1) + with support.captured_stderr() as stderr: + smtp.connect(HOST, self.port) + smtp.close() + expected = re.compile(r"^connect:", re.MULTILINE) + self.assertRegex(stderr.getvalue(), expected) + + def test_debuglevel_2(self): + mock_socket.reply_with(b"220 Hello world") + smtp = smtplib.SMTP() + smtp.set_debuglevel(2) + with support.captured_stderr() as stderr: + smtp.connect(HOST, self.port) + smtp.close() + expected = re.compile(r"^\d{2}:\d{2}:\d{2}\.\d{6} connect: ", + re.MULTILINE) + self.assertRegex(stderr.getvalue(), expected) + + +# Test server thread using the specified SMTP server class +def debugging_server(serv, serv_evt, client_evt): + serv_evt.set() + + try: + if hasattr(select, 'poll'): + poll_fun = asyncore.poll2 + else: + poll_fun = asyncore.poll + + n = 1000 + while asyncore.socket_map and n > 0: + poll_fun(0.01, asyncore.socket_map) + + # when the client conversation is finished, it will + # set client_evt, and it's then ok to kill the server + if client_evt.is_set(): + serv.close() + break + + n -= 1 + + except socket.timeout: + pass + finally: + if not client_evt.is_set(): + # allow some time for the client to read the result + time.sleep(0.5) + serv.close() + asyncore.close_all() + serv_evt.set() + +MSG_BEGIN = '---------- MESSAGE FOLLOWS ----------\n' +MSG_END = '------------ END MESSAGE ------------\n' + +# NOTE: Some SMTP objects in the tests below are created with a non-default +# local_hostname argument to the constructor, since (on some systems) the FQDN +# lookup caused by the default local_hostname sometimes takes so long that the +# test server times out, causing the test to fail. + +# Test behavior of smtpd.DebuggingServer +@unittest.skipUnless(threading, 'Threading required for this test.') +class DebuggingServerTests(unittest.TestCase): + + maxDiff = None + + def setUp(self): + self.real_getfqdn = socket.getfqdn + socket.getfqdn = mock_socket.getfqdn + # temporarily replace sys.stdout to capture DebuggingServer output + self.old_stdout = sys.stdout + self.output = io.StringIO() + sys.stdout = self.output + + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Capture SMTPChannel debug output + self.old_DEBUGSTREAM = smtpd.DEBUGSTREAM + smtpd.DEBUGSTREAM = io.StringIO() + # Pick a random unused port by passing 0 for the port number + self.serv = smtpd.DebuggingServer((HOST, 0), ('nowhere', -1), + decode_data=True) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + socket.getfqdn = self.real_getfqdn + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + self.thread.join() + # restore sys.stdout + sys.stdout = self.old_stdout + # restore DEBUGSTREAM + smtpd.DEBUGSTREAM.close() + smtpd.DEBUGSTREAM = self.old_DEBUGSTREAM + + def testBasic(self): + # connect + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.quit() + + def testSourceAddress(self): + # connect + port = support.find_unused_port() + try: + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=3, source_address=('127.0.0.1', port)) + self.assertEqual(smtp.source_address, ('127.0.0.1', port)) + self.assertEqual(smtp.local_hostname, 'localhost') + smtp.quit() + except OSError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + + def testNOOP(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (250, b'OK') + self.assertEqual(smtp.noop(), expected) + smtp.quit() + + def testRSET(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (250, b'OK') + self.assertEqual(smtp.rset(), expected) + smtp.quit() + + def testELHO(self): + # EHLO isn't implemented in DebuggingServer + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (250, b'\nSIZE 33554432\nHELP') + self.assertEqual(smtp.ehlo(), expected) + smtp.quit() + + def testEXPNNotImplemented(self): + # EXPN isn't implemented in DebuggingServer + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (502, b'EXPN not implemented') + smtp.putcmd('EXPN') + self.assertEqual(smtp.getreply(), expected) + smtp.quit() + + def testVRFY(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + expected = (252, b'Cannot VRFY user, but will accept message ' + \ + b'and attempt delivery') + self.assertEqual(smtp.vrfy('nobody@nowhere.com'), expected) + self.assertEqual(smtp.verify('nobody@nowhere.com'), expected) + smtp.quit() + + def testSecondHELO(self): + # check that a second HELO returns a message that it's a duplicate + # (this behavior is specific to smtpd.SMTPChannel) + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.helo() + expected = (503, b'Duplicate HELO/EHLO') + self.assertEqual(smtp.helo(), expected) + smtp.quit() + + def testHELP(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + self.assertEqual(smtp.help(), b'Supported commands: EHLO HELO MAIL ' + \ + b'RCPT DATA RSET NOOP QUIT VRFY') + smtp.quit() + + def testSend(self): + # connect and send mail + m = 'A test message' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.sendmail('John', 'Sally', m) + # XXX(nnorwitz): this test is flaky and dies with a bad file descriptor + # in asyncore. This sleep might help, but should really be fixed + # properly by using an Event variable. + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + def testSendBinary(self): + m = b'A test message' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.sendmail('John', 'Sally', m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.decode('ascii'), MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + def testSendNeedingDotQuote(self): + # Issue 12283 + m = '.A test\n.mes.sage.' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.sendmail('John', 'Sally', m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + def testSendNullSender(self): + m = 'A test message' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.sendmail('<>', 'Sally', m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + mexpect = '%s%s\n%s' % (MSG_BEGIN, m, MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: <>$", re.MULTILINE) + self.assertRegex(debugout, sender) + + def testSendMessage(self): + m = email.mime.text.MIMEText('A test message') + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.send_message(m, from_addr='John', to_addrs='Sally') + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Add the X-Peer header that DebuggingServer adds + m['X-Peer'] = socket.gethostbyname('localhost') + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + + def testSendMessageWithAddresses(self): + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John' + m['CC'] = 'Sally, Fred' + m['Bcc'] = 'John Root , "Dinsdale" ' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.send_message(m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + # make sure the Bcc header is still in the message. + self.assertEqual(m['Bcc'], 'John Root , "Dinsdale" ' + '') + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Add the X-Peer header that DebuggingServer adds + m['X-Peer'] = socket.gethostbyname('localhost') + # The Bcc header should not be transmitted. + del m['Bcc'] + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: foo@bar.com$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('John', 'Sally', 'Fred', 'root@localhost', + 'warped@silly.walks.com'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertRegex(debugout, to_addr) + + def testSendMessageWithSomeAddresses(self): + # Make sure nothing breaks if not all of the three 'to' headers exist + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John, Dinsdale' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.send_message(m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Add the X-Peer header that DebuggingServer adds + m['X-Peer'] = socket.gethostbyname('localhost') + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: foo@bar.com$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('John', 'Dinsdale'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertRegex(debugout, to_addr) + + def testSendMessageWithSpecifiedAddresses(self): + # Make sure addresses specified in call override those in message. + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John, Dinsdale' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.send_message(m, from_addr='joe@example.com', to_addrs='foo@example.net') + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Add the X-Peer header that DebuggingServer adds + m['X-Peer'] = socket.gethostbyname('localhost') + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: joe@example.com$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('John', 'Dinsdale'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertNotRegex(debugout, to_addr) + recip = re.compile(r"^recips: .*'foo@example.net'.*$", re.MULTILINE) + self.assertRegex(debugout, recip) + + def testSendMessageWithMultipleFrom(self): + # Sender overrides To + m = email.mime.text.MIMEText('A test message') + m['From'] = 'Bernard, Bianca' + m['Sender'] = 'the_rescuers@Rescue-Aid-Society.com' + m['To'] = 'John, Dinsdale' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.send_message(m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # Add the X-Peer header that DebuggingServer adds + m['X-Peer'] = socket.gethostbyname('localhost') + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: the_rescuers@Rescue-Aid-Society.com$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('John', 'Dinsdale'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertRegex(debugout, to_addr) + + def testSendMessageResent(self): + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John' + m['CC'] = 'Sally, Fred' + m['Bcc'] = 'John Root , "Dinsdale" ' + m['Resent-Date'] = 'Thu, 1 Jan 1970 17:42:00 +0000' + m['Resent-From'] = 'holy@grail.net' + m['Resent-To'] = 'Martha , Jeff' + m['Resent-Bcc'] = 'doe@losthope.net' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + smtp.send_message(m) + # XXX (see comment in testSend) + time.sleep(0.01) + smtp.quit() + + self.client_evt.set() + self.serv_evt.wait() + self.output.flush() + # The Resent-Bcc headers are deleted before serialization. + del m['Bcc'] + del m['Resent-Bcc'] + # Add the X-Peer header that DebuggingServer adds + m['X-Peer'] = socket.gethostbyname('localhost') + mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END) + self.assertEqual(self.output.getvalue(), mexpect) + debugout = smtpd.DEBUGSTREAM.getvalue() + sender = re.compile("^sender: holy@grail.net$", re.MULTILINE) + self.assertRegex(debugout, sender) + for addr in ('my_mom@great.cooker.com', 'Jeff', 'doe@losthope.net'): + to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr), + re.MULTILINE) + self.assertRegex(debugout, to_addr) + + def testSendMessageMultipleResentRaises(self): + m = email.mime.text.MIMEText('A test message') + m['From'] = 'foo@bar.com' + m['To'] = 'John' + m['CC'] = 'Sally, Fred' + m['Bcc'] = 'John Root , "Dinsdale" ' + m['Resent-Date'] = 'Thu, 1 Jan 1970 17:42:00 +0000' + m['Resent-From'] = 'holy@grail.net' + m['Resent-To'] = 'Martha , Jeff' + m['Resent-Bcc'] = 'doe@losthope.net' + m['Resent-Date'] = 'Thu, 2 Jan 1970 17:42:00 +0000' + m['Resent-To'] = 'holy@grail.net' + m['Resent-From'] = 'Martha , Jeff' + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) + with self.assertRaises(ValueError): + smtp.send_message(m) + smtp.close() + +class NonConnectingTests(unittest.TestCase): + + def testNotConnected(self): + # Test various operations on an unconnected SMTP object that + # should raise exceptions (at present the attempt in SMTP.send + # to reference the nonexistent 'sock' attribute of the SMTP object + # causes an AttributeError) + smtp = smtplib.SMTP() + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.ehlo) + self.assertRaises(smtplib.SMTPServerDisconnected, + smtp.send, 'test msg') + + def testNonnumericPort(self): + # check that non-numeric port raises OSError + self.assertRaises(OSError, smtplib.SMTP, + "localhost", "bogus") + self.assertRaises(OSError, smtplib.SMTP, + "localhost:bogus") + + +# test response of client to a non-successful HELO message +@unittest.skipUnless(threading, 'Threading required for this test.') +class BadHELOServerTests(unittest.TestCase): + + def setUp(self): + smtplib.socket = mock_socket + mock_socket.reply_with(b"199 no hello for you!") + self.old_stdout = sys.stdout + self.output = io.StringIO() + sys.stdout = self.output + self.port = 25 + + def tearDown(self): + smtplib.socket = socket + sys.stdout = self.old_stdout + + def testFailingHELO(self): + self.assertRaises(smtplib.SMTPConnectError, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class TooLongLineTests(unittest.TestCase): + respdata = b'250 OK' + (b'.' * smtplib._MAXLINE * 2) + b'\n' + + def setUp(self): + self.old_stdout = sys.stdout + self.output = io.StringIO() + sys.stdout = self.output + + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(15) + self.port = support.bind_port(self.sock) + servargs = (self.evt, self.respdata, self.sock) + threading.Thread(target=server, args=servargs).start() + self.evt.wait() + self.evt.clear() + + def tearDown(self): + self.evt.wait() + sys.stdout = self.old_stdout + + def testLineTooLong(self): + self.assertRaises(smtplib.SMTPResponseException, smtplib.SMTP, + HOST, self.port, 'localhost', 3) + + +sim_users = {'Mr.A@somewhere.com':'John A', + 'Ms.B@xn--fo-fka.com':'Sally B', + 'Mrs.C@somewhereesle.com':'Ruth C', + } + +sim_auth = ('Mr.A@somewhere.com', 'somepassword') +sim_cram_md5_challenge = ('PENCeUxFREJoU0NnbmhNWitOMjNGNn' + 'dAZWx3b29kLmlubm9zb2Z0LmNvbT4=') +sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'], + 'list-2':['Ms.B@xn--fo-fka.com',], + } + +# Simulated SMTP channel & server +class ResponseException(Exception): pass +class SimSMTPChannel(smtpd.SMTPChannel): + + quit_response = None + mail_response = None + rcpt_response = None + data_response = None + rcpt_count = 0 + rset_count = 0 + disconnect = 0 + AUTH = 99 # Add protocol state to enable auth testing. + authenticated_user = None + + def __init__(self, extra_features, *args, **kw): + self._extrafeatures = ''.join( + [ "250-{0}\r\n".format(x) for x in extra_features ]) + super(SimSMTPChannel, self).__init__(*args, **kw) + + # AUTH related stuff. It would be nice if support for this were in smtpd. + def found_terminator(self): + if self.smtp_state == self.AUTH: + line = self._emptystring.join(self.received_lines) + print('Data:', repr(line), file=smtpd.DEBUGSTREAM) + self.received_lines = [] + try: + self.auth_object(line) + except ResponseException as e: + self.smtp_state = self.COMMAND + self.push('%s %s' % (e.smtp_code, e.smtp_error)) + return + super().found_terminator() + + + def smtp_AUTH(self, arg): + if not self.seen_greeting: + self.push('503 Error: send EHLO first') + return + if not self.extended_smtp or 'AUTH' not in self._extrafeatures: + self.push('500 Error: command "AUTH" not recognized') + return + if self.authenticated_user is not None: + self.push( + '503 Bad sequence of commands: already authenticated') + return + args = arg.split() + if len(args) not in [1, 2]: + self.push('501 Syntax: AUTH [initial-response]') + return + auth_object_name = '_auth_%s' % args[0].lower().replace('-', '_') + try: + self.auth_object = getattr(self, auth_object_name) + except AttributeError: + self.push('504 Command parameter not implemented: unsupported ' + ' authentication mechanism {!r}'.format(auth_object_name)) + return + self.smtp_state = self.AUTH + self.auth_object(args[1] if len(args) == 2 else None) + + def _authenticated(self, user, valid): + if valid: + self.authenticated_user = user + self.push('235 Authentication Succeeded') + else: + self.push('535 Authentication credentials invalid') + self.smtp_state = self.COMMAND + + def _decode_base64(self, string): + return base64.decodebytes(string.encode('ascii')).decode('utf-8') + + def _auth_plain(self, arg=None): + if arg is None: + self.push('334 ') + else: + logpass = self._decode_base64(arg) + try: + *_, user, password = logpass.split('\0') + except ValueError as e: + self.push('535 Splitting response {!r} into user and password' + ' failed: {}'.format(logpass, e)) + return + self._authenticated(user, password == sim_auth[1]) + + def _auth_login(self, arg=None): + if arg is None: + # base64 encoded 'Username:' + self.push('334 VXNlcm5hbWU6') + elif not hasattr(self, '_auth_login_user'): + self._auth_login_user = self._decode_base64(arg) + # base64 encoded 'Password:' + self.push('334 UGFzc3dvcmQ6') + else: + password = self._decode_base64(arg) + self._authenticated(self._auth_login_user, password == sim_auth[1]) + del self._auth_login_user + + def _auth_cram_md5(self, arg=None): + if arg is None: + self.push('334 {}'.format(sim_cram_md5_challenge)) + else: + logpass = self._decode_base64(arg) + try: + user, hashed_pass = logpass.split() + except ValueError as e: + self.push('535 Splitting response {!r} into user and password' + 'failed: {}'.format(logpass, e)) + return False + valid_hashed_pass = hmac.HMAC( + sim_auth[1].encode('ascii'), + self._decode_base64(sim_cram_md5_challenge).encode('ascii'), + 'md5').hexdigest() + self._authenticated(user, hashed_pass == valid_hashed_pass) + # end AUTH related stuff. + + def smtp_EHLO(self, arg): + resp = ('250-testhost\r\n' + '250-EXPN\r\n' + '250-SIZE 20000000\r\n' + '250-STARTTLS\r\n' + '250-DELIVERBY\r\n') + resp = resp + self._extrafeatures + '250 HELP' + self.push(resp) + self.seen_greeting = arg + self.extended_smtp = True + + def smtp_VRFY(self, arg): + # For max compatibility smtplib should be sending the raw address. + if arg in sim_users: + self.push('250 %s %s' % (sim_users[arg], smtplib.quoteaddr(arg))) + else: + self.push('550 No such user: %s' % arg) + + def smtp_EXPN(self, arg): + list_name = arg.lower() + if list_name in sim_lists: + user_list = sim_lists[list_name] + for n, user_email in enumerate(user_list): + quoted_addr = smtplib.quoteaddr(user_email) + if n < len(user_list) - 1: + self.push('250-%s %s' % (sim_users[user_email], quoted_addr)) + else: + self.push('250 %s %s' % (sim_users[user_email], quoted_addr)) + else: + self.push('550 No access for you!') + + def smtp_QUIT(self, arg): + if self.quit_response is None: + super(SimSMTPChannel, self).smtp_QUIT(arg) + else: + self.push(self.quit_response) + self.close_when_done() + + def smtp_MAIL(self, arg): + if self.mail_response is None: + super().smtp_MAIL(arg) + else: + self.push(self.mail_response) + if self.disconnect: + self.close_when_done() + + def smtp_RCPT(self, arg): + if self.rcpt_response is None: + super().smtp_RCPT(arg) + return + self.rcpt_count += 1 + self.push(self.rcpt_response[self.rcpt_count-1]) + + def smtp_RSET(self, arg): + self.rset_count += 1 + super().smtp_RSET(arg) + + def smtp_DATA(self, arg): + if self.data_response is None: + super().smtp_DATA(arg) + else: + self.push(self.data_response) + + def handle_error(self): + raise + + +class SimSMTPServer(smtpd.SMTPServer): + + channel_class = SimSMTPChannel + + def __init__(self, *args, **kw): + self._extra_features = [] + smtpd.SMTPServer.__init__(self, *args, **kw) + + def handle_accepted(self, conn, addr): + self._SMTPchannel = self.channel_class( + self._extra_features, self, conn, addr, + decode_data=self._decode_data) + + def process_message(self, peer, mailfrom, rcpttos, data): + pass + + def add_feature(self, feature): + self._extra_features.append(feature) + + def handle_error(self): + raise + + +# Test various SMTP & ESMTP commands/behaviors that require a simulated server +# (i.e., something with more features than DebuggingServer) +@unittest.skipUnless(threading, 'Threading required for this test.') +class SMTPSimTests(unittest.TestCase): + + def setUp(self): + self.real_getfqdn = socket.getfqdn + socket.getfqdn = mock_socket.getfqdn + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = SimSMTPServer((HOST, 0), ('nowhere', -1), decode_data=True) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + socket.getfqdn = self.real_getfqdn + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + self.thread.join() + + def testBasic(self): + # smoke test + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.quit() + + def testEHLO(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + # no features should be present before the EHLO + self.assertEqual(smtp.esmtp_features, {}) + + # features expected from the test server + expected_features = {'expn':'', + 'size': '20000000', + 'starttls': '', + 'deliverby': '', + 'help': '', + } + + smtp.ehlo() + self.assertEqual(smtp.esmtp_features, expected_features) + for k in expected_features: + self.assertTrue(smtp.has_extn(k)) + self.assertFalse(smtp.has_extn('unsupported-feature')) + smtp.quit() + + def testVRFY(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + for addr_spec, name in sim_users.items(): + expected_known = (250, bytes('%s %s' % + (name, smtplib.quoteaddr(addr_spec)), + "ascii")) + self.assertEqual(smtp.vrfy(addr_spec), expected_known) + + u = 'nobody@nowhere.com' + expected_unknown = (550, ('No such user: %s' % u).encode('ascii')) + self.assertEqual(smtp.vrfy(u), expected_unknown) + smtp.quit() + + def testEXPN(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + + for listname, members in sim_lists.items(): + users = [] + for m in members: + users.append('%s %s' % (sim_users[m], smtplib.quoteaddr(m))) + expected_known = (250, bytes('\n'.join(users), "ascii")) + self.assertEqual(smtp.expn(listname), expected_known) + + u = 'PSU-Members-List' + expected_unknown = (550, b'No access for you!') + self.assertEqual(smtp.expn(u), expected_unknown) + smtp.quit() + + def testAUTH_PLAIN(self): + self.serv.add_feature("AUTH PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + resp = smtp.login(sim_auth[0], sim_auth[1]) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def testAUTH_LOGIN(self): + self.serv.add_feature("AUTH LOGIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + resp = smtp.login(sim_auth[0], sim_auth[1]) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def testAUTH_CRAM_MD5(self): + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + resp = smtp.login(sim_auth[0], sim_auth[1]) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def testAUTH_multiple(self): + # Test that multiple authentication methods are tried. + self.serv.add_feature("AUTH BOGUS PLAIN LOGIN CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + resp = smtp.login(sim_auth[0], sim_auth[1]) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def test_auth_function(self): + supported = {'CRAM-MD5', 'PLAIN', 'LOGIN'} + for mechanism in supported: + self.serv.add_feature("AUTH {}".format(mechanism)) + for mechanism in supported: + with self.subTest(mechanism=mechanism): + smtp = smtplib.SMTP(HOST, self.port, + local_hostname='localhost', timeout=15) + smtp.ehlo('foo') + smtp.user, smtp.password = sim_auth[0], sim_auth[1] + method = 'auth_' + mechanism.lower().replace('-', '_') + resp = smtp.auth(mechanism, getattr(smtp, method)) + self.assertEqual(resp, (235, b'Authentication Succeeded')) + smtp.close() + + def test_quit_resets_greeting(self): + smtp = smtplib.SMTP(HOST, self.port, + local_hostname='localhost', + timeout=15) + code, message = smtp.ehlo() + self.assertEqual(code, 250) + self.assertIn('size', smtp.esmtp_features) + smtp.quit() + self.assertNotIn('size', smtp.esmtp_features) + smtp.connect(HOST, self.port) + self.assertNotIn('size', smtp.esmtp_features) + smtp.ehlo_or_helo_if_needed() + self.assertIn('size', smtp.esmtp_features) + smtp.quit() + + def test_with_statement(self): + with smtplib.SMTP(HOST, self.port) as smtp: + code, message = smtp.noop() + self.assertEqual(code, 250) + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo') + with smtplib.SMTP(HOST, self.port) as smtp: + smtp.close() + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo') + + def test_with_statement_QUIT_failure(self): + with self.assertRaises(smtplib.SMTPResponseException) as error: + with smtplib.SMTP(HOST, self.port) as smtp: + smtp.noop() + self.serv._SMTPchannel.quit_response = '421 QUIT FAILED' + self.assertEqual(error.exception.smtp_code, 421) + self.assertEqual(error.exception.smtp_error, b'QUIT FAILED') + + #TODO: add tests for correct AUTH method fallback now that the + #test infrastructure can support it. + + # Issue 17498: make sure _rset does not raise SMTPServerDisconnected exception + def test__rest_from_mail_cmd(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.noop() + self.serv._SMTPchannel.mail_response = '451 Requested action aborted' + self.serv._SMTPchannel.disconnect = True + with self.assertRaises(smtplib.SMTPSenderRefused): + smtp.sendmail('John', 'Sally', 'test message') + self.assertIsNone(smtp.sock) + + # Issue 5713: make sure close, not rset, is called if we get a 421 error + def test_421_from_mail_cmd(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.noop() + self.serv._SMTPchannel.mail_response = '421 closing connection' + with self.assertRaises(smtplib.SMTPSenderRefused): + smtp.sendmail('John', 'Sally', 'test message') + self.assertIsNone(smtp.sock) + self.assertEqual(self.serv._SMTPchannel.rset_count, 0) + + def test_421_from_rcpt_cmd(self): + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.noop() + self.serv._SMTPchannel.rcpt_response = ['250 accepted', '421 closing'] + with self.assertRaises(smtplib.SMTPRecipientsRefused) as r: + smtp.sendmail('John', ['Sally', 'Frank', 'George'], 'test message') + self.assertIsNone(smtp.sock) + self.assertEqual(self.serv._SMTPchannel.rset_count, 0) + self.assertDictEqual(r.exception.args[0], {'Frank': (421, b'closing')}) + + def test_421_from_data_cmd(self): + class MySimSMTPChannel(SimSMTPChannel): + def found_terminator(self): + if self.smtp_state == self.DATA: + self.push('421 closing') + else: + super().found_terminator() + self.serv.channel_class = MySimSMTPChannel + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=15) + smtp.noop() + with self.assertRaises(smtplib.SMTPDataError): + smtp.sendmail('John@foo.org', ['Sally@foo.org'], 'test message') + self.assertIsNone(smtp.sock) + self.assertEqual(self.serv._SMTPchannel.rcpt_count, 0) + + def test_smtputf8_NotSupportedError_if_no_server_support(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + smtp.ehlo() + self.assertTrue(smtp.does_esmtp) + self.assertFalse(smtp.has_extn('smtputf8')) + self.assertRaises( + smtplib.SMTPNotSupportedError, + smtp.sendmail, + 'John', 'Sally', '', mail_options=['BODY=8BITMIME', 'SMTPUTF8']) + self.assertRaises( + smtplib.SMTPNotSupportedError, + smtp.mail, 'John', options=['BODY=8BITMIME', 'SMTPUTF8']) + + def test_send_unicode_without_SMTPUTF8(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + self.assertRaises(UnicodeEncodeError, smtp.sendmail, 'Alice', 'Böb', '') + self.assertRaises(UnicodeEncodeError, smtp.mail, 'Älice') + + +class SimSMTPUTF8Server(SimSMTPServer): + + def __init__(self, *args, **kw): + # The base SMTP server turns these on automatically, but our test + # server is set up to munge the EHLO response, so we need to provide + # them as well. And yes, the call is to SMTPServer not SimSMTPServer. + self._extra_features = ['SMTPUTF8', '8BITMIME'] + smtpd.SMTPServer.__init__(self, *args, **kw) + + def handle_accepted(self, conn, addr): + self._SMTPchannel = self.channel_class( + self._extra_features, self, conn, addr, + decode_data=self._decode_data, + enable_SMTPUTF8=self.enable_SMTPUTF8, + ) + + def process_message(self, peer, mailfrom, rcpttos, data, mail_options=None, + rcpt_options=None): + self.last_peer = peer + self.last_mailfrom = mailfrom + self.last_rcpttos = rcpttos + self.last_message = data + self.last_mail_options = mail_options + self.last_rcpt_options = rcpt_options + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class SMTPUTF8SimTests(unittest.TestCase): + + maxDiff = None + + def setUp(self): + self.real_getfqdn = socket.getfqdn + socket.getfqdn = mock_socket.getfqdn + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = SimSMTPUTF8Server((HOST, 0), ('nowhere', -1), + decode_data=False, + enable_SMTPUTF8=True) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + socket.getfqdn = self.real_getfqdn + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + self.thread.join() + + def test_test_server_supports_extensions(self): + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + smtp.ehlo() + self.assertTrue(smtp.does_esmtp) + self.assertTrue(smtp.has_extn('smtputf8')) + + def test_send_unicode_with_SMTPUTF8_via_sendmail(self): + m = '¡a test message containing unicode!'.encode('utf-8') + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + smtp.sendmail('Jőhn', 'Sálly', m, + mail_options=['BODY=8BITMIME', 'SMTPUTF8']) + self.assertEqual(self.serv.last_mailfrom, 'Jőhn') + self.assertEqual(self.serv.last_rcpttos, ['Sálly']) + self.assertEqual(self.serv.last_message, m) + self.assertIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) + + def test_send_unicode_with_SMTPUTF8_via_low_level_API(self): + m = '¡a test message containing unicode!'.encode('utf-8') + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + smtp.ehlo() + self.assertEqual( + smtp.mail('Jő', options=['BODY=8BITMIME', 'SMTPUTF8']), + (250, b'OK')) + self.assertEqual(smtp.rcpt('János'), (250, b'OK')) + self.assertEqual(smtp.data(m), (250, b'OK')) + self.assertEqual(self.serv.last_mailfrom, 'Jő') + self.assertEqual(self.serv.last_rcpttos, ['János']) + self.assertEqual(self.serv.last_message, m) + self.assertIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) + + def test_send_message_uses_smtputf8_if_addrs_non_ascii(self): + msg = EmailMessage() + msg['From'] = "Páolo " + msg['To'] = 'Dinsdale' + msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' + # XXX I don't know why I need two \n's here, but this is an existing + # bug (if it is one) and not a problem with the new functionality. + msg.set_content("oh là là, know what I mean, know what I mean?\n\n") + # XXX smtpd converts received /r/n to /n, so we can't easily test that + # we are successfully sending /r/n :(. + expected = textwrap.dedent("""\ + From: Páolo + To: Dinsdale + Subject: Nudge nudge, wink, wink \u1F609 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + MIME-Version: 1.0 + + oh là là, know what I mean, know what I mean? + """) + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + self.assertEqual(smtp.send_message(msg), {}) + self.assertEqual(self.serv.last_mailfrom, 'főo@bar.com') + self.assertEqual(self.serv.last_rcpttos, ['Dinsdale']) + self.assertEqual(self.serv.last_message.decode(), expected) + self.assertIn('BODY=8BITMIME', self.serv.last_mail_options) + self.assertIn('SMTPUTF8', self.serv.last_mail_options) + self.assertEqual(self.serv.last_rcpt_options, []) + + def test_send_message_error_on_non_ascii_addrs_if_no_smtputf8(self): + msg = EmailMessage() + msg['From'] = "Páolo " + msg['To'] = 'Dinsdale' + msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', timeout=3) + self.addCleanup(smtp.close) + self.assertRaises(smtplib.SMTPNotSupportedError, + smtp.send_message(msg)) + + +EXPECTED_RESPONSE = encode_base64(b'\0psu\0doesnotexist', eol='') + +class SimSMTPAUTHInitialResponseChannel(SimSMTPChannel): + def smtp_AUTH(self, arg): + # RFC 4954's AUTH command allows for an optional initial-response. + # Not all AUTH methods support this; some require a challenge. AUTH + # PLAIN does those, so test that here. See issue #15014. + args = arg.split() + if args[0].lower() == 'plain': + if len(args) == 2: + # AUTH PLAIN with the response base 64 + # encoded. Hard code the expected response for the test. + if args[1] == EXPECTED_RESPONSE: + self.push('235 Ok') + return + self.push('571 Bad authentication') + +class SimSMTPAUTHInitialResponseServer(SimSMTPServer): + channel_class = SimSMTPAUTHInitialResponseChannel + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class SMTPAUTHInitialResponseSimTests(unittest.TestCase): + def setUp(self): + self.real_getfqdn = socket.getfqdn + socket.getfqdn = mock_socket.getfqdn + self.serv_evt = threading.Event() + self.client_evt = threading.Event() + # Pick a random unused port by passing 0 for the port number + self.serv = SimSMTPAUTHInitialResponseServer( + (HOST, 0), ('nowhere', -1), decode_data=True) + # Keep a note of what port was assigned + self.port = self.serv.socket.getsockname()[1] + serv_args = (self.serv, self.serv_evt, self.client_evt) + self.thread = threading.Thread(target=debugging_server, args=serv_args) + self.thread.start() + + # wait until server thread has assigned a port number + self.serv_evt.wait() + self.serv_evt.clear() + + def tearDown(self): + socket.getfqdn = self.real_getfqdn + # indicate that the client is finished + self.client_evt.set() + # wait for the server thread to terminate + self.serv_evt.wait() + self.thread.join() + + def testAUTH_PLAIN_initial_response_login(self): + self.serv.add_feature('AUTH PLAIN') + smtp = smtplib.SMTP(HOST, self.port, + local_hostname='localhost', timeout=15) + smtp.login('psu', 'doesnotexist') + smtp.close() + + def testAUTH_PLAIN_initial_response_auth(self): + self.serv.add_feature('AUTH PLAIN') + smtp = smtplib.SMTP(HOST, self.port, + local_hostname='localhost', timeout=15) + smtp.user = 'psu' + smtp.password = 'doesnotexist' + code, response = smtp.auth('plain', smtp.auth_plain) + smtp.close() + self.assertEqual(code, 235) + + +@support.reap_threads +def test_main(verbose=None): + support.run_unittest( + BadHELOServerTests, + DebuggingServerTests, + GeneralTests, + NonConnectingTests, + SMTPAUTHInitialResponseSimTests, + SMTPSimTests, + TooLongLineTests, + ) + + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/3.5pypy/test_socket.py b/src/greentest/3.5pypy/test_socket.py new file mode 100644 index 0000000..17b74df --- /dev/null +++ b/src/greentest/3.5pypy/test_socket.py @@ -0,0 +1,5403 @@ +import unittest +from test import support + +import errno +import io +import itertools +import socket +import select +import tempfile +import time +import traceback +import queue +import sys +import os +import array +import platform +import contextlib +from weakref import proxy +import signal +import math +import pickle +import struct +import random +import string +try: + import multiprocessing +except ImportError: + multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None + +HOST = support.HOST +MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return + +try: + import _thread as thread + import threading +except ImportError: + thread = None + threading = None +try: + import _socket +except ImportError: + _socket = None + + +def _have_socket_can(): + """Check whether CAN sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_rds(): + """Check whether RDS sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +HAVE_SOCKET_CAN = _have_socket_can() + +HAVE_SOCKET_RDS = _have_socket_rds() + +# Size in bytes of the int type +SIZEOF_INT = array.array("i").itemsize + +class SocketTCPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.serv.listen() + + def tearDown(self): + self.serv.close() + self.serv = None + +class SocketUDPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.port = support.bind_port(self.serv) + + def tearDown(self): + self.serv.close() + self.serv = None + +class ThreadSafeCleanupTestCase(unittest.TestCase): + """Subclass of unittest.TestCase with thread-safe cleanup methods. + + This subclass protects the addCleanup() and doCleanups() methods + with a recursive lock. + """ + + if threading: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cleanup_lock = threading.RLock() + + def addCleanup(self, *args, **kwargs): + with self._cleanup_lock: + return super().addCleanup(*args, **kwargs) + + def doCleanups(self, *args, **kwargs): + with self._cleanup_lock: + return super().doCleanups(*args, **kwargs) + +class SocketCANTest(unittest.TestCase): + + """To be able to run this test, a `vcan0` CAN interface can be created with + the following commands: + # modprobe vcan + # ip link add dev vcan0 type vcan + # ifconfig vcan0 up + """ + interface = 'vcan0' + bufsize = 128 + + """The CAN frame structure is defined in : + + struct can_frame { + canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ + __u8 can_dlc; /* data length code: 0 .. 8 */ + __u8 data[8] __attribute__((aligned(8))); + }; + """ + can_frame_fmt = "=IB3x8s" + can_frame_size = struct.calcsize(can_frame_fmt) + + """The Broadcast Management Command frame structure is defined + in : + + struct bcm_msg_head { + __u32 opcode; + __u32 flags; + __u32 count; + struct timeval ival1, ival2; + canid_t can_id; + __u32 nframes; + struct can_frame frames[0]; + } + + `bcm_msg_head` must be 8 bytes aligned because of the `frames` member (see + `struct can_frame` definition). Must use native not standard types for packing. + """ + bcm_cmd_msg_fmt = "@3I4l2I" + bcm_cmd_msg_fmt += "x" * (struct.calcsize(bcm_cmd_msg_fmt) % 8) + + def setUp(self): + self.s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + self.addCleanup(self.s.close) + try: + self.s.bind((self.interface,)) + except OSError: + self.skipTest('network interface `%s` does not exist' % + self.interface) + + +class SocketRDSTest(unittest.TestCase): + + """To be able to run this test, the `rds` kernel module must be loaded: + # modprobe rds + """ + bufsize = 8192 + + def setUp(self): + self.serv = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + self.addCleanup(self.serv.close) + try: + self.port = support.bind_port(self.serv) + except OSError: + self.skipTest('unable to bind RDS socket') + + +class ThreadableTest: + """Threadable Test class + + The ThreadableTest class makes it easy to create a threaded + client/server pair from an existing unit test. To create a + new threaded class from an existing unit test, use multiple + inheritance: + + class NewClass (OldClass, ThreadableTest): + pass + + This class defines two new fixture functions with obvious + purposes for overriding: + + clientSetUp () + clientTearDown () + + Any new test functions within the class must then define + tests in pairs, where the test name is preceded with a + '_' to indicate the client portion of the test. Ex: + + def testFoo(self): + # Server portion + + def _testFoo(self): + # Client portion + + Any exceptions raised by the clients during their tests + are caught and transferred to the main thread to alert + the testing framework. + + Note, the server setup function cannot call any blocking + functions that rely on the client thread during setup, + unless serverExplicitReady() is called just before + the blocking call (such as in setting up a client/server + connection and performing the accept() in setUp(). + """ + + def __init__(self): + # Swap the true setup function + self.__setUp = self.setUp + self.__tearDown = self.tearDown + self.setUp = self._setUp + self.tearDown = self._tearDown + + def serverExplicitReady(self): + """This method allows the server to explicitly indicate that + it wants the client thread to proceed. This is useful if the + server is about to execute a blocking routine that is + dependent upon the client thread during its setup routine.""" + self.server_ready.set() + + def _setUp(self): + self.server_ready = threading.Event() + self.client_ready = threading.Event() + self.done = threading.Event() + self.queue = queue.Queue(1) + self.server_crashed = False + + # Do some munging to start the client test. + methodname = self.id() + i = methodname.rfind('.') + methodname = methodname[i+1:] + test_method = getattr(self, '_' + methodname) + self.client_thread = thread.start_new_thread( + self.clientRun, (test_method,)) + + try: + self.__setUp() + except: + self.server_crashed = True + raise + finally: + self.server_ready.set() + self.client_ready.wait() + + def _tearDown(self): + self.__tearDown() + self.done.wait() + + if self.queue.qsize(): + exc = self.queue.get() + raise exc + + def clientRun(self, test_func): + self.server_ready.wait() + self.clientSetUp() + self.client_ready.set() + if self.server_crashed: + self.clientTearDown() + return + if not hasattr(test_func, '__call__'): + raise TypeError("test_func must be a callable function") + try: + test_func() + except BaseException as e: + self.queue.put(e) + finally: + self.clientTearDown() + + def clientSetUp(self): + raise NotImplementedError("clientSetUp must be implemented.") + + def clientTearDown(self): + self.done.set() + thread.exit() + +class ThreadedTCPSocketTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketUDPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedCANSocketTest(SocketCANTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketCANTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + try: + self.cli.bind((self.interface,)) + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedRDSSocketTest(SocketRDSTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketRDSTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + try: + # RDS sockets must be bound explicitly to send or receive data + self.cli.bind((HOST, 0)) + self.cli_addr = self.cli.getsockname() + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class SocketConnectedTest(ThreadedTCPSocketTest): + """Socket tests for client-server connection. + + self.cli_conn is a client socket connected to the server. The + setUp() method guarantees that it is connected to the server. + """ + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + ThreadedTCPSocketTest.setUp(self) + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + ThreadedTCPSocketTest.tearDown(self) + + def clientSetUp(self): + ThreadedTCPSocketTest.clientSetUp(self) + self.cli.connect((HOST, self.port)) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + ThreadedTCPSocketTest.clientTearDown(self) + +class SocketPairTest(unittest.TestCase, ThreadableTest): + + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.serv, self.cli = socket.socketpair() + + def tearDown(self): + self.serv.close() + self.serv = None + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +# The following classes are used by the sendmsg()/recvmsg() tests. +# Combining, for instance, ConnectedStreamTestMixin and TCPTestBase +# gives a drop-in replacement for SocketConnectedTest, but different +# address families can be used, and the attributes serv_addr and +# cli_addr will be set to the addresses of the endpoints. + +class SocketTestBase(unittest.TestCase): + """A base class for socket tests. + + Subclasses must provide methods newSocket() to return a new socket + and bindSock(sock) to bind it to an unused address. + + Creates a socket self.serv and sets self.serv_addr to its address. + """ + + def setUp(self): + self.serv = self.newSocket() + self.bindServer() + + def bindServer(self): + """Bind server socket and set self.serv_addr to its address.""" + self.bindSock(self.serv) + self.serv_addr = self.serv.getsockname() + + def tearDown(self): + self.serv.close() + self.serv = None + + +class SocketListeningTestMixin(SocketTestBase): + """Mixin to listen on the server socket.""" + + def setUp(self): + super().setUp() + self.serv.listen() + + +class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, + ThreadableTest): + """Mixin to add client socket and allow client/server tests. + + Client socket is self.cli and its address is self.cli_addr. See + ThreadableTest for usage information. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = self.newClientSocket() + self.bindClient() + + def newClientSocket(self): + """Return a new socket for use as client.""" + return self.newSocket() + + def bindClient(self): + """Bind client socket and set self.cli_addr to its address.""" + self.bindSock(self.cli) + self.cli_addr = self.cli.getsockname() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +class ConnectedStreamTestMixin(SocketListeningTestMixin, + ThreadedSocketTestMixin): + """Mixin to allow client/server stream tests with connected client. + + Server's socket representing connection to client is self.cli_conn + and client's connection to server is self.serv_conn. (Based on + SocketConnectedTest.) + """ + + def setUp(self): + super().setUp() + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + super().tearDown() + + def clientSetUp(self): + super().clientSetUp() + self.cli.connect(self.serv_addr) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + super().clientTearDown() + + +class UnixSocketTestBase(SocketTestBase): + """Base class for Unix-domain socket tests.""" + + # This class is used for file descriptor passing tests, so we + # create the sockets in a private directory so that other users + # can't send anything that might be problematic for a privileged + # user running the tests. + + def setUp(self): + self.dir_path = tempfile.mkdtemp() + self.addCleanup(os.rmdir, self.dir_path) + super().setUp() + + def bindSock(self, sock): + path = tempfile.mktemp(dir=self.dir_path) + sock.bind(path) + self.addCleanup(support.unlink, path) + +class UnixStreamBase(UnixSocketTestBase): + """Base class for Unix-domain SOCK_STREAM tests.""" + + def newSocket(self): + return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + +class InetTestBase(SocketTestBase): + """Base class for IPv4 socket tests.""" + + host = HOST + + def setUp(self): + super().setUp() + self.port = self.serv_addr[1] + + def bindSock(self, sock): + support.bind_port(sock, host=self.host) + +class TCPTestBase(InetTestBase): + """Base class for TCP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + +class UDPTestBase(InetTestBase): + """Base class for UDP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + +class SCTPStreamBase(InetTestBase): + """Base class for SCTP tests in one-to-one (SOCK_STREAM) mode.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM, + socket.IPPROTO_SCTP) + + +class Inet6TestBase(InetTestBase): + """Base class for IPv6 socket tests.""" + + host = support.HOSTv6 + +class UDP6TestBase(Inet6TestBase): + """Base class for UDP-over-IPv6 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + + +# Test-skipping decorators for use with ThreadableTest. + +def skipWithClientIf(condition, reason): + """Skip decorated test if condition is true, add client_skip decorator. + + If the decorated object is not a class, sets its attribute + "client_skip" to a decorator which will return an empty function + if the test is to be skipped, or the original function if it is + not. This can be used to avoid running the client part of a + skipped test when using ThreadableTest. + """ + def client_pass(*args, **kwargs): + pass + def skipdec(obj): + retval = unittest.skip(reason)(obj) + if not isinstance(obj, type): + retval.client_skip = lambda f: client_pass + return retval + def noskipdec(obj): + if not (isinstance(obj, type) or hasattr(obj, "client_skip")): + obj.client_skip = lambda f: f + return obj + return skipdec if condition else noskipdec + + +def requireAttrs(obj, *attributes): + """Skip decorated test if obj is missing any of the given attributes. + + Sets client_skip attribute as skipWithClientIf() does. + """ + missing = [name for name in attributes if not hasattr(obj, name)] + return skipWithClientIf( + missing, "don't have " + ", ".join(name for name in missing)) + + +def requireSocket(*args): + """Skip decorated test if a socket cannot be created with given arguments. + + When an argument is given as a string, will use the value of that + attribute of the socket module, or skip the test if it doesn't + exist. Sets client_skip attribute as skipWithClientIf() does. + """ + err = None + missing = [obj for obj in args if + isinstance(obj, str) and not hasattr(socket, obj)] + if missing: + err = "don't have " + ", ".join(name for name in missing) + else: + callargs = [getattr(socket, obj) if isinstance(obj, str) else obj + for obj in args] + try: + s = socket.socket(*callargs) + except OSError as e: + # XXX: check errno? + err = str(e) + else: + s.close() + return skipWithClientIf( + err is not None, + "can't create socket({0}): {1}".format( + ", ".join(str(o) for o in args), err)) + + +####################################################################### +## Begin Tests + +class GeneralModuleTests(unittest.TestCase): + + def test_SocketType_is_socketobject(self): + import _socket + self.assertTrue(socket.SocketType is _socket.socket) + s = socket.socket() + self.assertIsInstance(s, socket.SocketType) + s.close() + + def test_repr(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with s: + self.assertIn('fd=%i' % s.fileno(), repr(s)) + self.assertIn('family=%s' % socket.AF_INET, repr(s)) + self.assertIn('type=%s' % socket.SOCK_STREAM, repr(s)) + self.assertIn('proto=0', repr(s)) + self.assertNotIn('raddr', repr(s)) + s.bind(('127.0.0.1', 0)) + self.assertIn('laddr', repr(s)) + self.assertIn(str(s.getsockname()), repr(s)) + self.assertIn('[closed]', repr(s)) + self.assertNotIn('laddr', repr(s)) + + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_csocket_repr(self): + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + try: + expected = ('' + % (s.fileno(), s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + finally: + s.close() + expected = ('' + % (s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + + def test_weakref(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + p = proxy(s) + self.assertEqual(p.fileno(), s.fileno()) + s.close() + s = None + support.gc_collect() + try: + p.fileno() + except ReferenceError: + pass + else: + self.fail('Socket proxy still exists') + + def testSocketError(self): + # Testing socket module exceptions + msg = "Error raising socket exception (%s)." + with self.assertRaises(OSError, msg=msg % 'OSError'): + raise OSError + with self.assertRaises(OSError, msg=msg % 'socket.herror'): + raise socket.herror + with self.assertRaises(OSError, msg=msg % 'socket.gaierror'): + raise socket.gaierror + + def testSendtoErrors(self): + # Testing that sendto doesn't mask failures. See #10169. + # PyPy note: made the test accept broader messages: PyPy's + # messages are equivalent but worded differently. + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind(('', 0)) + sockname = s.getsockname() + # 2 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', sockname) + self.assertIn(str(cm.exception), + ["a bytes-like object is required, not 'str'", # cpython + "a bytes-like object is required, not str"]) # pypy + with self.assertRaises(TypeError) as cm: + s.sendto(5j, sockname) + self.assertIn(str(cm.exception), + ["a bytes-like object is required, not 'complex'", + "a bytes-like object is required, not complex"]) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None) + self.assertIn('NoneType', str(cm.exception)) + # 3 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', 0, sockname) + self.assertIn(str(cm.exception), + ["a bytes-like object is required, not 'str'", + "a bytes-like object is required, not str"]) + with self.assertRaises(TypeError) as cm: + s.sendto(5j, 0, sockname) + self.assertIn(str(cm.exception), + ["a bytes-like object is required, not 'complex'", + "a bytes-like object is required, not complex"]) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, None) + self.assertIn('NoneType', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 'bar', sockname) + self.assertIn('integer', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None, None) + self.assertIn('integer', str(cm.exception)) + # wrong number of args + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo') + if support.check_impl_detail(): + self.assertIn(' given)', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, sockname, 4) + self.assertIn(' given', str(cm.exception)) + + def testCrucialConstants(self): + # Testing for mission critical constants + socket.AF_INET + socket.SOCK_STREAM + socket.SOCK_DGRAM + socket.SOCK_RAW + socket.SOCK_RDM + socket.SOCK_SEQPACKET + socket.SOL_SOCKET + socket.SO_REUSEADDR + + def testHostnameRes(self): + # Testing hostname resolution mechanisms + hostname = socket.gethostname() + try: + ip = socket.gethostbyname(hostname) + except OSError: + # Probably name lookup wasn't set up right; skip this test + self.skipTest('name lookup failure') + self.assertTrue(ip.find('.') >= 0, "Error resolving host to ip.") + try: + hname, aliases, ipaddrs = socket.gethostbyaddr(ip) + except OSError: + # Probably a similar problem as above; skip this test + self.skipTest('name lookup failure') + all_host_names = [hostname, hname] + aliases + fqhn = socket.getfqdn(ip) + if not fqhn in all_host_names: + self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + + def test_host_resolution(self): + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + self.assertRaises(OSError, socket.gethostbyname, addr) + self.assertRaises(OSError, socket.gethostbyaddr, addr) + + for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: + self.assertEqual(socket.gethostbyname(addr), addr) + + # we don't test support.HOSTv6 because there's a chance it doesn't have + # a matching name entry (e.g. 'ip6-localhost') + for host in [support.HOST]: + self.assertIn(host, socket.gethostbyaddr(host)[2]) + + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") + @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") + def test_sethostname(self): + oldhn = socket.gethostname() + try: + socket.sethostname('new') + except OSError as e: + if e.errno == errno.EPERM: + self.skipTest("test should be run as root") + else: + raise + try: + # running test as root! + self.assertEqual(socket.gethostname(), 'new') + # Should work with bytes objects too + socket.sethostname(b'bar') + self.assertEqual(socket.gethostname(), 'bar') + finally: + socket.sethostname(oldhn) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInterfaceNameIndex(self): + interfaces = socket.if_nameindex() + for index, name in interfaces: + self.assertIsInstance(index, int) + self.assertIsInstance(name, str) + # interface indices are non-zero integers + self.assertGreater(index, 0) + _index = socket.if_nametoindex(name) + self.assertIsInstance(_index, int) + self.assertEqual(index, _index) + _name = socket.if_indextoname(index) + self.assertIsInstance(_name, str) + self.assertEqual(name, _name) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInvalidInterfaceNameIndex(self): + # test nonexistent interface index/name + self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') + # test with invalid values + self.assertRaises(TypeError, socket.if_nametoindex, 0) + self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') + def testRefCountGetNameInfo(self): + # Testing reference count for getnameinfo + try: + # On some versions, this loses a reference + orig = sys.getrefcount(__name__) + socket.getnameinfo(__name__,0) + except TypeError: + if sys.getrefcount(__name__) != orig: + self.fail("socket.getnameinfo loses a reference") + + def testInterpreterCrash(self): + # Making sure getnameinfo doesn't crash the interpreter + try: + # On some versions, this crashes the interpreter. + socket.getnameinfo(('x', 0, 0, 0), 0) + except OSError: + pass + + def testNtoH(self): + # This just checks that htons etc. are their own inverse, + # when looking at the lower 16 or 32 bits. + sizes = {socket.htonl: 32, socket.ntohl: 32, + socket.htons: 16, socket.ntohs: 16} + for func, size in sizes.items(): + mask = (1<") + + def test_unusable_closed_socketio(self): + with socket.socket() as sock: + fp = sock.makefile("rb", buffering=0) + self.assertTrue(fp.readable()) + self.assertFalse(fp.writable()) + self.assertFalse(fp.seekable()) + fp.close() + self.assertRaises(ValueError, fp.readable) + self.assertRaises(ValueError, fp.writable) + self.assertRaises(ValueError, fp.seekable) + + def test_makefile_mode(self): + for mode in 'r', 'rb', 'rw', 'w', 'wb': + with self.subTest(mode=mode): + with socket.socket() as sock: + with sock.makefile(mode) as fp: + self.assertEqual(fp.mode, mode) + + def test_makefile_invalid_mode(self): + for mode in 'rt', 'x', '+', 'a': + with self.subTest(mode=mode): + with socket.socket() as sock: + with self.assertRaisesRegex(ValueError, 'invalid mode'): + sock.makefile(mode) + + def test_pickle(self): + sock = socket.socket() + with sock: + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, sock, protocol) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + family = pickle.loads(pickle.dumps(socket.AF_INET, protocol)) + self.assertEqual(family, socket.AF_INET) + type = pickle.loads(pickle.dumps(socket.SOCK_STREAM, protocol)) + self.assertEqual(type, socket.SOCK_STREAM) + + def test_listen_backlog(self): + for backlog in 0, -1: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv: + srv.bind((HOST, 0)) + srv.listen(backlog) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv: + srv.bind((HOST, 0)) + srv.listen() + + @support.cpython_only + def test_listen_backlog_overflow(self): + # Issue 15989 + import _testcapi + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.bind((HOST, 0)) + self.assertRaises(OverflowError, srv.listen, _testcapi.INT_MAX + 1) + srv.close() + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + def test_flowinfo(self): + self.assertRaises(OverflowError, socket.getnameinfo, + (support.HOSTv6, 0, 0xffffffff), 0) + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + self.assertRaises(OverflowError, s.bind, (support.HOSTv6, 0, -10)) + + def test_str_for_enums(self): + # Make sure that the AF_* and SOCK_* constants have enum-like string + # reprs. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + self.assertEqual(str(s.family), 'AddressFamily.AF_INET') + self.assertEqual(str(s.type), 'SocketKind.SOCK_STREAM') + + @unittest.skipIf(os.name == 'nt', 'Will not work on Windows') + def test_uknown_socket_family_repr(self): + # Test that when created with a family that's not one of the known + # AF_*/SOCK_* constants, socket.family just returns the number. + # + # To do this we fool socket.socket into believing it already has an + # open fd because on this path it doesn't actually verify the family and + # type and populates the socket object. + # + # On Windows this trick won't work, so the test is skipped. + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + with socket.socket(family=42424, type=13331, fileno=fd) as s: + self.assertEqual(s.family, 42424) + self.assertEqual(s.type, 13331) + + @unittest.skipUnless(hasattr(os, 'sendfile'), 'test needs os.sendfile()') + def test__sendfile_use_sendfile(self): + class File: + def __init__(self, fd): + self.fd = fd + + def fileno(self): + return self.fd + with socket.socket() as sock: + fd = os.open(os.curdir, os.O_RDONLY) + os.close(fd) + with self.assertRaises(socket._GiveupOnSendfile): + sock._sendfile_use_sendfile(File(fd)) + with self.assertRaises(OverflowError): + sock._sendfile_use_sendfile(File(2**1000)) + with self.assertRaises(TypeError): + sock._sendfile_use_sendfile(File(None)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +class BasicCANTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_CAN + socket.PF_CAN + socket.CAN_RAW + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCMConstants(self): + socket.CAN_BCM + + # opcodes + socket.CAN_BCM_TX_SETUP # create (cyclic) transmission task + socket.CAN_BCM_TX_DELETE # remove (cyclic) transmission task + socket.CAN_BCM_TX_READ # read properties of (cyclic) transmission task + socket.CAN_BCM_TX_SEND # send one CAN frame + socket.CAN_BCM_RX_SETUP # create RX content filter subscription + socket.CAN_BCM_RX_DELETE # remove RX content filter subscription + socket.CAN_BCM_RX_READ # read properties of RX content filter subscription + socket.CAN_BCM_TX_STATUS # reply to TX_READ request + socket.CAN_BCM_TX_EXPIRED # notification on performed transmissions (count=0) + socket.CAN_BCM_RX_STATUS # reply to RX_READ request + socket.CAN_BCM_RX_TIMEOUT # cyclic message is absent + socket.CAN_BCM_RX_CHANGED # updated CAN frame (detected content change) + + def testCreateSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + pass + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testCreateBCMSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) as s: + pass + + def testBindAny(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.bind(('', )) + + def testTooLongInterfaceName(self): + # most systems limit IFNAMSIZ to 16, take 1024 to be sure + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + self.assertRaisesRegex(OSError, 'interface name too long', + s.bind, ('x' * 1024,)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_LOOPBACK"), + 'socket.CAN_RAW_LOOPBACK required for this test.') + def testLoopback(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + for loopback in (0, 1): + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK, + loopback) + self.assertEqual(loopback, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_FILTER"), + 'socket.CAN_RAW_FILTER required for this test.') + def testFilter(self): + can_id, can_mask = 0x200, 0x700 + can_filter = struct.pack("=II", can_id, can_mask) + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, can_filter) + self.assertEqual(can_filter, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, 8)) + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, bytearray(can_filter)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class CANTest(ThreadedCANSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedCANSocketTest.__init__(self, methodName=methodName) + + @classmethod + def build_can_frame(cls, can_id, data): + """Build a CAN frame.""" + can_dlc = len(data) + data = data.ljust(8, b'\x00') + return struct.pack(cls.can_frame_fmt, can_id, can_dlc, data) + + @classmethod + def dissect_can_frame(cls, frame): + """Dissect a CAN frame.""" + can_id, can_dlc, data = struct.unpack(cls.can_frame_fmt, frame) + return (can_id, can_dlc, data[:can_dlc]) + + def testSendFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + self.assertEqual(addr[0], self.interface) + self.assertEqual(addr[1], socket.AF_CAN) + + def _testSendFrame(self): + self.cf = self.build_can_frame(0x00, b'\x01\x02\x03\x04\x05') + self.cli.send(self.cf) + + def testSendMaxFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + + def _testSendMaxFrame(self): + self.cf = self.build_can_frame(0x00, b'\x07' * 8) + self.cli.send(self.cf) + + def testSendMultiFrames(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf1, cf) + + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf2, cf) + + def _testSendMultiFrames(self): + self.cf1 = self.build_can_frame(0x07, b'\x44\x33\x22\x11') + self.cli.send(self.cf1) + + self.cf2 = self.build_can_frame(0x12, b'\x99\x22\x33') + self.cli.send(self.cf2) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def _testBCM(self): + cf, addr = self.cli.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + can_id, can_dlc, data = self.dissect_can_frame(cf) + self.assertEqual(self.can_id, can_id) + self.assertEqual(self.data, data) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCM(self): + bcm = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) + self.addCleanup(bcm.close) + bcm.connect((self.interface,)) + self.can_id = 0x123 + self.data = bytes([0xc0, 0xff, 0xee]) + self.cf = self.build_can_frame(self.can_id, self.data) + opcode = socket.CAN_BCM_TX_SEND + flags = 0 + count = 0 + ival1_seconds = ival1_usec = ival2_seconds = ival2_usec = 0 + bcm_can_id = 0x0222 + nframes = 1 + assert len(self.cf) == 16 + header = struct.pack(self.bcm_cmd_msg_fmt, + opcode, + flags, + count, + ival1_seconds, + ival1_usec, + ival2_seconds, + ival2_usec, + bcm_can_id, + nframes, + ) + header_plus_frame = header + self.cf + bytes_sent = bcm.send(header_plus_frame) + self.assertEqual(bytes_sent, len(header_plus_frame)) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +class BasicRDSTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_RDS + socket.PF_RDS + + def testCreateSocket(self): + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + pass + + def testSocketBufferSize(self): + bufsize = 16384 + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, bufsize) + s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, bufsize) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class RDSTest(ThreadedRDSSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedRDSSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + super().setUp() + self.evt = threading.Event() + + def testSendAndRecv(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + self.assertEqual(self.cli_addr, addr) + + def _testSendAndRecv(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testPeek(self): + data, addr = self.serv.recvfrom(self.bufsize, socket.MSG_PEEK) + self.assertEqual(self.data, data) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testPeek(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + @requireAttrs(socket.socket, 'recvmsg') + def testSendAndRecvMsg(self): + data, ancdata, msg_flags, addr = self.serv.recvmsg(self.bufsize) + self.assertEqual(self.data, data) + + @requireAttrs(socket.socket, 'sendmsg') + def _testSendAndRecvMsg(self): + self.data = b'hello ' * 10 + self.cli.sendmsg([self.data], (), 0, (HOST, self.port)) + + def testSendAndRecvMulti(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data1, data) + + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data2, data) + + def _testSendAndRecvMulti(self): + self.data1 = b'bacon' + self.cli.sendto(self.data1, 0, (HOST, self.port)) + + self.data2 = b'egg' + self.cli.sendto(self.data2, 0, (HOST, self.port)) + + def testSelect(self): + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testSelect(self): + self.data = b'select' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testCongestion(self): + # wait until the sender is done + self.evt.wait() + + def _testCongestion(self): + # test the behavior in case of congestion + self.data = b'fill' + self.cli.setblocking(False) + try: + # try to lower the receiver's socket buffer size + self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384) + except OSError: + pass + with self.assertRaises(OSError) as cm: + try: + # fill the receiver's socket buffer + while True: + self.cli.sendto(self.data, 0, (HOST, self.port)) + finally: + # signal the receiver we're done + self.evt.set() + # sendto() should have failed with ENOBUFS + self.assertEqual(cm.exception.errno, errno.ENOBUFS) + # and we should have received a congestion notification through poll + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicTCPTest(SocketConnectedTest): + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecv(self): + # Testing large receive over TCP + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.serv_conn.send(MSG) + + def testOverFlowRecv(self): + # Testing receive in chunks over TCP + seg1 = self.cli_conn.recv(len(MSG) - 3) + seg2 = self.cli_conn.recv(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecv(self): + self.serv_conn.send(MSG) + + def testRecvFrom(self): + # Testing large recvfrom() over TCP + msg, addr = self.cli_conn.recvfrom(1024) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.serv_conn.send(MSG) + + def testOverFlowRecvFrom(self): + # Testing recvfrom() in chunks over TCP + seg1, addr = self.cli_conn.recvfrom(len(MSG)-3) + seg2, addr = self.cli_conn.recvfrom(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecvFrom(self): + self.serv_conn.send(MSG) + + def testSendAll(self): + # Testing sendall() with a 2048 byte string over TCP + msg = b'' + while 1: + read = self.cli_conn.recv(1024) + if not read: + break + msg += read + self.assertEqual(msg, b'f' * 2048) + + def _testSendAll(self): + big_chunk = b'f' * 2048 + self.serv_conn.sendall(big_chunk) + + def testFromFd(self): + # Testing fromfd() + fd = self.cli_conn.fileno() + sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + self.assertIsInstance(sock, socket.socket) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testFromFd(self): + self.serv_conn.send(MSG) + + def testDup(self): + # Testing dup() + sock = self.cli_conn.dup() + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDup(self): + self.serv_conn.send(MSG) + + def testShutdown(self): + # Testing shutdown() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + # wait for _testShutdown to finish: on OS X, when the server + # closes the connection the client also becomes disconnected, + # and the client's shutdown call will fail. (Issue #4397.) + self.done.wait() + + def _testShutdown(self): + self.serv_conn.send(MSG) + self.serv_conn.shutdown(2) + + testShutdown_overflow = support.cpython_only(testShutdown) + + @support.cpython_only + def _testShutdown_overflow(self): + import _testcapi + self.serv_conn.send(MSG) + # Issue 15989 + self.assertRaises(OverflowError, self.serv_conn.shutdown, + _testcapi.INT_MAX + 1) + self.assertRaises(OverflowError, self.serv_conn.shutdown, + 2 + (_testcapi.UINT_MAX + 1)) + self.serv_conn.shutdown(2) + + def testDetach(self): + # Testing detach() + fileno = self.cli_conn.fileno() + f = self.cli_conn.detach() + self.assertEqual(f, fileno) + # cli_conn cannot be used anymore... + self.assertTrue(self.cli_conn._closed) + self.assertRaises(OSError, self.cli_conn.recv, 1024) + self.cli_conn.close() + # ...but we can create another socket using the (still open) + # file descriptor + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=f) + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDetach(self): + self.serv_conn.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicUDPTest(ThreadedUDPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedUDPSocketTest.__init__(self, methodName=methodName) + + def testSendtoAndRecv(self): + # Testing sendto() and Recv() over UDP + msg = self.serv.recv(len(MSG)) + self.assertEqual(msg, MSG) + + def _testSendtoAndRecv(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFrom(self): + # Testing recvfrom() over UDP + msg, addr = self.serv.recvfrom(len(MSG)) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFromNegative(self): + # Negative lengths passed to recvfrom should give ValueError. + self.assertRaises(ValueError, self.serv.recvfrom, -1) + + def _testRecvFromNegative(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + +# Tests for the sendmsg()/recvmsg() interface. Where possible, the +# same test code is used with different families and types of socket +# (e.g. stream, datagram), and tests using recvmsg() are repeated +# using recvmsg_into(). +# +# The generic test classes such as SendmsgTests and +# RecvmsgGenericTests inherit from SendrecvmsgBase and expect to be +# supplied with sockets cli_sock and serv_sock representing the +# client's and the server's end of the connection respectively, and +# attributes cli_addr and serv_addr holding their (numeric where +# appropriate) addresses. +# +# The final concrete test classes combine these with subclasses of +# SocketTestBase which set up client and server sockets of a specific +# type, and with subclasses of SendrecvmsgBase such as +# SendrecvmsgDgramBase and SendrecvmsgConnectedBase which map these +# sockets to cli_sock and serv_sock and override the methods and +# attributes of SendrecvmsgBase to fill in destination addresses if +# needed when sending, check for specific flags in msg_flags, etc. +# +# RecvmsgIntoMixin provides a version of doRecvmsg() implemented using +# recvmsg_into(). + +# XXX: like the other datagram (UDP) tests in this module, the code +# here assumes that datagram delivery on the local machine will be +# reliable. + +class SendrecvmsgBase(ThreadSafeCleanupTestCase): + # Base class for sendmsg()/recvmsg() tests. + + # Time in seconds to wait before considering a test failed, or + # None for no timeout. Not all tests actually set a timeout. + fail_timeout = 3.0 + + def setUp(self): + self.misc_event = threading.Event() + super().setUp() + + def sendToServer(self, msg): + # Send msg to the server. + return self.cli_sock.send(msg) + + # Tuple of alternative default arguments for sendmsg() when called + # via sendmsgToServer() (e.g. to include a destination address). + sendmsg_to_server_defaults = () + + def sendmsgToServer(self, *args): + # Call sendmsg() on self.cli_sock with the given arguments, + # filling in any arguments which are not supplied with the + # corresponding items of self.sendmsg_to_server_defaults, if + # any. + return self.cli_sock.sendmsg( + *(args + self.sendmsg_to_server_defaults[len(args):])) + + def doRecvmsg(self, sock, bufsize, *args): + # Call recvmsg() on sock with given arguments and return its + # result. Should be used for tests which can use either + # recvmsg() or recvmsg_into() - RecvmsgIntoMixin overrides + # this method with one which emulates it using recvmsg_into(), + # thus allowing the same test to be used for both methods. + result = sock.recvmsg(bufsize, *args) + self.registerRecvmsgResult(result) + return result + + def registerRecvmsgResult(self, result): + # Called by doRecvmsg() with the return value of recvmsg() or + # recvmsg_into(). Can be overridden to arrange cleanup based + # on the returned ancillary data, for instance. + pass + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer. + self.assertEqual(addr1, addr2) + + # Flags that are normally unset in msg_flags + msg_flags_common_unset = 0 + for name in ("MSG_CTRUNC", "MSG_OOB"): + msg_flags_common_unset |= getattr(socket, name, 0) + + # Flags that are normally set + msg_flags_common_set = 0 + + # Flags set when a complete record has been received (e.g. MSG_EOR + # for SCTP) + msg_flags_eor_indicator = 0 + + # Flags set when a complete record has not been received + # (e.g. MSG_TRUNC for datagram sockets) + msg_flags_non_eor_indicator = 0 + + def checkFlags(self, flags, eor=None, checkset=0, checkunset=0, ignore=0): + # Method to check the value of msg_flags returned by recvmsg[_into](). + # + # Checks that all bits in msg_flags_common_set attribute are + # set in "flags" and all bits in msg_flags_common_unset are + # unset. + # + # The "eor" argument specifies whether the flags should + # indicate that a full record (or datagram) has been received. + # If "eor" is None, no checks are done; otherwise, checks + # that: + # + # * if "eor" is true, all bits in msg_flags_eor_indicator are + # set and all bits in msg_flags_non_eor_indicator are unset + # + # * if "eor" is false, all bits in msg_flags_non_eor_indicator + # are set and all bits in msg_flags_eor_indicator are unset + # + # If "checkset" and/or "checkunset" are supplied, they require + # the given bits to be set or unset respectively, overriding + # what the attributes require for those bits. + # + # If any bits are set in "ignore", they will not be checked, + # regardless of the other inputs. + # + # Will raise Exception if the inputs require a bit to be both + # set and unset, and it is not ignored. + + defaultset = self.msg_flags_common_set + defaultunset = self.msg_flags_common_unset + + if eor: + defaultset |= self.msg_flags_eor_indicator + defaultunset |= self.msg_flags_non_eor_indicator + elif eor is not None: + defaultset |= self.msg_flags_non_eor_indicator + defaultunset |= self.msg_flags_eor_indicator + + # Function arguments override defaults + defaultset &= ~checkunset + defaultunset &= ~checkset + + # Merge arguments with remaining defaults, and check for conflicts + checkset |= defaultset + checkunset |= defaultunset + inboth = checkset & checkunset & ~ignore + if inboth: + raise Exception("contradictory set, unset requirements for flags " + "{0:#x}".format(inboth)) + + # Compare with given msg_flags value + mask = (checkset | checkunset) & ~ignore + self.assertEqual(flags & mask, checkset & mask) + + +class RecvmsgIntoMixin(SendrecvmsgBase): + # Mixin to implement doRecvmsg() using recvmsg_into(). + + def doRecvmsg(self, sock, bufsize, *args): + buf = bytearray(bufsize) + result = sock.recvmsg_into([buf], *args) + self.registerRecvmsgResult(result) + self.assertGreaterEqual(result[0], 0) + self.assertLessEqual(result[0], bufsize) + return (bytes(buf[:result[0]]),) + result[1:] + + +class SendrecvmsgDgramFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for datagram sockets. + + @property + def msg_flags_non_eor_indicator(self): + return super().msg_flags_non_eor_indicator | socket.MSG_TRUNC + + +class SendrecvmsgSCTPFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for SCTP sockets. + + @property + def msg_flags_eor_indicator(self): + return super().msg_flags_eor_indicator | socket.MSG_EOR + + +class SendrecvmsgConnectionlessBase(SendrecvmsgBase): + # Base class for tests on connectionless-mode sockets. Users must + # supply sockets on attributes cli and serv to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.serv + + @property + def cli_sock(self): + return self.cli + + @property + def sendmsg_to_server_defaults(self): + return ([], [], 0, self.serv_addr) + + def sendToServer(self, msg): + return self.cli_sock.sendto(msg, self.serv_addr) + + +class SendrecvmsgConnectedBase(SendrecvmsgBase): + # Base class for tests on connected sockets. Users must supply + # sockets on attributes serv_conn and cli_conn (representing the + # connections *to* the server and the client), to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.cli_conn + + @property + def cli_sock(self): + return self.serv_conn + + def checkRecvmsgAddress(self, addr1, addr2): + # Address is currently "unspecified" for a connected socket, + # so we don't examine it + pass + + +class SendrecvmsgServerTimeoutBase(SendrecvmsgBase): + # Base class to set a timeout on server's socket. + + def setUp(self): + super().setUp() + self.serv_sock.settimeout(self.fail_timeout) + + +class SendmsgTests(SendrecvmsgServerTimeoutBase): + # Tests for sendmsg() which can use any socket type and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsg(self): + # Send a simple message with sendmsg(). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG]), len(MSG)) + + def testSendmsgDataGenerator(self): + # Send from buffer obtained from a generator (not a sequence). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgDataGenerator(self): + self.assertEqual(self.sendmsgToServer((o for o in [MSG])), + len(MSG)) + + def testSendmsgAncillaryGenerator(self): + # Gather (empty) ancillary data from a generator. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgAncillaryGenerator(self): + self.assertEqual(self.sendmsgToServer([MSG], (o for o in [])), + len(MSG)) + + def testSendmsgArray(self): + # Send data from an array instead of the usual bytes object. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgArray(self): + self.assertEqual(self.sendmsgToServer([array.array("B", MSG)]), + len(MSG)) + + def testSendmsgGather(self): + # Send message data from more than one buffer (gather write). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgGather(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + def testSendmsgBadArgs(self): + # Check that sendmsg() rejects invalid arguments. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadArgs(self): + self.assertRaises(TypeError, self.cli_sock.sendmsg) + self.assertRaises(TypeError, self.sendmsgToServer, + b"not in an iterable") + self.assertRaises(TypeError, self.sendmsgToServer, + object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG, object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], 0, object()) + self.sendToServer(b"done") + + def testSendmsgBadCmsg(self): + # Check that invalid ancillary data items are rejected. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(object(), 0, b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, object(), b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, object())]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0)]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b"data", 42)]) + self.sendToServer(b"done") + + @requireAttrs(socket, "CMSG_SPACE") + def testSendmsgBadMultiCmsg(self): + # Check that invalid ancillary data items are rejected when + # more than one item is present. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + @testSendmsgBadMultiCmsg.client_skip + def _testSendmsgBadMultiCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [0, 0, b""]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b""), object()]) + self.sendToServer(b"done") + + def testSendmsgExcessCmsgReject(self): + # Check that sendmsg() rejects excess ancillary data items + # when the number that can be sent is limited. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgExcessCmsgReject(self): + if not hasattr(socket, "CMSG_SPACE"): + # Can only send one item + with self.assertRaises(OSError) as cm: + self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")]) + self.assertIsNone(cm.exception.errno) + self.sendToServer(b"done") + + def testSendmsgAfterClose(self): + # Check that sendmsg() fails on a closed socket. + pass + + def _testSendmsgAfterClose(self): + self.cli_sock.close() + self.assertRaises(OSError, self.sendmsgToServer, [MSG]) + + +class SendmsgStreamTests(SendmsgTests): + # Tests for sendmsg() which require a stream socket and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsgExplicitNoneAddr(self): + # Check that peer address can be specified as None. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgExplicitNoneAddr(self): + self.assertEqual(self.sendmsgToServer([MSG], [], 0, None), len(MSG)) + + def testSendmsgTimeout(self): + # Check that timeout works with sendmsg(). + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + def _testSendmsgTimeout(self): + try: + self.cli_sock.settimeout(0.03) + with self.assertRaises(socket.timeout): + while True: + self.sendmsgToServer([b"a"*512]) + finally: + self.misc_event.set() + + # XXX: would be nice to have more tests for sendmsg flags argument. + + # Linux supports MSG_DONTWAIT when sending, but in general, it + # only works when receiving. Could add other platforms if they + # support it too. + @skipWithClientIf(sys.platform not in {"linux"}, + "MSG_DONTWAIT not known to work on this platform when " + "sending") + def testSendmsgDontWait(self): + # Check that MSG_DONTWAIT in flags causes non-blocking behaviour. + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @testSendmsgDontWait.client_skip + def _testSendmsgDontWait(self): + try: + with self.assertRaises(OSError) as cm: + while True: + self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT) + self.assertIn(cm.exception.errno, + (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + self.misc_event.set() + + +class SendmsgConnectionlessTests(SendmsgTests): + # Tests for sendmsg() which require a connectionless-mode + # (e.g. datagram) socket, and do not involve recvmsg() or + # recvmsg_into(). + + def testSendmsgNoDestAddr(self): + # Check that sendmsg() fails when no destination address is + # given for unconnected socket. + pass + + def _testSendmsgNoDestAddr(self): + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG]) + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG], [], 0, None) + + +class RecvmsgGenericTests(SendrecvmsgBase): + # Tests for recvmsg() which can also be emulated using + # recvmsg_into(), and can use any socket type. + + def testRecvmsg(self): + # Receive a simple message with recvmsg[_into](). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsg(self): + self.sendToServer(MSG) + + def testRecvmsgExplicitDefaults(self): + # Test recvmsg[_into]() with default arguments provided explicitly. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgExplicitDefaults(self): + self.sendToServer(MSG) + + def testRecvmsgShorter(self): + # Receive a message smaller than buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) + 42) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShorter(self): + self.sendToServer(MSG) + + # FreeBSD < 8 doesn't always set the MSG_TRUNC flag when a truncated + # datagram is received (issue #13001). + @support.requires_freebsd_version(8) + def testRecvmsgTrunc(self): + # Receive part of message, check for truncation indicators. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + @support.requires_freebsd_version(8) + def _testRecvmsgTrunc(self): + self.sendToServer(MSG) + + def testRecvmsgShortAncillaryBuf(self): + # Test ancillary data buffer too small to hold any ancillary data. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 1) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShortAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgLongAncillaryBuf(self): + # Test large ancillary data buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgLongAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgAfterClose(self): + # Check that recvmsg[_into]() fails on a closed socket. + self.serv_sock.close() + self.assertRaises(OSError, self.doRecvmsg, self.serv_sock, 1024) + + def _testRecvmsgAfterClose(self): + pass + + def testRecvmsgTimeout(self): + # Check that timeout works. + try: + self.serv_sock.settimeout(0.03) + self.assertRaises(socket.timeout, + self.doRecvmsg, self.serv_sock, len(MSG)) + finally: + self.misc_event.set() + + def _testRecvmsgTimeout(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @requireAttrs(socket, "MSG_PEEK") + def testRecvmsgPeek(self): + # Check that MSG_PEEK in flags enables examination of pending + # data without consuming it. + + # Receive part of data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3, 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + # Ignoring MSG_TRUNC here (so this test is the same for stream + # and datagram sockets). Some wording in POSIX seems to + # suggest that it needn't be set when peeking, but that may + # just be a slip. + self.checkFlags(flags, eor=False, + ignore=getattr(socket, "MSG_TRUNC", 0)) + + # Receive all data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + # Check that the same data can still be received normally. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgPeek.client_skip + def _testRecvmsgPeek(self): + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + def testRecvmsgFromSendmsg(self): + # Test receiving with recvmsg[_into]() when message is sent + # using sendmsg(). + self.serv_sock.settimeout(self.fail_timeout) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgFromSendmsg.client_skip + def _testRecvmsgFromSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + +class RecvmsgGenericStreamTests(RecvmsgGenericTests): + # Tests which require a stream socket and can use either recvmsg() + # or recvmsg_into(). + + def testRecvmsgEOF(self): + # Receive end-of-stream indicator (b"", peer socket closed). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.assertEqual(msg, b"") + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=None) # Might not have end-of-record marker + + def _testRecvmsgEOF(self): + self.cli_sock.close() + + def testRecvmsgOverflow(self): + # Receive a message in more than one chunk. + seg1, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + seg2, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testRecvmsgOverflow(self): + self.sendToServer(MSG) + + +class RecvmsgTests(RecvmsgGenericTests): + # Tests for recvmsg() which can use any socket type. + + def testRecvmsgBadArgs(self): + # Check that recvmsg() rejects invalid arguments. + self.assertRaises(TypeError, self.serv_sock.recvmsg) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + -1, 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + len(MSG), -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + [bytearray(10)], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + object(), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), 0, object()) + + msg, ancdata, flags, addr = self.serv_sock.recvmsg(len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgBadArgs(self): + self.sendToServer(MSG) + + +class RecvmsgIntoTests(RecvmsgIntoMixin, RecvmsgGenericTests): + # Tests for recvmsg_into() which can use any socket type. + + def testRecvmsgIntoBadArgs(self): + # Check that recvmsg_into() rejects invalid arguments. + buf = bytearray(len(MSG)) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + len(MSG), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + buf, 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [object()], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [b"I'm not writable"], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf, object()], 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg_into, + [buf], -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], 0, object()) + + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf], 0, 0) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoBadArgs(self): + self.sendToServer(MSG) + + def testRecvmsgIntoGenerator(self): + # Receive into buffer obtained from a generator (not a sequence). + buf = bytearray(len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + (o for o in [buf])) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoGenerator(self): + self.sendToServer(MSG) + + def testRecvmsgIntoArray(self): + # Receive into an array rather than the usual bytearray. + buf = array.array("B", [0] * len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf]) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf.tobytes(), MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoArray(self): + self.sendToServer(MSG) + + def testRecvmsgIntoScatter(self): + # Receive into multiple buffers (scatter write). + b1 = bytearray(b"----") + b2 = bytearray(b"0123456789") + b3 = bytearray(b"--------------") + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + [b1, memoryview(b2)[2:9], b3]) + self.assertEqual(nbytes, len(b"Mary had a little lamb")) + self.assertEqual(b1, bytearray(b"Mary")) + self.assertEqual(b2, bytearray(b"01 had a 9")) + self.assertEqual(b3, bytearray(b"little lamb---")) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoScatter(self): + self.sendToServer(b"Mary had a little lamb") + + +class CmsgMacroTests(unittest.TestCase): + # Test the functions CMSG_LEN() and CMSG_SPACE(). Tests + # assumptions used by sendmsg() and recvmsg[_into](), which share + # code with these functions. + + # Match the definition in socketmodule.c + try: + import _testcapi + except ImportError: + socklen_t_limit = 0x7fffffff + else: + socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) + + @requireAttrs(socket, "CMSG_LEN") + def testCMSG_LEN(self): + # Test CMSG_LEN() with various valid and invalid values, + # checking the assumptions used by recvmsg() and sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_LEN(0) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(socket.CMSG_LEN(0), array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_LEN(n) + # This is how recvmsg() calculates the data size + self.assertEqual(ret - socket.CMSG_LEN(0), n) + self.assertLessEqual(ret, self.socklen_t_limit) + + self.assertRaises(OverflowError, socket.CMSG_LEN, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_LEN, toobig) + self.assertRaises(OverflowError, socket.CMSG_LEN, sys.maxsize) + + @requireAttrs(socket, "CMSG_SPACE") + def testCMSG_SPACE(self): + # Test CMSG_SPACE() with various valid and invalid values, + # checking the assumptions used by sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + last = socket.CMSG_SPACE(0) + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(last, array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_SPACE(n) + self.assertGreaterEqual(ret, last) + self.assertGreaterEqual(ret, socket.CMSG_LEN(n)) + self.assertGreaterEqual(ret, n + socket.CMSG_LEN(0)) + self.assertLessEqual(ret, self.socklen_t_limit) + last = ret + + self.assertRaises(OverflowError, socket.CMSG_SPACE, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_SPACE, toobig) + self.assertRaises(OverflowError, socket.CMSG_SPACE, sys.maxsize) + + +class SCMRightsTest(SendrecvmsgServerTimeoutBase): + # Tests for file descriptor passing on Unix-domain sockets. + + # Invalid file descriptor value that's unlikely to evaluate to a + # real FD even if one of its bytes is replaced with a different + # value (which shouldn't actually happen). + badfd = -0x5555 + + def newFDs(self, n): + # Return a list of n file descriptors for newly-created files + # containing their list indices as ASCII numbers. + fds = [] + for i in range(n): + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + self.addCleanup(os.close, fd) + os.write(fd, str(i).encode()) + fds.append(fd) + return fds + + def checkFDs(self, fds): + # Check that the file descriptors in the given list contain + # their correct list indices as ASCII numbers. + for n, fd in enumerate(fds): + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(os.read(fd, 1024), str(n).encode()) + + def registerRecvmsgResult(self, result): + self.addCleanup(self.closeRecvmsgFDs, result) + + def closeRecvmsgFDs(self, recvmsg_result): + # Close all file descriptors specified in the ancillary data + # of the given return value from recvmsg() or recvmsg_into(). + for cmsg_level, cmsg_type, cmsg_data in recvmsg_result[1]: + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + for fd in fds: + os.close(fd) + + def createAndSendFDs(self, n): + # Send n new file descriptors created by newFDs() to the + # server, with the constant MSG as the non-ancillary data. + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(n)))]), + len(MSG)) + + def checkRecvmsgFDs(self, numfds, result, maxcmsgs=1, ignoreflags=0): + # Check that constant MSG was received with numfds file + # descriptors in a maximum of maxcmsgs control messages (which + # must contain only complete integers). By default, check + # that MSG_CTRUNC is unset, but ignore any flags in + # ignoreflags. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertIsInstance(ancdata, list) + self.assertLessEqual(len(ancdata), maxcmsgs) + fds = array.array("i") + for item in ancdata: + self.assertIsInstance(item, tuple) + cmsg_level, cmsg_type, cmsg_data = item + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data) % SIZEOF_INT, 0) + fds.frombytes(cmsg_data) + + self.assertEqual(len(fds), numfds) + self.checkFDs(fds) + + def testFDPassSimple(self): + # Pass a single FD (array read from bytes object). + self.checkRecvmsgFDs(1, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testFDPassSimple(self): + self.assertEqual( + self.sendmsgToServer( + [MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(1)).tobytes())]), + len(MSG)) + + def testMultipleFDPass(self): + # Pass multiple FDs in a single array. + self.checkRecvmsgFDs(4, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testMultipleFDPass(self): + self.createAndSendFDs(4) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassCMSG_SPACE(self): + # Test using CMSG_SPACE() to calculate ancillary buffer size. + self.checkRecvmsgFDs( + 4, self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(4 * SIZEOF_INT))) + + @testFDPassCMSG_SPACE.client_skip + def _testFDPassCMSG_SPACE(self): + self.createAndSendFDs(4) + + def testFDPassCMSG_LEN(self): + # Test using CMSG_LEN() to calculate ancillary buffer size. + self.checkRecvmsgFDs(1, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(4 * SIZEOF_INT)), + # RFC 3542 says implementations may set + # MSG_CTRUNC if there isn't enough space + # for trailing padding. + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassCMSG_LEN(self): + self.createAndSendFDs(1) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparate(self): + # Pass two FDs in two separate arrays. Arrays may be combined + # into a single control message by the OS. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), 10240), + maxcmsgs=2) + + @testFDPassSeparate.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparate(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparateMinSpace(self): + # Pass two FDs in two separate arrays, receiving them into the + # minimum space for two arrays. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(SIZEOF_INT)), + maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) + + @testFDPassSeparateMinSpace.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparateMinSpace(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + def sendAncillaryIfPossible(self, msg, ancdata): + # Try to send msg and ancdata to server, but if the system + # call fails, just send msg with no ancillary data. + try: + nbytes = self.sendmsgToServer([msg], ancdata) + except OSError as e: + # Check that it was the system call that failed + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer([msg]) + self.assertEqual(nbytes, len(msg)) + + def testFDPassEmpty(self): + # Try to pass an empty FD array. Can receive either no array + # or an empty array. + self.checkRecvmsgFDs(0, self.doRecvmsg(self.serv_sock, + len(MSG), 10240), + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassEmpty(self): + self.sendAncillaryIfPossible(MSG, [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + b"")]) + + def testFDPassPartialInt(self): + # Try to pass a truncated FD array. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 1) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + def _testFDPassPartialInt(self): + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [self.badfd]).tobytes()[:-1])]) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassPartialIntInMiddle(self): + # Try to pass two FD arrays, the first of which is truncated. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 2) + fds = array.array("i") + # Arrays may have been combined in a single control message + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.assertLessEqual(len(fds), 2) + self.checkFDs(fds) + + @testFDPassPartialIntInMiddle.client_skip + def _testFDPassPartialIntInMiddle(self): + fd0, fd1 = self.newFDs(2) + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0, self.badfd]).tobytes()[:-1]), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]) + + def checkTruncatedHeader(self, result, ignoreflags=0): + # Check that no ancillary data items are returned when data is + # truncated inside the cmsghdr structure. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no buffer size + # is specified. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG)), + # BSD seems to set MSG_CTRUNC only + # if an item has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTruncNoBufSize(self): + self.createAndSendFDs(1) + + def testCmsgTrunc0(self): + # Check that no ancillary data is received when buffer size is 0. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTrunc0(self): + self.createAndSendFDs(1) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + def testCmsgTrunc1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + + def _testCmsgTrunc1(self): + self.createAndSendFDs(1) + + def testCmsgTrunc2Int(self): + # The cmsghdr structure has at least three members, two of + # which are ints, so we still shouldn't see any ancillary + # data. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + SIZEOF_INT * 2)) + + def _testCmsgTrunc2Int(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Minus1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(0) - 1)) + + def _testCmsgTruncLen0Minus1(self): + self.createAndSendFDs(1) + + # The following tests try to truncate the control message in the + # middle of the FD array. + + def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): + # Check that file descriptor data is truncated to between + # mindata and maxdata bytes when received with buffer size + # ancbuf, and that any complete file descriptor numbers are + # valid. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + if mindata == 0 and ancdata == []: + return + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertGreaterEqual(len(cmsg_data), mindata) + self.assertLessEqual(len(cmsg_data), maxdata) + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.checkFDs(fds) + + def testCmsgTruncLen0(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + + def _testCmsgTruncLen0(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Plus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + + def _testCmsgTruncLen0Plus1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), + maxdata=SIZEOF_INT) + + def _testCmsgTruncLen1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen2Minus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, + maxdata=(2 * SIZEOF_INT) - 1) + + def _testCmsgTruncLen2Minus1(self): + self.createAndSendFDs(2) + + +class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase): + # Test sendmsg() and recvmsg[_into]() using the ancillary data + # features of the RFC 3542 Advanced Sockets API for IPv6. + # Currently we can only handle certain data items (e.g. traffic + # class, hop limit, MTU discovery and fragmentation settings) + # without resorting to unportable means such as the struct module, + # but the tests here are aimed at testing the ancillary data + # handling in sendmsg() and recvmsg() rather than the IPv6 API + # itself. + + # Test value to use when setting hop limit of packet + hop_limit = 2 + + # Test value to use when setting traffic class of packet. + # -1 means "use kernel default". + traffic_class = -1 + + def ancillaryMapping(self, ancdata): + # Given ancillary data list ancdata, return a mapping from + # pairs (cmsg_level, cmsg_type) to corresponding cmsg_data. + # Check that no (level, type) pair appears more than once. + d = {} + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertNotIn((cmsg_level, cmsg_type), d) + d[(cmsg_level, cmsg_type)] = cmsg_data + return d + + def checkHopLimit(self, ancbufsize, maxhop=255, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space. Check that data is MSG, ancillary data is not + # truncated (but ignore any flags in ignoreflags), and hop + # limit is between 0 and maxhop inclusive. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + self.assertIsInstance(ancdata[0], tuple) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimit(self): + # Test receiving the packet hop limit as ancillary data. + self.checkHopLimit(ancbufsize=10240) + + @testRecvHopLimit.client_skip + def _testRecvHopLimit(self): + # Need to wait until server has asked to receive ancillary + # data, as implementations are not required to buffer it + # otherwise. + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimitCMSG_SPACE(self): + # Test receiving hop limit, using CMSG_SPACE to calculate buffer size. + self.checkHopLimit(ancbufsize=socket.CMSG_SPACE(SIZEOF_INT)) + + @testRecvHopLimitCMSG_SPACE.client_skip + def _testRecvHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Could test receiving into buffer sized using CMSG_LEN, but RFC + # 3542 says portable applications must provide space for trailing + # padding. Implementations may set MSG_CTRUNC if there isn't + # enough space for the padding. + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSetHopLimit(self): + # Test setting hop limit on outgoing packet and receiving it + # at the other end. + self.checkHopLimit(ancbufsize=10240, maxhop=self.hop_limit) + + @testSetHopLimit.client_skip + def _testSetHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + def checkTrafficClassAndHopLimit(self, ancbufsize, maxhop=255, + ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space. Check that data is MSG, ancillary + # data is not truncated (but ignore any flags in ignoreflags), + # and traffic class and hop limit are in range (hop limit no + # more than maxhop). + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + self.assertEqual(len(ancdata), 2) + ancmap = self.ancillaryMapping(ancdata) + + tcdata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_TCLASS)] + self.assertEqual(len(tcdata), SIZEOF_INT) + a = array.array("i") + a.frombytes(tcdata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + hldata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT)] + self.assertEqual(len(hldata), SIZEOF_INT) + a = array.array("i") + a.frombytes(hldata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimit(self): + # Test receiving traffic class and hop limit as ancillary data. + self.checkTrafficClassAndHopLimit(ancbufsize=10240) + + @testRecvTrafficClassAndHopLimit.client_skip + def _testRecvTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + # Test receiving traffic class and hop limit, using + # CMSG_SPACE() to calculate buffer size. + self.checkTrafficClassAndHopLimit( + ancbufsize=socket.CMSG_SPACE(SIZEOF_INT) * 2) + + @testRecvTrafficClassAndHopLimitCMSG_SPACE.client_skip + def _testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSetTrafficClassAndHopLimit(self): + # Test setting traffic class and hop limit on outgoing packet, + # and receiving them at the other end. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testSetTrafficClassAndHopLimit.client_skip + def _testSetTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testOddCmsgSize(self): + # Try to send ancillary data with first item one byte too + # long. Fall back to sending with correct size if this fails, + # and check that second item was handled correctly. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testOddCmsgSize.client_skip + def _testOddCmsgSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + try: + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class]).tobytes() + b"\x00"), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + except OSError as e: + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + self.assertEqual(nbytes, len(MSG)) + + # Tests for proper handling of truncated ancillary data + + def checkHopLimitTruncatedHeader(self, ancbufsize, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space, which should be too small to contain the ancillary + # data header (if ancbufsize is None, pass no second argument + # to recvmsg()). Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and no ancillary data is + # returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + args = () if ancbufsize is None else (ancbufsize,) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), *args) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no ancillary + # buffer size is provided. + self.checkHopLimitTruncatedHeader(ancbufsize=None, + # BSD seems to set + # MSG_CTRUNC only if an item + # has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + @testCmsgTruncNoBufSize.client_skip + def _testCmsgTruncNoBufSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc0(self): + # Check that no ancillary data is received when ancillary + # buffer size is zero. + self.checkHopLimitTruncatedHeader(ancbufsize=0, + ignoreflags=socket.MSG_CTRUNC) + + @testSingleCmsgTrunc0.client_skip + def _testSingleCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=1) + + @testSingleCmsgTrunc1.client_skip + def _testSingleCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc2Int(self): + self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) + + @testSingleCmsgTrunc2Int.client_skip + def _testSingleCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncLen0Minus1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) + + @testSingleCmsgTruncLen0Minus1.client_skip + def _testSingleCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncInData(self): + # Test truncation of a control message inside its associated + # data. The message may be returned with its data truncated, + # or not returned at all. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + self.assertLessEqual(len(ancdata), 1) + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + @testSingleCmsgTruncInData.client_skip + def _testSingleCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + def checkTruncatedSecondHeader(self, ancbufsize, ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space, which should be large enough to + # contain the first item, but too small to contain the header + # of the second. Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and only one ancillary + # data item is returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertIn(cmsg_type, {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT}) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + # Try the above test with various buffer sizes. + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc0(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT), + ignoreflags=socket.MSG_CTRUNC) + + @testSecondCmsgTrunc0.client_skip + def _testSecondCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) + + @testSecondCmsgTrunc1.client_skip + def _testSecondCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc2Int(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + 2 * SIZEOF_INT) + + @testSecondCmsgTrunc2Int.client_skip + def _testSecondCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTruncLen0Minus1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(0) - 1) + + @testSecondCmsgTruncLen0Minus1.client_skip + def _testSecondCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecomdCmsgTruncInData(self): + # Test truncation of the second of two control messages inside + # its associated data. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + cmsg_types = {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT} + + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + self.assertEqual(ancdata, []) + + @testSecomdCmsgTruncInData.client_skip + def _testSecomdCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + +# Derive concrete test classes for different socket types. + +class SendrecvmsgUDPTestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDPTest(SendmsgConnectionlessTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDPTest(RecvmsgTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDPTest(RecvmsgIntoTests, SendrecvmsgUDPTestBase): + pass + + +class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDP6TestBase): + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer, ignoring scope ID + self.assertEqual(addr1[:-1], addr2[:-1]) + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDP6Test(SendmsgConnectionlessTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDP6Test(RecvmsgTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDP6Test(RecvmsgIntoTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgRFC3542AncillaryUDP6Test(RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoRFC3542AncillaryUDP6Test(RecvmsgIntoMixin, + RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + + +class SendrecvmsgTCPTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, TCPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgTCPTest(SendmsgStreamTests, SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgTCPTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoTCPTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + + +class SendrecvmsgSCTPStreamTestBase(SendrecvmsgSCTPFlagsBase, + SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, SCTPStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgSCTPStreamTest(SendmsgStreamTests, SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCTPStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + +@requireAttrs(socket.socket, "recvmsg_into") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCTPStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgIntoSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + + +class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, UnixStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUnixStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUnixStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCMRightsStreamTest(SCMRightsTest, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg_into") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCMRightsStreamTest(RecvmsgIntoMixin, SCMRightsTest, + SendrecvmsgUnixStreamTestBase): + pass + + +# Test interrupting the interruptible send/receive methods with a +# signal when a timeout is set. These tests avoid having multiple +# threads alive during the test so that the OS cannot deliver the +# signal to the wrong one. + +class InterruptedTimeoutBase(unittest.TestCase): + # Base class for interrupted send/receive tests. Installs an + # empty handler for SIGALRM and removes it on teardown, along with + # any scheduled alarms. + + def setUp(self): + super().setUp() + orig_alrm_handler = signal.signal(signal.SIGALRM, + lambda signum, frame: 1 / 0) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(self.setAlarm, 0) + + # Timeout for socket operations + timeout = 4.0 + + # Provide setAlarm() method to schedule delivery of SIGALRM after + # given number of seconds, or cancel it if zero, and an + # appropriate time value to use. Use setitimer() if available. + if hasattr(signal, "setitimer"): + alarm_time = 0.05 + + def setAlarm(self, seconds): + signal.setitimer(signal.ITIMER_REAL, seconds) + else: + # Old systems may deliver the alarm up to one second early + alarm_time = 2 + + def setAlarm(self, seconds): + signal.alarm(seconds) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase): + # Test interrupting the recv*() methods with signals when a + # timeout is set. + + def setUp(self): + super().setUp() + self.serv.settimeout(self.timeout) + + def checkInterruptedRecv(self, func, *args, **kwargs): + # Check that func(*args, **kwargs) raises + # errno of EINTR when interrupted by a signal. + self.setAlarm(self.alarm_time) + with self.assertRaises(ZeroDivisionError) as cm: + func(*args, **kwargs) + + def testInterruptedRecvTimeout(self): + self.checkInterruptedRecv(self.serv.recv, 1024) + + def testInterruptedRecvIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recv_into, bytearray(1024)) + + def testInterruptedRecvfromTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom, 1024) + + def testInterruptedRecvfromIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom_into, bytearray(1024)) + + @requireAttrs(socket.socket, "recvmsg") + def testInterruptedRecvmsgTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg, 1024) + + @requireAttrs(socket.socket, "recvmsg_into") + def testInterruptedRecvmsgIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg_into, [bytearray(1024)]) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +@unittest.skipUnless(thread, 'Threading required for this test.') +class InterruptedSendTimeoutTest(InterruptedTimeoutBase, + ThreadSafeCleanupTestCase, + SocketListeningTestMixin, TCPTestBase): + # Test interrupting the interruptible send*() methods with signals + # when a timeout is set. + + def setUp(self): + super().setUp() + self.serv_conn = self.newSocket() + self.addCleanup(self.serv_conn.close) + # Use a thread to complete the connection, but wait for it to + # terminate before running the test, so that there is only one + # thread to accept the signal. + cli_thread = threading.Thread(target=self.doConnect) + cli_thread.start() + self.cli_conn, addr = self.serv.accept() + self.addCleanup(self.cli_conn.close) + cli_thread.join() + self.serv_conn.settimeout(self.timeout) + + def doConnect(self): + self.serv_conn.connect(self.serv_addr) + + def checkInterruptedSend(self, func, *args, **kwargs): + # Check that func(*args, **kwargs), run in a loop, raises + # OSError with an errno of EINTR when interrupted by a + # signal. + with self.assertRaises(ZeroDivisionError) as cm: + while True: + self.setAlarm(self.alarm_time) + func(*args, **kwargs) + + # Issue #12958: The following tests have problems on OS X prior to 10.7 + @support.requires_mac_ver(10, 7) + def testInterruptedSendTimeout(self): + self.checkInterruptedSend(self.serv_conn.send, b"a"*512) + + @support.requires_mac_ver(10, 7) + def testInterruptedSendtoTimeout(self): + # Passing an actual address here as Python's wrapper for + # sendto() doesn't allow passing a zero-length one; POSIX + # requires that the address is ignored since the socket is + # connection-mode, however. + self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512, + self.serv_addr) + + @support.requires_mac_ver(10, 7) + @requireAttrs(socket.socket, "sendmsg") + def testInterruptedSendmsgTimeout(self): + self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512]) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class TCPCloserTest(ThreadedTCPSocketTest): + + def testClose(self): + conn, addr = self.serv.accept() + conn.close() + + sd = self.cli + read, write, err = select.select([sd], [], [], 1.0) + self.assertEqual(read, [sd]) + self.assertEqual(sd.recv(1), b'') + + # Calling close() many times should be safe. + conn.close() + conn.close() + + def _testClose(self): + self.cli.connect((HOST, self.port)) + time.sleep(1.0) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicSocketPairTest(SocketPairTest): + + def __init__(self, methodName='runTest'): + SocketPairTest.__init__(self, methodName=methodName) + + def _check_defaults(self, sock): + self.assertIsInstance(sock, socket.socket) + if hasattr(socket, 'AF_UNIX'): + self.assertEqual(sock.family, socket.AF_UNIX) + else: + self.assertEqual(sock.family, socket.AF_INET) + self.assertEqual(sock.type, socket.SOCK_STREAM) + self.assertEqual(sock.proto, 0) + + def _testDefaults(self): + self._check_defaults(self.cli) + + def testDefaults(self): + self._check_defaults(self.serv) + + def testRecv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.cli.send(MSG) + + def testSend(self): + self.serv.send(MSG) + + def _testSend(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NonBlockingTCPTests(ThreadedTCPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def testSetBlocking(self): + # Testing whether set blocking works + self.serv.setblocking(True) + self.assertIsNone(self.serv.gettimeout()) + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error setting non-blocking mode.") + + def _testSetBlocking(self): + pass + + @support.cpython_only + def testSetBlocking_overflow(self): + # Issue 15989 + import _testcapi + if _testcapi.UINT_MAX >= _testcapi.ULONG_MAX: + self.skipTest('needs UINT_MAX < ULONG_MAX') + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.serv.setblocking(_testcapi.UINT_MAX + 1) + self.assertIsNone(self.serv.gettimeout()) + + _testSetBlocking_overflow = support.cpython_only(_testSetBlocking) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'test needs socket.SOCK_NONBLOCK') + @support.requires_linux_version(2, 6, 28) + def testInitNonBlocking(self): + # reinit server socket + self.serv.close() + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM | + socket.SOCK_NONBLOCK) + self.port = support.bind_port(self.serv) + self.serv.listen() + # actual testing + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error creating with non-blocking mode.") + + def _testInitNonBlocking(self): + pass + + def testInheritFlags(self): + # Issue #7995: when calling accept() on a listening socket with a + # timeout, the resulting socket should not be non-blocking. + self.serv.settimeout(10) + try: + conn, addr = self.serv.accept() + message = conn.recv(len(MSG)) + finally: + conn.close() + self.serv.settimeout(None) + + def _testInheritFlags(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + time.sleep(0.5) + self.cli.send(MSG) + + def testAccept(self): + # Testing non-blocking accept + self.serv.setblocking(0) + try: + conn, addr = self.serv.accept() + except OSError: + pass + else: + self.fail("Error trying to do non-blocking accept.") + read, write, err = select.select([self.serv], [], []) + if self.serv in read: + conn, addr = self.serv.accept() + self.assertIsNone(conn.gettimeout()) + conn.close() + else: + self.fail("Error trying to do accept after select.") + + def _testAccept(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + + def testConnect(self): + # Testing non-blocking connect + conn, addr = self.serv.accept() + conn.close() + + def _testConnect(self): + self.cli.settimeout(10) + self.cli.connect((HOST, self.port)) + + def testRecv(self): + # Testing non-blocking recv + conn, addr = self.serv.accept() + conn.setblocking(0) + try: + msg = conn.recv(len(MSG)) + except OSError: + pass + else: + self.fail("Error trying to do non-blocking recv.") + read, write, err = select.select([conn], [], []) + if conn in read: + msg = conn.recv(len(MSG)) + conn.close() + self.assertEqual(msg, MSG) + else: + self.fail("Error during select call to non-blocking socket.") + + def _testRecv(self): + self.cli.connect((HOST, self.port)) + time.sleep(0.1) + self.cli.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class FileObjectClassTestCase(SocketConnectedTest): + """Unit tests for the object returned by socket.makefile() + + self.read_file is the io object returned by makefile() on + the client connection. You can read from this file to + get output from the server. + + self.write_file is the io object returned by makefile() on the + server connection. You can write to this file to send output + to the client. + """ + + bufsize = -1 # Use default buffer size + encoding = 'utf-8' + errors = 'strict' + newline = None + + read_mode = 'rb' + read_msg = MSG + write_mode = 'wb' + write_msg = MSG + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def setUp(self): + self.evt1, self.evt2, self.serv_finished, self.cli_finished = [ + threading.Event() for i in range(4)] + SocketConnectedTest.setUp(self) + self.read_file = self.cli_conn.makefile( + self.read_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def tearDown(self): + self.serv_finished.set() + self.read_file.close() + self.assertTrue(self.read_file.closed) + self.read_file = None + SocketConnectedTest.tearDown(self) + + def clientSetUp(self): + SocketConnectedTest.clientSetUp(self) + self.write_file = self.serv_conn.makefile( + self.write_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def clientTearDown(self): + self.cli_finished.set() + self.write_file.close() + self.assertTrue(self.write_file.closed) + self.write_file = None + SocketConnectedTest.clientTearDown(self) + + def testReadAfterTimeout(self): + # Issue #7322: A file object must disallow further reads + # after a timeout has occurred. + self.cli_conn.settimeout(1) + self.read_file.read(3) + # First read raises a timeout + self.assertRaises(socket.timeout, self.read_file.read, 1) + # Second read is disallowed + with self.assertRaises(OSError) as ctx: + self.read_file.read(1) + self.assertIn("cannot read from timed out object", str(ctx.exception)) + + def _testReadAfterTimeout(self): + self.write_file.write(self.write_msg[0:3]) + self.write_file.flush() + self.serv_finished.wait() + + def testSmallRead(self): + # Performing small file read test + first_seg = self.read_file.read(len(self.read_msg)-3) + second_seg = self.read_file.read(3) + msg = first_seg + second_seg + self.assertEqual(msg, self.read_msg) + + def _testSmallRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testFullRead(self): + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testFullRead(self): + self.write_file.write(self.write_msg) + self.write_file.close() + + def testUnbufferedRead(self): + # Performing unbuffered file read test + buf = type(self.read_msg)() + while 1: + char = self.read_file.read(1) + if not char: + break + buf += char + self.assertEqual(buf, self.read_msg) + + def _testUnbufferedRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testReadline(self): + # Performing file readline test + line = self.read_file.readline() + self.assertEqual(line, self.read_msg) + + def _testReadline(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testCloseAfterMakefile(self): + # The file returned by makefile should keep the socket open. + self.cli_conn.close() + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testCloseAfterMakefile(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileAfterMakefileClose(self): + self.read_file.close() + msg = self.cli_conn.recv(len(MSG)) + if isinstance(self.read_msg, str): + msg = msg.decode() + self.assertEqual(msg, self.read_msg) + + def _testMakefileAfterMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testClosedAttr(self): + self.assertTrue(not self.read_file.closed) + + def _testClosedAttr(self): + self.assertTrue(not self.write_file.closed) + + def testAttributes(self): + self.assertEqual(self.read_file.mode, self.read_mode) + self.assertEqual(self.read_file.name, self.cli_conn.fileno()) + + def _testAttributes(self): + self.assertEqual(self.write_file.mode, self.write_mode) + self.assertEqual(self.write_file.name, self.serv_conn.fileno()) + + def testRealClose(self): + self.read_file.close() + self.assertRaises(ValueError, self.read_file.fileno) + self.cli_conn.close() + self.assertRaises(OSError, self.cli_conn.getsockname) + + def _testRealClose(self): + pass + + +class UnbufferedFileObjectClassTestCase(FileObjectClassTestCase): + + """Repeat the tests from FileObjectClassTestCase with bufsize==0. + + In this case (and in this case only), it should be possible to + create a file object, read a line from it, create another file + object, read another line from it, without loss of data in the + first file object's buffer. Note that http.client relies on this + when reading multiple requests from the same socket.""" + + bufsize = 0 # Use unbuffered mode + + def testUnbufferedReadline(self): + # Read a line, create a new file object, read another line with it + line = self.read_file.readline() # first line + self.assertEqual(line, b"A. " + self.write_msg) # first line + self.read_file = self.cli_conn.makefile('rb', 0) + line = self.read_file.readline() # second line + self.assertEqual(line, b"B. " + self.write_msg) # second line + + def _testUnbufferedReadline(self): + self.write_file.write(b"A. " + self.write_msg) + self.write_file.write(b"B. " + self.write_msg) + self.write_file.flush() + + def testMakefileClose(self): + # The file returned by makefile should keep the socket open... + self.cli_conn.close() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, self.read_msg) + # ...until the file is itself closed + self.read_file.close() + self.assertRaises(OSError, self.cli_conn.recv, 1024) + + def _testMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileCloseSocketDestroy(self): + if hasattr(sys, "getrefcount"): + refcount_before = sys.getrefcount(self.cli_conn) + self.read_file.close() + refcount_after = sys.getrefcount(self.cli_conn) + self.assertEqual(refcount_before - 1, refcount_after) + + def _testMakefileCloseSocketDestroy(self): + pass + + # Non-blocking ops + # NOTE: to set `read_file` as non-blocking, we must call + # `cli_conn.setblocking` and vice-versa (see setUp / clientSetUp). + + def testSmallReadNonBlocking(self): + self.cli_conn.setblocking(False) + self.assertEqual(self.read_file.readinto(bytearray(10)), None) + self.assertEqual(self.read_file.read(len(self.read_msg) - 3), None) + self.evt1.set() + self.evt2.wait(1.0) + first_seg = self.read_file.read(len(self.read_msg) - 3) + if first_seg is None: + # Data not arrived (can happen under Windows), wait a bit + time.sleep(0.5) + first_seg = self.read_file.read(len(self.read_msg) - 3) + buf = bytearray(10) + n = self.read_file.readinto(buf) + self.assertEqual(n, 3) + msg = first_seg + buf[:n] + self.assertEqual(msg, self.read_msg) + self.assertEqual(self.read_file.readinto(bytearray(16)), None) + self.assertEqual(self.read_file.read(1), None) + + def _testSmallReadNonBlocking(self): + self.evt1.wait(1.0) + self.write_file.write(self.write_msg) + self.write_file.flush() + self.evt2.set() + # Avoid cloding the socket before the server test has finished, + # otherwise system recv() will return 0 instead of EWOULDBLOCK. + self.serv_finished.wait(5.0) + + def testWriteNonBlocking(self): + self.cli_finished.wait(5.0) + # The client thread can't skip directly - the SkipTest exception + # would appear as a failure. + if self.serv_skipped: + self.skipTest(self.serv_skipped) + + def _testWriteNonBlocking(self): + self.serv_skipped = None + self.serv_conn.setblocking(False) + # Try to saturate the socket buffer pipe with repeated large writes. + BIG = b"x" * support.SOCK_MAX_SIZE + LIMIT = 10 + # The first write() succeeds since a chunk of data can be buffered + n = self.write_file.write(BIG) + self.assertGreater(n, 0) + for i in range(LIMIT): + n = self.write_file.write(BIG) + if n is None: + # Succeeded + break + self.assertGreater(n, 0) + else: + # Let us know that this test didn't manage to establish + # the expected conditions. This is not a failure in itself but, + # if it happens repeatedly, the test should be fixed. + self.serv_skipped = "failed to saturate the socket buffer" + + +class LineBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 1 # Default-buffered for reading; line-buffered for writing + + +class SmallBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 2 # Exercise the buffering code + + +class UnicodeReadFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'wb' + write_msg = MSG + newline = '' + + +class UnicodeWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'rb' + read_msg = MSG + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class UnicodeReadWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class NetworkConnectionTest(object): + """Prove network connection.""" + + def clientSetUp(self): + # We're inherited below by BasicTCPTest2, which also inherits + # BasicTCPTest, which defines self.port referenced below. + self.cli = socket.create_connection((HOST, self.port)) + self.serv_conn = self.cli + +class BasicTCPTest2(NetworkConnectionTest, BasicTCPTest): + """Tests that NetworkConnection does not break existing TCP functionality. + """ + +class NetworkConnectionNoServer(unittest.TestCase): + + class MockSocket(socket.socket): + def connect(self, *args): + raise socket.timeout('timed out') + + @contextlib.contextmanager + def mocked_socket_module(self): + """Return a socket which times out on connect""" + old_socket = socket.socket + socket.socket = self.MockSocket + try: + yield + finally: + socket.socket = old_socket + + def test_connect(self): + port = support.find_unused_port() + cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(cli.close) + with self.assertRaises(OSError) as cm: + cli.connect((HOST, port)) + self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + + def test_create_connection(self): + # Issue #9792: errors raised by create_connection() should have + # a proper errno attribute. + port = support.find_unused_port() + with self.assertRaises(OSError) as cm: + socket.create_connection((HOST, port)) + + # Issue #16257: create_connection() calls getaddrinfo() against + # 'localhost'. This may result in an IPV6 addr being returned + # as well as an IPV4 one: + # >>> socket.getaddrinfo('localhost', port, 0, SOCK_STREAM) + # >>> [(2, 2, 0, '', ('127.0.0.1', 41230)), + # (26, 2, 0, '', ('::1', 41230, 0, 0))] + # + # create_connection() enumerates through all the addresses returned + # and if it doesn't successfully bind to any of them, it propagates + # the last exception it encountered. + # + # On Solaris, ENETUNREACH is returned in this circumstance instead + # of ECONNREFUSED. So, if that errno exists, add it to our list of + # expected errnos. + expected_errnos = [ errno.ECONNREFUSED, ] + if hasattr(errno, 'ENETUNREACH'): + expected_errnos.append(errno.ENETUNREACH) + + self.assertIn(cm.exception.errno, expected_errnos) + + def test_create_connection_timeout(self): + # Issue #9792: create_connection() should not recast timeout errors + # as generic socket errors. + with self.mocked_socket_module(): + with self.assertRaises(socket.timeout): + socket.create_connection((HOST, 1234)) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.source_port = support.find_unused_port() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def _justAccept(self): + conn, addr = self.serv.accept() + conn.close() + + testFamily = _justAccept + def _testFamily(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.family, 2) + + testSourceAddress = _justAccept + def _testSourceAddress(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30, + source_address=('', self.source_port)) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.getsockname()[1], self.source_port) + # The port number being used is sufficient to show that the bind() + # call happened. + + testTimeoutDefault = _justAccept + def _testTimeoutDefault(self): + # passing no explicit timeout uses socket's global default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(42) + try: + self.cli = socket.create_connection((HOST, self.port)) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), 42) + + testTimeoutNone = _justAccept + def _testTimeoutNone(self): + # None timeout means the same as sock.settimeout(None) + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + self.cli = socket.create_connection((HOST, self.port), timeout=None) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), None) + + testTimeoutValueNamed = _justAccept + def _testTimeoutValueNamed(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.assertEqual(self.cli.gettimeout(), 30) + + testTimeoutValueNonamed = _justAccept + def _testTimeoutValueNonamed(self): + self.cli = socket.create_connection((HOST, self.port), 30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.gettimeout(), 30) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionBehaviourTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def testInsideTimeout(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + time.sleep(3) + conn.send(b"done!") + testOutsideTimeout = testInsideTimeout + + def _testInsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port)) + data = sock.recv(5) + self.assertEqual(data, b"done!") + + def _testOutsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port), timeout=1) + self.assertRaises(socket.timeout, lambda: sock.recv(5)) + + +class TCPTimeoutTest(SocketTCPTest): + + def testTCPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.accept() + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (TCP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of error (TCP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (TCP)") + if not ok: + self.fail("accept() returned success when we did not expect it") + + @unittest.skipUnless(hasattr(signal, 'alarm'), + 'test needs signal.alarm()') + def testInterruptedTimeout(self): + # XXX I don't know how to do this test on MSWindows or any other + # plaform that doesn't support signal.alarm() or os.kill(), though + # the bug should have existed on all platforms. + self.serv.settimeout(5.0) # must be longer than alarm + class Alarm(Exception): + pass + def alarm_handler(signal, frame): + raise Alarm + old_alarm = signal.signal(signal.SIGALRM, alarm_handler) + try: + signal.alarm(2) # POSIX allows alarm to be up to 1 second early + try: + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of Alarm") + except Alarm: + pass + except: + self.fail("caught other exception instead of Alarm:" + " %s(%s):\n%s" % + (sys.exc_info()[:2] + (traceback.format_exc(),))) + else: + self.fail("nothing caught") + finally: + signal.alarm(0) # shut off alarm + except Alarm: + self.fail("got Alarm in wrong place") + finally: + # no alarm can be pending. Safe to restore old handler. + signal.signal(signal.SIGALRM, old_alarm) + +class UDPTimeoutTest(SocketUDPTest): + + def testUDPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.recv(1024) + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (UDP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.recv(1024) + except socket.timeout: + self.fail("caught timeout instead of error (UDP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (UDP)") + if not ok: + self.fail("recv() returned success when we did not expect it") + +class TestExceptions(unittest.TestCase): + + def testExceptionTree(self): + self.assertTrue(issubclass(OSError, Exception)) + self.assertTrue(issubclass(socket.herror, OSError)) + self.assertTrue(issubclass(socket.gaierror, OSError)) + self.assertTrue(issubclass(socket.timeout, OSError)) + +@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +class TestLinuxAbstractNamespace(unittest.TestCase): + + UNIX_PATH_MAX = 108 + + def testLinuxAbstractNamespace(self): + address = b"\x00python-test-hello\x00\xff" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s1: + s1.bind(address) + s1.listen() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s2: + s2.connect(s1.getsockname()) + with s1.accept()[0] as s3: + self.assertEqual(s1.getsockname(), address) + self.assertEqual(s2.getpeername(), address) + + def testMaxName(self): + address = b"\x00" + b"h" * (self.UNIX_PATH_MAX - 1) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(address) + self.assertEqual(s.getsockname(), address) + + def testNameOverflow(self): + address = "\x00" + "h" * self.UNIX_PATH_MAX + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + self.assertRaises(OSError, s.bind, address) + + def testStrName(self): + # Check that an abstract name can be passed as a string. + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s.bind("\x00python\x00test\x00") + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + finally: + s.close() + + def testBytearrayName(self): + # Check that an abstract name can be passed as a bytearray. + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(bytearray(b"\x00python\x00test\x00")) + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + +@unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'test needs socket.AF_UNIX') +class TestUnixDomain(unittest.TestCase): + + def setUp(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + def tearDown(self): + self.sock.close() + + def encoded(self, path): + # Return the given path encoded in the file system encoding, + # or skip the test if this is not possible. + try: + return os.fsencode(path) + except UnicodeEncodeError: + self.skipTest( + "Pathname {0!a} cannot be represented in file " + "system encoding {1!r}".format( + path, sys.getfilesystemencoding())) + + def bind(self, sock, path): + # Bind the socket + try: + sock.bind(path) + except OSError as e: + if str(e) == "AF_UNIX path too long": + self.skipTest( + "Pathname {0!a} is too long to serve as an AF_UNIX path" + .format(path)) + else: + raise + + def testStrAddr(self): + # Test binding to and retrieving a normal string pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testBytesAddr(self): + # Test binding to a bytes pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, self.encoded(path)) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testSurrogateescapeBind(self): + # Test binding to a valid non-ASCII pathname, with the + # non-ASCII bytes supplied using surrogateescape encoding. + path = os.path.abspath(support.TESTFN_UNICODE) + b = self.encoded(path) + self.bind(self.sock, b.decode("ascii", "surrogateescape")) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testUnencodableAddr(self): + # Test binding to a pathname that cannot be encoded in the + # file system encoding. + if support.TESTFN_UNENCODABLE is None: + self.skipTest("No unencodable filename available") + path = os.path.abspath(support.TESTFN_UNENCODABLE) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BufferIOTest(SocketConnectedTest): + """ + Test the buffer versions of socket.recv() and socket.send(). + """ + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecvIntoArray(self): + buf = array.array("B", [0] * len(MSG)) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + buf = buf.tobytes() + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvIntoBytearray(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoBytearray = _testRecvIntoArray + + def testRecvIntoMemoryview(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoMemoryview = _testRecvIntoArray + + def testRecvFromIntoArray(self): + buf = array.array("B", [0] * len(MSG)) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + buf = buf.tobytes() + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvFromIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoBytearray(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoBytearray = _testRecvFromIntoArray + + def testRecvFromIntoMemoryview(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoMemoryview = _testRecvFromIntoArray + + def testRecvFromIntoSmallBuffer(self): + # See issue #20246. + buf = bytearray(8) + self.assertRaises(ValueError, self.cli_conn.recvfrom_into, buf, 1024) + + def _testRecvFromIntoSmallBuffer(self): + self.serv_conn.send(MSG) + + def testRecvFromIntoEmptyBuffer(self): + buf = bytearray() + self.cli_conn.recvfrom_into(buf) + self.cli_conn.recvfrom_into(buf, 0) + + _testRecvFromIntoEmptyBuffer = _testRecvFromIntoArray + + +TIPC_STYPE = 2000 +TIPC_LOWER = 200 +TIPC_UPPER = 210 + +def isTipcAvailable(): + """Check if the TIPC module is loaded + + The TIPC module is not loaded automatically on Ubuntu and probably + other Linux distros. + """ + if not hasattr(socket, "AF_TIPC"): + return False + try: + f = open("/proc/modules") + except IOError as e: + # It's ok if the file does not exist, is a directory or if we + # have not the permission to read it. In any other case it's a + # real error, so raise it again. + if e.errno in (errno.ENOENT, errno.EISDIR, errno.EACCES): + return False + else: + raise + with f: + for line in f: + if line.startswith("tipc "): + return True + return False + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCTest(unittest.TestCase): + def testRDM(self): + srv = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + cli = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + self.addCleanup(srv.close) + self.addCleanup(cli.close) + + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + srv.bind(srvaddr) + + sendaddr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + cli.sendto(MSG, sendaddr) + + msg, recvaddr = srv.recvfrom(1024) + + self.assertEqual(cli.getsockname(), recvaddr) + self.assertEqual(msg, MSG) + + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCThreadableTest(unittest.TestCase, ThreadableTest): + def __init__(self, methodName = 'runTest'): + unittest.TestCase.__init__(self, methodName = methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.srv = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.srv.close) + self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + self.srv.bind(srvaddr) + self.srv.listen() + self.serverExplicitReady() + self.conn, self.connaddr = self.srv.accept() + self.addCleanup(self.conn.close) + + def clientSetUp(self): + # There is a hittable race between serverExplicitReady() and the + # accept() call; sleep a little while to avoid it, otherwise + # we could get an exception + time.sleep(0.1) + self.cli = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.cli.close) + addr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + self.cli.connect(addr) + self.cliaddr = self.cli.getsockname() + + def testStream(self): + msg = self.conn.recv(1024) + self.assertEqual(msg, MSG) + self.assertEqual(self.cliaddr, self.connaddr) + + def _testStream(self): + self.cli.send(MSG) + self.cli.close() + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class ContextManagersTest(ThreadedTCPSocketTest): + + def _testSocketClass(self): + # base test + with socket.socket() as sock: + self.assertFalse(sock._closed) + self.assertTrue(sock._closed) + # close inside with block + with socket.socket() as sock: + sock.close() + self.assertTrue(sock._closed) + # exception inside with block + with socket.socket() as sock: + self.assertRaises(OSError, sock.sendall, b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionBase(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionBase(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + self.assertFalse(sock._closed) + sock.sendall(b'foo') + self.assertEqual(sock.recv(1024), b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionClose(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionClose(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + sock.close() + self.assertTrue(sock._closed) + self.assertRaises(OSError, sock.sendall, b'foo') + + +class InheritanceTest(unittest.TestCase): + @unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), + "SOCK_CLOEXEC not defined") + @support.requires_linux_version(2, 6, 28) + def test_SOCK_CLOEXEC(self): + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: + self.assertTrue(s.type & socket.SOCK_CLOEXEC) + self.assertFalse(s.get_inheritable()) + + def test_default_inheritable(self): + sock = socket.socket() + with sock: + self.assertEqual(sock.get_inheritable(), False) + + def test_dup(self): + sock = socket.socket() + with sock: + newsock = sock.dup() + sock.close() + with newsock: + self.assertEqual(newsock.get_inheritable(), False) + + def test_set_inheritable(self): + sock = socket.socket() + with sock: + sock.set_inheritable(True) + self.assertEqual(sock.get_inheritable(), True) + + sock.set_inheritable(False) + self.assertEqual(sock.get_inheritable(), False) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + + @unittest.skipUnless(hasattr(socket, "socketpair"), + "need socket.socketpair()") + def test_socketpair(self): + s1, s2 = socket.socketpair() + self.addCleanup(s1.close) + self.addCleanup(s2.close) + self.assertEqual(s1.get_inheritable(), False) + self.assertEqual(s2.get_inheritable(), False) + + +@unittest.skipUnless(hasattr(socket, "SOCK_NONBLOCK"), + "SOCK_NONBLOCK not defined") +class NonblockConstantTest(unittest.TestCase): + def checkNonblock(self, s, nonblock=True, timeout=0.0): + if nonblock: + self.assertTrue(s.type & socket.SOCK_NONBLOCK) + self.assertEqual(s.gettimeout(), timeout) + else: + self.assertFalse(s.type & socket.SOCK_NONBLOCK) + self.assertEqual(s.gettimeout(), None) + + @support.requires_linux_version(2, 6, 28) + def test_SOCK_NONBLOCK(self): + # a lot of it seems silly and redundant, but I wanted to test that + # changing back and forth worked ok + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) as s: + self.checkNonblock(s) + s.setblocking(1) + self.checkNonblock(s, False) + s.setblocking(0) + self.checkNonblock(s) + s.settimeout(None) + self.checkNonblock(s, False) + s.settimeout(2.0) + self.checkNonblock(s, timeout=2.0) + s.setblocking(1) + self.checkNonblock(s, False) + # defaulttimeout + t = socket.getdefaulttimeout() + socket.setdefaulttimeout(0.0) + with socket.socket() as s: + self.checkNonblock(s) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(2.0) + with socket.socket() as s: + self.checkNonblock(s, timeout=2.0) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(t) + + +@unittest.skipUnless(os.name == "nt", "Windows specific") +@unittest.skipUnless(multiprocessing, "need multiprocessing") +class TestSocketSharing(SocketTCPTest): + # This must be classmethod and not staticmethod or multiprocessing + # won't be able to bootstrap it. + @classmethod + def remoteProcessServer(cls, q): + # Recreate socket from shared data + sdata = q.get() + message = q.get() + + s = socket.fromshare(sdata) + s2, c = s.accept() + + # Send the message + s2.sendall(message) + s2.close() + s.close() + + def testShare(self): + # Transfer the listening server socket to another process + # and service it from there. + + # Create process: + q = multiprocessing.Queue() + p = multiprocessing.Process(target=self.remoteProcessServer, args=(q,)) + p.start() + + # Get the shared socket data + data = self.serv.share(p.pid) + + # Pass the shared socket to the other process + addr = self.serv.getsockname() + self.serv.close() + q.put(data) + + # The data that the server will send us + message = b"slapmahfro" + q.put(message) + + # Connect + s = socket.create_connection(addr) + # listen for the data + m = [] + while True: + data = s.recv(100) + if not data: + break + m.append(data) + s.close() + received = b"".join(m) + self.assertEqual(received, message) + p.join() + + def testShareLength(self): + data = self.serv.share(os.getpid()) + self.assertRaises(ValueError, socket.fromshare, data[:-1]) + self.assertRaises(ValueError, socket.fromshare, data+b"foo") + + def compareSockets(self, org, other): + # socket sharing is expected to work only for blocking socket + # since the internal python timeout value isn't transferred. + self.assertEqual(org.gettimeout(), None) + self.assertEqual(org.gettimeout(), other.gettimeout()) + + self.assertEqual(org.family, other.family) + self.assertEqual(org.type, other.type) + # If the user specified "0" for proto, then + # internally windows will have picked the correct value. + # Python introspection on the socket however will still return + # 0. For the shared socket, the python value is recreated + # from the actual value, so it may not compare correctly. + if org.proto != 0: + self.assertEqual(org.proto, other.proto) + + def testShareLocal(self): + data = self.serv.share(os.getpid()) + s = socket.fromshare(data) + try: + self.compareSockets(self.serv, s) + finally: + s.close() + + def testTypes(self): + families = [socket.AF_INET, socket.AF_INET6] + types = [socket.SOCK_STREAM, socket.SOCK_DGRAM] + for f in families: + for t in types: + try: + source = socket.socket(f, t) + except OSError: + continue # This combination is not supported + try: + data = source.share(os.getpid()) + shared = socket.fromshare(data) + try: + self.compareSockets(source, shared) + finally: + shared.close() + finally: + source.close() + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendfileUsingSendTest(ThreadedTCPSocketTest): + """ + Test the send() implementation of socket.sendfile(). + """ + + FILESIZE = (10 * 1024 * 1024) # 10MB + BUFSIZE = 8192 + FILEDATA = b"" + TIMEOUT = 2 + + @classmethod + def setUpClass(cls): + def chunks(total, step): + assert total >= step + while total > step: + yield step + total -= step + if total: + yield total + + chunk = b"".join([random.choice(string.ascii_letters).encode() + for i in range(cls.BUFSIZE)]) + with open(support.TESTFN, 'wb') as f: + for csize in chunks(cls.FILESIZE, cls.BUFSIZE): + f.write(chunk) + with open(support.TESTFN, 'rb') as f: + cls.FILEDATA = f.read() + assert len(cls.FILEDATA) == cls.FILESIZE + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + + def accept_conn(self): + self.serv.settimeout(self.TIMEOUT) + conn, addr = self.serv.accept() + conn.settimeout(self.TIMEOUT) + self.addCleanup(conn.close) + return conn + + def recv_data(self, conn): + received = [] + while True: + chunk = conn.recv(self.BUFSIZE) + if not chunk: + break + received.append(chunk) + return b''.join(received) + + def meth_from_sock(self, sock): + # Depending on the mixin class being run return either send() + # or sendfile() method implementation. + return getattr(sock, "_sendfile_use_send") + + # regular file + + def _testRegularFile(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, self.FILESIZE) + self.assertEqual(file.tell(), self.FILESIZE) + + def testRegularFile(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # non regular file + + def _testNonRegularFile(self): + address = self.serv.getsockname() + file = io.BytesIO(self.FILEDATA) + with socket.create_connection(address) as sock, file as file: + sent = sock.sendfile(file) + self.assertEqual(sent, self.FILESIZE) + self.assertEqual(file.tell(), self.FILESIZE) + self.assertRaises(socket._GiveupOnSendfile, + sock._sendfile_use_sendfile, file) + + def testNonRegularFile(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # empty file + + def _testEmptyFileSend(self): + address = self.serv.getsockname() + filename = support.TESTFN + "2" + with open(filename, 'wb'): + self.addCleanup(support.unlink, filename) + file = open(filename, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, 0) + self.assertEqual(file.tell(), 0) + + def testEmptyFileSend(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(data, b"") + + # offset + + def _testOffset(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file, offset=5000) + self.assertEqual(sent, self.FILESIZE - 5000) + self.assertEqual(file.tell(), self.FILESIZE) + + def testOffset(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE - 5000) + self.assertEqual(data, self.FILEDATA[5000:]) + + # count + + def _testCount(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 5000007 + meth = self.meth_from_sock(sock) + sent = meth(file, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count) + + def testCount(self): + count = 5000007 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[:count]) + + # count small + + def _testCountSmall(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 1 + meth = self.meth_from_sock(sock) + sent = meth(file, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count) + + def testCountSmall(self): + count = 1 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[:count]) + + # count + offset + + def _testCountWithOffset(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 100007 + meth = self.meth_from_sock(sock) + sent = meth(file, offset=2007, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count + 2007) + + def testCountWithOffset(self): + count = 100007 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[2007:count+2007]) + + # non blocking sockets are not supposed to work + + def _testNonBlocking(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + sock.setblocking(False) + meth = self.meth_from_sock(sock) + self.assertRaises(ValueError, meth, file) + self.assertRaises(ValueError, sock.sendfile, file) + + def testNonBlocking(self): + conn = self.accept_conn() + if conn.recv(8192): + self.fail('was not supposed to receive any data') + + # timeout (non-triggered) + + def _testWithTimeout(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, self.FILESIZE) + + def testWithTimeout(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # timeout (triggered) + + def _testWithTimeoutTriggeredSend(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=0.01) as sock, \ + file as file: + meth = self.meth_from_sock(sock) + self.assertRaises(socket.timeout, meth, file) + + def testWithTimeoutTriggeredSend(self): + conn = self.accept_conn() + conn.recv(88192) + + # errors + + def _test_errors(self): + pass + + def test_errors(self): + with open(support.TESTFN, 'rb') as file: + with socket.socket(type=socket.SOCK_DGRAM) as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex( + ValueError, "SOCK_STREAM", meth, file) + with open(support.TESTFN, 'rt') as file: + with socket.socket() as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex( + ValueError, "binary mode", meth, file) + with open(support.TESTFN, 'rb') as file: + with socket.socket() as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex(TypeError, "positive integer", + meth, file, count='2') + self.assertRaisesRegex(TypeError, "positive integer", + meth, file, count=0.1) + self.assertRaisesRegex(ValueError, "positive integer", + meth, file, count=0) + self.assertRaisesRegex(ValueError, "positive integer", + meth, file, count=-1) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +@unittest.skipUnless(hasattr(os, "sendfile"), + 'os.sendfile() required for this test.') +@unittest.skipUnless(hasattr(os, 'gevent_uses_sendfile'), + 'gevent sockets do not support this') +class SendfileUsingSendfileTest(SendfileUsingSendTest): + """ + Test the sendfile() implementation of socket.sendfile(). + """ + def meth_from_sock(self, sock): + return getattr(sock, "_sendfile_use_sendfile") + + +def test_main(): + tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest, + TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ] + + tests.extend([ + NonBlockingTCPTests, + FileObjectClassTestCase, + UnbufferedFileObjectClassTestCase, + LineBufferedFileObjectClassTestCase, + SmallBufferedFileObjectClassTestCase, + UnicodeReadFileObjectClassTestCase, + UnicodeWriteFileObjectClassTestCase, + UnicodeReadWriteFileObjectClassTestCase, + NetworkConnectionNoServer, + NetworkConnectionAttributesTest, + NetworkConnectionBehaviourTest, + ContextManagersTest, + InheritanceTest, + NonblockConstantTest + ]) + tests.append(BasicSocketPairTest) + tests.append(TestUnixDomain) + tests.append(TestLinuxAbstractNamespace) + tests.extend([TIPCTest, TIPCThreadableTest]) + tests.extend([BasicCANTest, CANTest]) + tests.extend([BasicRDSTest, RDSTest]) + tests.extend([ + CmsgMacroTests, + SendmsgUDPTest, + RecvmsgUDPTest, + RecvmsgIntoUDPTest, + SendmsgUDP6Test, + RecvmsgUDP6Test, + RecvmsgRFC3542AncillaryUDP6Test, + RecvmsgIntoRFC3542AncillaryUDP6Test, + RecvmsgIntoUDP6Test, + SendmsgTCPTest, + RecvmsgTCPTest, + RecvmsgIntoTCPTest, + SendmsgSCTPStreamTest, + RecvmsgSCTPStreamTest, + RecvmsgIntoSCTPStreamTest, + SendmsgUnixStreamTest, + RecvmsgUnixStreamTest, + RecvmsgIntoUnixStreamTest, + RecvmsgSCMRightsStreamTest, + RecvmsgIntoSCMRightsStreamTest, + # These are slow when setitimer() is not available + InterruptedRecvTimeoutTest, + InterruptedSendTimeoutTest, + TestSocketSharing, + SendfileUsingSendTest, + SendfileUsingSendfileTest, + ]) + + thread_info = support.threading_setup() + support.run_unittest(*tests) + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.5pypy/test_socketserver.py b/src/greentest/3.5pypy/test_socketserver.py new file mode 100644 index 0000000..0d0f86f --- /dev/null +++ b/src/greentest/3.5pypy/test_socketserver.py @@ -0,0 +1,318 @@ +""" +Test suite for socketserver. +""" + +import contextlib +import os +import select +import signal +import socket +import select +import errno +import tempfile +import unittest +import socketserver + +import test.support +from test.support import reap_children, reap_threads, verbose +try: + import threading +except ImportError: + threading = None + +test.support.requires("network") + +TEST_STR = b"hello world\n" +HOST = test.support.HOST + +HAVE_UNIX_SOCKETS = hasattr(socket, "AF_UNIX") +requires_unix_sockets = unittest.skipUnless(HAVE_UNIX_SOCKETS, + 'requires Unix sockets') +HAVE_FORKING = hasattr(os, "fork") +requires_forking = unittest.skipUnless(HAVE_FORKING, 'requires forking') + +def signal_alarm(n): + """Call signal.alarm when it exists (i.e. not on Windows).""" + if hasattr(signal, 'alarm'): + signal.alarm(n) + +# Remember real select() to avoid interferences with mocking +_real_select = select.select + +def receive(sock, n, timeout=20): + r, w, x = _real_select([sock], [], [], timeout) + if sock in r: + return sock.recv(n) + else: + raise RuntimeError("timed out on %r" % (sock,)) + +if HAVE_UNIX_SOCKETS: + class ForkingUnixStreamServer(socketserver.ForkingMixIn, + socketserver.UnixStreamServer): + pass + + class ForkingUnixDatagramServer(socketserver.ForkingMixIn, + socketserver.UnixDatagramServer): + pass + + +@contextlib.contextmanager +def simple_subprocess(testcase): + pid = os.fork() + if pid == 0: + # Don't raise an exception; it would be caught by the test harness. + os._exit(72) + yield None + pid2, status = os.waitpid(pid, 0) + testcase.assertEqual(pid2, pid) + testcase.assertEqual(72 << 8, status) + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class SocketServerTest(unittest.TestCase): + """Test all socket servers.""" + + def setUp(self): + signal_alarm(60) # Kill deadlocks after 60 seconds. + self.port_seed = 0 + self.test_files = [] + + def tearDown(self): + signal_alarm(0) # Didn't deadlock. + reap_children() + + for fn in self.test_files: + try: + os.remove(fn) + except OSError: + pass + self.test_files[:] = [] + + def pickaddr(self, proto): + if proto == socket.AF_INET: + return (HOST, 0) + else: + # XXX: We need a way to tell AF_UNIX to pick its own name + # like AF_INET provides port==0. + dir = None + fn = tempfile.mktemp(prefix='unix_socket.', dir=dir) + self.test_files.append(fn) + return fn + + def make_server(self, addr, svrcls, hdlrbase): + class MyServer(svrcls): + def handle_error(self, request, client_address): + self.close_request(request) + self.server_close() + raise + + class MyHandler(hdlrbase): + def handle(self): + line = self.rfile.readline() + self.wfile.write(line) + + if verbose: print("creating server") + server = MyServer(addr, MyHandler) + self.assertEqual(server.server_address, server.socket.getsockname()) + return server + + @reap_threads + def run_server(self, svrcls, hdlrbase, testfunc): + server = self.make_server(self.pickaddr(svrcls.address_family), + svrcls, hdlrbase) + # We had the OS pick a port, so pull the real address out of + # the server. + addr = server.server_address + if verbose: + print("ADDR =", addr) + print("CLASS =", svrcls) + + t = threading.Thread( + name='%s serving' % svrcls, + target=server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval':0.01}) + t.daemon = True # In case this function raises. + t.start() + if verbose: print("server running") + for i in range(3): + if verbose: print("test client", i) + testfunc(svrcls.address_family, addr) + if verbose: print("waiting for server") + server.shutdown() + t.join() + server.server_close() + self.assertEqual(-1, server.socket.fileno()) + if verbose: print("done") + + def stream_examine(self, proto, addr): + s = socket.socket(proto, socket.SOCK_STREAM) + s.connect(addr) + s.sendall(TEST_STR) + buf = data = receive(s, 100) + while data and b'\n' not in buf: + data = receive(s, 100) + buf += data + self.assertEqual(buf, TEST_STR) + s.close() + + def dgram_examine(self, proto, addr): + s = socket.socket(proto, socket.SOCK_DGRAM) + if HAVE_UNIX_SOCKETS and proto == socket.AF_UNIX: + s.bind(self.pickaddr(proto)) + s.sendto(TEST_STR, addr) + buf = data = receive(s, 100) + while data and b'\n' not in buf: + data = receive(s, 100) + buf += data + self.assertEqual(buf, TEST_STR) + s.close() + + def test_TCPServer(self): + self.run_server(socketserver.TCPServer, + socketserver.StreamRequestHandler, + self.stream_examine) + + def test_ThreadingTCPServer(self): + self.run_server(socketserver.ThreadingTCPServer, + socketserver.StreamRequestHandler, + self.stream_examine) + + @requires_forking + def test_ForkingTCPServer(self): + with simple_subprocess(self): + self.run_server(socketserver.ForkingTCPServer, + socketserver.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + def test_UnixStreamServer(self): + self.run_server(socketserver.UnixStreamServer, + socketserver.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + def test_ThreadingUnixStreamServer(self): + self.run_server(socketserver.ThreadingUnixStreamServer, + socketserver.StreamRequestHandler, + self.stream_examine) + + @requires_unix_sockets + @requires_forking + def test_ForkingUnixStreamServer(self): + with simple_subprocess(self): + self.run_server(ForkingUnixStreamServer, + socketserver.StreamRequestHandler, + self.stream_examine) + + def test_UDPServer(self): + self.run_server(socketserver.UDPServer, + socketserver.DatagramRequestHandler, + self.dgram_examine) + + def test_ThreadingUDPServer(self): + self.run_server(socketserver.ThreadingUDPServer, + socketserver.DatagramRequestHandler, + self.dgram_examine) + + @requires_forking + def test_ForkingUDPServer(self): + with simple_subprocess(self): + self.run_server(socketserver.ForkingUDPServer, + socketserver.DatagramRequestHandler, + self.dgram_examine) + + @requires_unix_sockets + def test_UnixDatagramServer(self): + self.run_server(socketserver.UnixDatagramServer, + socketserver.DatagramRequestHandler, + self.dgram_examine) + + @requires_unix_sockets + def test_ThreadingUnixDatagramServer(self): + self.run_server(socketserver.ThreadingUnixDatagramServer, + socketserver.DatagramRequestHandler, + self.dgram_examine) + + @requires_unix_sockets + @requires_forking + def test_ForkingUnixDatagramServer(self): + self.run_server(ForkingUnixDatagramServer, + socketserver.DatagramRequestHandler, + self.dgram_examine) + + @reap_threads + def test_shutdown(self): + # Issue #2302: shutdown() should always succeed in making an + # other thread leave serve_forever(). + class MyServer(socketserver.TCPServer): + pass + + class MyHandler(socketserver.StreamRequestHandler): + pass + + threads = [] + for i in range(20): + s = MyServer((HOST, 0), MyHandler) + t = threading.Thread( + name='MyServer serving', + target=s.serve_forever, + kwargs={'poll_interval':0.01}) + t.daemon = True # In case this function raises. + threads.append((t, s)) + for t, s in threads: + t.start() + s.shutdown() + for t, s in threads: + t.join() + s.server_close() + + def test_tcpserver_bind_leak(self): + # Issue #22435: the server socket wouldn't be closed if bind()/listen() + # failed. + # Create many servers for which bind() will fail, to see if this result + # in FD exhaustion. + for i in range(1024): + with self.assertRaises(OverflowError): + socketserver.TCPServer((HOST, -1), + socketserver.StreamRequestHandler) + + +class MiscTestCase(unittest.TestCase): + + def test_all(self): + # objects defined in the module should be in __all__ + expected = [] + for name in dir(socketserver): + if not name.startswith('_'): + mod_object = getattr(socketserver, name) + if getattr(mod_object, '__module__', None) == 'socketserver': + expected.append(name) + self.assertCountEqual(socketserver.__all__, expected) + + def test_shutdown_request_called_if_verify_request_false(self): + # Issue #26309: BaseServer should call shutdown_request even if + # verify_request is False + + class MyServer(socketserver.TCPServer): + def verify_request(self, request, client_address): + return False + + shutdown_called = 0 + def shutdown_request(self, request): + self.shutdown_called += 1 + socketserver.TCPServer.shutdown_request(self, request) + + server = MyServer((HOST, 0), socketserver.StreamRequestHandler) + s = socket.socket(server.address_family, socket.SOCK_STREAM) + s.connect(server.server_address) + s.close() + server.handle_request() + self.assertEqual(server.shutdown_called, 1) + server.server_close() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_ssl.py b/src/greentest/3.5pypy/test_ssl.py new file mode 100644 index 0000000..df4ed18 --- /dev/null +++ b/src/greentest/3.5pypy/test_ssl.py @@ -0,0 +1,3454 @@ +# Test the support for SSL and sockets + +import sys +import unittest +from test import support +import socket +import select +import time +import datetime +import gc +import os +import errno +import pprint +import tempfile +import urllib.request +import traceback +import asyncore +import weakref +import platform +import functools + +ssl = support.import_module("ssl") + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = support.HOST +IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL') +IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0) + + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/make_ssl_certs.py. +# Other certificates are simply fetched from the Internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = os.fsencode(CERTFILE) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = os.fsencode(ONLYCERT) +BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = os.fsencode(CAPATH) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") + + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNING_CA = data_file("pycacert.pem") +# cert with all kinds of subject alt names +ALLSANFILE = data_file("allsans.pem") + +REMOTE_HOST = "self-signed.pythontest.net" +REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +NONEXISTINGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") + +DHFILE = data_file("dh1024.pem") +BYTES_DHFILE = os.fsencode(DHFILE) + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + +def can_clear_options(): + # 0.9.8m or higher + return ssl._OPENSSL_API_VERSION >= (0, 9, 8, 13, 15) + +def no_sslv2_implies_sslv3_hello(): + # 0.9.7h or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) + +def have_verify_flags(): + # 0.9.8 or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 8, 0, 15) + +def utc_offset(): #NOTE: ignore issues like #1647654 + # local time = utc time + utc offset + if time.daylight and time.localtime().tm_isdst > 0: + return -time.altzone # seconds + return -time.timezone + +def asn1time(cert_time): + # Some versions of OpenSSL ignore seconds, see #18207 + # 0.9.8.i + if ssl._OPENSSL_API_VERSION == (0, 9, 8, 9, 15): + fmt = "%b %d %H:%M:%S %Y GMT" + dt = datetime.datetime.strptime(cert_time, fmt) + dt = dt.replace(second=0) + cert_time = dt.strftime(fmt) + # %d adds leading zero but ASN1_TIME_print() uses leading space + if cert_time[4] == "0": + cert_time = cert_time[:4] + " " + cert_time[5:] + + return cert_time + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): + if hasattr(ssl, 'PROTOCOL_SSLv2'): + @functools.wraps(func) + def f(*args, **kwargs): + try: + ssl.SSLContext(ssl.PROTOCOL_SSLv2) + except ssl.SSLError: + if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and + platform.linux_distribution() == ('debian', 'squeeze/sid', '')): + raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") + return func(*args, **kwargs) + return f + else: + return func + +needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + if ssl.HAS_ECDH: + ssl.OP_SINGLE_ECDH_USE + if ssl.OPENSSL_VERSION_INFO >= (1, 0): + ssl.OP_NO_COMPRESSION + self.assertIn(ssl.HAS_SNI, {True, False}) + self.assertIn(ssl.HAS_ECDH, {True, False}) + + def test_str_for_enums(self): + # Make sure that the PROTOCOL_* constants have enum-like string + # reprs. + proto = ssl.PROTOCOL_TLS + self.assertEqual(str(proto), '_SSLMethod.PROTOCOL_TLS') + ctx = ssl.SSLContext(proto) + self.assertIs(ctx.protocol, proto) + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + + data, is_cryptographic = ssl.RAND_pseudo_bytes(16) + self.assertEqual(len(data), 16) + self.assertEqual(is_cryptographic, v == 1) + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + + # negative num is invalid + self.assertRaises(ValueError, ssl.RAND_bytes, -5) + self.assertRaises(ValueError, ssl.RAND_pseudo_bytes, -5) + + if hasattr(ssl, 'RAND_egd'): + self.assertRaises(TypeError, ssl.RAND_egd, 1) + self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1) + ssl.RAND_add("this is a random string", 75.0) + ssl.RAND_add(b"this is a random bytes object", 75.0) + ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) + + @unittest.skipUnless(os.name == 'posix', 'requires posix') + def test_random_fork(self): + status = ssl.RAND_status() + if not status: + self.fail("OpenSSL's PRNG has insufficient randomness") + + rfd, wfd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(rfd) + child_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(child_random), 16) + os.write(wfd, child_random) + os.close(wfd) + except BaseException: + os._exit(1) + else: + os._exit(0) + else: + os.close(wfd) + self.addCleanup(os.close, rfd) + _, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + child_random = os.read(rfd, 16) + self.assertEqual(len(child_random), 16) + parent_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(parent_random), 16) + + self.assertNotEqual(child_random, parent_random) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + p = ssl._ssl._test_decode_cert(CERTFILE) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['issuer'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + # Note the next three asserts will fail if the keys are regenerated + self.assertEqual(p['notAfter'], asn1time('Oct 5 23:01:56 2020 GMT')) + self.assertEqual(p['notBefore'], asn1time('Oct 8 23:01:56 2010 GMT')) + self.assertEqual(p['serialNumber'], 'D7C7381919AFC24E') + self.assertEqual(p['subject'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + + def test_parse_all_sans(self): + p = ssl._ssl._test_decode_cert(ALLSANFILE) + self.assertEqual(p['subjectAltName'], + ( + ('DNS', 'allsans'), + ('othername', ''), + ('othername', ''), + ('email', 'user@example.org'), + ('DNS', 'www.example.org'), + ('DirName', + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'dirname example'),))), + ('URI', 'https://www.python.org/'), + ('IP Address', '127.0.0.1'), + ('IP Address', '0:0:0:0:0:0:0:1\n'), + ('Registered ID', '1.2.3.4.5') + ) + ) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, int) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 0.9 + self.assertGreaterEqual(n, 0x900000) + # < 3.0 + self.assertLess(n, 0x30000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 0) + self.assertLess(major, 3) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + # Version string as returned by {Open,Libre}SSL, the format might change + if IS_LIBRESSL: + self.assertTrue(s.startswith("LibreSSL {:d}".format(major)), + (s, t, hex(n))) + else: + self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)), + (s, t, hex(n))) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + wr = weakref.ref(ss) + with support.check_warnings(("", ResourceWarning)): + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # OSError raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s) as ss: + self.assertRaises(OSError, ss.recv, 1) + self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) + self.assertRaises(OSError, ss.recvfrom, 1) + self.assertRaises(OSError, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(OSError, ss.send, b'x') + self.assertRaises(OSError, ss.sendto, b'x', ('0.0.0.0', 0)) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with ssl.wrap_socket(s) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + def test_errors(self): + sock = socket.socket() + self.assertRaisesRegex(ValueError, + "certfile must be specified", + ssl.wrap_socket, sock, keyfile=CERTFILE) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True, certfile="") + with ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) as s: + self.assertRaisesRegex(ValueError, "can't connect in server-side mode", + s.connect, (HOST, 8080)) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, + certfile=CERTFILE, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, + certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def bad_cert_test(self, certfile): + """Check that trying to use the given client certificate fails""" + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + certfile) + sock = socket.socket() + self.addCleanup(sock.close) + with self.assertRaises(ssl.SSLError): + ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) + + def test_empty_cert(self): + """Wrapping with an empty cert file""" + self.bad_cert_test("nullcert.pem") + + def test_malformed_cert(self): + """Wrapping with a badly formatted certificate (syntax error)""" + self.bad_cert_test("badcert.pem") + + def test_malformed_key(self): + """Wrapping with a badly formatted key (syntax error)""" + self.bad_cert_test("badkey.pem") + + def test_match_hostname(self): + def ok(cert, hostname): + ssl.match_hostname(cert, hostname) + def fail(cert, hostname): + self.assertRaises(ssl.CertificateError, + ssl.match_hostname, cert, hostname) + + # -- Hostname matching -- + + cert = {'subject': ((('commonName', 'example.com'),),)} + ok(cert, 'example.com') + ok(cert, 'ExAmple.cOm') + fail(cert, 'www.example.com') + fail(cert, '.example.com') + fail(cert, 'example.org') + fail(cert, 'exampleXcom') + + cert = {'subject': ((('commonName', '*.a.com'),),)} + ok(cert, 'foo.a.com') + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + # only match one left-most wildcard + cert = {'subject': ((('commonName', 'f*.com'),),)} + ok(cert, 'foo.com') + ok(cert, 'f.com') + fail(cert, 'bar.com') + fail(cert, 'foo.a.com') + fail(cert, 'bar.foo.com') + + # NULL bytes are bad, CVE-2013-4073 + cert = {'subject': ((('commonName', + 'null.python.org\x00example.org'),),)} + ok(cert, 'null.python.org\x00example.org') # or raise an error? + fail(cert, 'example.org') + fail(cert, 'null.python.org') + + # error cases with wildcards + cert = {'subject': ((('commonName', '*.*.a.com'),),)} + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + cert = {'subject': ((('commonName', 'a.*.com'),),)} + fail(cert, 'a.foo.com') + fail(cert, 'a..com') + fail(cert, 'a.com') + + # wildcard doesn't match IDNA prefix 'xn--' + idna = 'püthon.python.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, idna) + cert = {'subject': ((('commonName', 'x*.python.org'),),)} + fail(cert, idna) + cert = {'subject': ((('commonName', 'xn--p*.python.org'),),)} + fail(cert, idna) + + # wildcard in first fragment and IDNA A-labels in sequent fragments + # are supported. + idna = 'www*.pythön.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, 'www.pythön.org'.encode("idna").decode("ascii")) + ok(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'ftp.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'pythön.org'.encode("idna").decode("ascii")) + + # Slightly fake real-world example + cert = {'notAfter': 'Jun 26 21:41:46 2011 GMT', + 'subject': ((('commonName', 'linuxfrz.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('DNS', 'linuxfr.com'), + ('othername', ''))} + ok(cert, 'linuxfr.org') + ok(cert, 'linuxfr.com') + # Not a "DNS" entry + fail(cert, '') + # When there is a subjectAltName, commonName isn't used + fail(cert, 'linuxfrz.org') + + # A pristine real-world example + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),), + (('commonName', 'mail.google.com'),))} + ok(cert, 'mail.google.com') + fail(cert, 'gmail.com') + # Only commonName is considered + fail(cert, 'California') + + # -- IPv4 matching -- + cert = {'subject': ((('commonName', 'example.com'),),), + 'subjectAltName': (('DNS', 'example.com'), + ('IP Address', '10.11.12.13'), + ('IP Address', '14.15.16.17'))} + ok(cert, '10.11.12.13') + ok(cert, '14.15.16.17') + fail(cert, '14.15.16.18') + fail(cert, 'example.net') + + # -- IPv6 matching -- + cert = {'subject': ((('commonName', 'example.com'),),), + 'subjectAltName': (('DNS', 'example.com'), + ('IP Address', '2001:0:0:0:0:0:0:CAFE\n'), + ('IP Address', '2003:0:0:0:0:0:0:BABA\n'))} + ok(cert, '2001::cafe') + ok(cert, '2003::baba') + fail(cert, '2003::bebe') + fail(cert, 'example.net') + + # -- Miscellaneous -- + + # Neither commonName nor subjectAltName + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),))} + fail(cert, 'mail.google.com') + + # No DNS entry in subjectAltName but a commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('commonName', 'mail.google.com'),)), + 'subjectAltName': (('othername', 'blabla'), )} + ok(cert, 'mail.google.com') + + # No DNS entry subjectAltName and no commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),)), + 'subjectAltName': (('othername', 'blabla'),)} + fail(cert, 'google.com') + + # Empty cert / no cert + self.assertRaises(ValueError, ssl.match_hostname, None, 'example.com') + self.assertRaises(ValueError, ssl.match_hostname, {}, 'example.com') + + # Issue #17980: avoid denials of service by refusing more than one + # wildcard per fragment. + cert = {'subject': ((('commonName', 'a*b.com'),),)} + ok(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b.co*'),),)} + fail(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b*.com'),),)} + with self.assertRaises(ssl.CertificateError) as cm: + ssl.match_hostname(cert, 'axxbxxc.com') + self.assertIn("too many wildcards", str(cm.exception)) + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with socket.socket() as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + s.bind(('127.0.0.1', 0)) + s.listen() + c = socket.socket(socket.AF_INET) + c.connect(s.getsockname()) + with ssl.wrap_socket(c, do_handshake_on_connect=False) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + s.close() + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with ssl.wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + def test_dealloc_warn(self): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + r = repr(ss) + with self.assertWarns(ResourceWarning) as cm: + ss = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (set, bool)) + if isinstance(trust, set): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegex(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegex(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + def cert_time_ok(self, timestring, timestamp): + self.assertEqual(ssl.cert_time_to_seconds(timestring), timestamp) + + def cert_time_fail(self, timestring): + with self.assertRaises(ValueError): + ssl.cert_time_to_seconds(timestring) + + @unittest.skipUnless(utc_offset(), + 'local time needs to be different from UTC') + def test_cert_time_to_seconds_timezone(self): + # Issue #19940: ssl.cert_time_to_seconds() returns wrong + # results if local timezone is not UTC + self.cert_time_ok("May 9 00:00:00 2007 GMT", 1178668800.0) + self.cert_time_ok("Jan 5 09:34:43 2018 GMT", 1515144883.0) + + def test_cert_time_to_seconds(self): + timestring = "Jan 5 09:34:43 2018 GMT" + ts = 1515144883.0 + self.cert_time_ok(timestring, ts) + # accept keyword parameter, assert its name + self.assertEqual(ssl.cert_time_to_seconds(cert_time=timestring), ts) + # accept both %e and %d (space or zero generated by strftime) + self.cert_time_ok("Jan 05 09:34:43 2018 GMT", ts) + # case-insensitive + self.cert_time_ok("JaN 5 09:34:43 2018 GmT", ts) + self.cert_time_fail("Jan 5 09:34 2018 GMT") # no seconds + self.cert_time_fail("Jan 5 09:34:43 2018") # no GMT + self.cert_time_fail("Jan 5 09:34:43 2018 UTC") # not GMT timezone + self.cert_time_fail("Jan 35 09:34:43 2018 GMT") # invalid day + self.cert_time_fail("Jon 5 09:34:43 2018 GMT") # invalid month + self.cert_time_fail("Jan 5 24:00:00 2018 GMT") # invalid hour + self.cert_time_fail("Jan 5 09:60:43 2018 GMT") # invalid minute + + newyear_ts = 1230768000.0 + # leap seconds + self.cert_time_ok("Dec 31 23:59:60 2008 GMT", newyear_ts) + # same timestamp + self.cert_time_ok("Jan 1 00:00:00 2009 GMT", newyear_ts) + + self.cert_time_ok("Jan 5 09:34:59 2018 GMT", 1515144899) + # allow 60th second (even if it is not a leap second) + self.cert_time_ok("Jan 5 09:34:60 2018 GMT", 1515144900) + # allow 2nd leap second for compatibility with time.strptime() + self.cert_time_ok("Jan 5 09:34:61 2018 GMT", 1515144901) + self.cert_time_fail("Jan 5 09:34:62 2018 GMT") # invalid seconds + + # no special treatement for the special value: + # 99991231235959Z (rfc 5280) + self.cert_time_ok("Dec 31 23:59:59 9999 GMT", 253402300799.0) + + @support.run_with_locale('LC_ALL', '') + def test_cert_time_to_seconds_locale(self): + # `cert_time_to_seconds()` should be locale independent + + def local_february_name(): + return time.strftime('%b', (1, 2, 3, 4, 5, 6, 0, 0, 0)) + + if local_february_name().lower() == 'feb': + self.skipTest("locale-specific month name needs to be " + "different from C locale") + + # locale-independent + self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) + self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + + +class ContextTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_constructor(self): + for protocol in PROTOCOLS: + ssl.SSLContext(protocol) + ctx = ssl.SSLContext() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + @skip_if_broken_ubuntu_ssl + def test_protocol(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.protocol, proto) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @skip_if_broken_ubuntu_ssl + def test_options(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + if not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0): + default |= ssl.OP_NO_COMPRESSION + self.assertEqual(default, ctx.options) + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + if can_clear_options(): + ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) + self.assertEqual(default, ctx.options) + ctx.options = 0 + # Ubuntu has OP_NO_SSLv3 forced on by default + self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + else: + with self.assertRaises(ValueError): + ctx.options = 0 + + def test_verify_mode(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read() + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read() + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegex(ssl.SSLError, "no start line"): + ctx.load_verify_locations(cadata="broken") + with self.assertRaisesRegex(ssl.SSLError, "not enough data"): + ctx.load_verify_locations(cadata=b"broken") + + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_dh_params(DHFILE) + if os.name != 'nt': + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(FileNotFoundError) as cm: + ctx.load_dh_params(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + @skip_if_broken_ubuntu_ssl + def test_session_stats(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + @needs_sni + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @needs_sni + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': asn1time('Mar 29 12:29:49 2033 GMT'), + 'notBefore': asn1time('Mar 30 12:29:49 2003 GMT'), + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + with open(SIGNING_CA) as f: + cadata = f.read() + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + self.assertEqual( + ctx.options & getattr(ssl, "OP_NO_COMPRESSION", 0), + getattr(ssl, "OP_NO_COMPRESSION", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_DH_USE", 0), + getattr(ssl, "OP_SINGLE_DH_USE", 0), + ) + self.assertEqual( + ctx.options & getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + getattr(ssl, "OP_SINGLE_ECDH_USE", 0), + ) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + + def test_check_hostname(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertFalse(ctx.check_hostname) + + # Requires CERT_REQUIRED or CERT_OPTIONAL + with self.assertRaises(ValueError): + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + self.assertEqual(cm.exception.library, 'PEM') + self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) + self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen() + c = socket.socket() + c.connect(s.getsockname()) + c.setblocking(False) + with ctx.wrap_socket(c, False, do_handshake_on_connect=False) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertTrue(s.startswith("The operation did not complete (read)"), s) + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + +class MemoryBIOTests(unittest.TestCase): + + def test_read_write(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + self.assertEqual(bio.read(), b'') + bio.write(b'foo') + bio.write(b'bar') + self.assertEqual(bio.read(), b'foobar') + self.assertEqual(bio.read(), b'') + bio.write(b'baz') + self.assertEqual(bio.read(2), b'ba') + self.assertEqual(bio.read(1), b'z') + self.assertEqual(bio.read(1), b'') + + def test_eof(self): + bio = ssl.MemoryBIO() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertFalse(bio.eof) + bio.write(b'foo') + self.assertFalse(bio.eof) + bio.write_eof() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(2), b'fo') + self.assertFalse(bio.eof) + self.assertEqual(bio.read(1), b'o') + self.assertTrue(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertTrue(bio.eof) + + def test_pending(self): + bio = ssl.MemoryBIO() + self.assertEqual(bio.pending, 0) + bio.write(b'foo') + self.assertEqual(bio.pending, 3) + for i in range(3): + bio.read(1) + self.assertEqual(bio.pending, 3-i-1) + for i in range(3): + bio.write(b'x') + self.assertEqual(bio.pending, i+1) + bio.read() + self.assertEqual(bio.pending, 0) + + def test_buffer_types(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + bio.write(bytearray(b'bar')) + self.assertEqual(bio.read(), b'bar') + bio.write(memoryview(b'baz')) + self.assertEqual(bio.read(), b'baz') + + def test_error_types(self): + bio = ssl.MemoryBIO() + self.assertRaises(TypeError, bio.write, 'foo') + self.assertRaises(TypeError, bio.write, None) + self.assertRaises(TypeError, bio.write, True) + self.assertRaises(TypeError, bio.write, 1) + + +class NetworkedTests(unittest.TestCase): + + def test_connect(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) + try: + s.connect((REMOTE_HOST, 443)) + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + + # this should fail because we have no verification certs + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + + # this should succeed because we specify the root cert + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + s.connect((REMOTE_HOST, 443)) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.setblocking(False) + rc = s.connect_ex((REMOTE_HOST, 443)) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + finally: + s.close() + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT, + do_handshake_on_connect=False) + try: + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + s.close() + + def test_connect_ex_error(self): + with support.transient_internet(REMOTE_HOST): + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=REMOTE_ROOT_CERT) + try: + rc = s.connect_ex((REMOTE_HOST, 444)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + finally: + s.close() + + def test_connect_with_context(self): + with support.transient_internet(REMOTE_HOST): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + self.assertEqual({}, s.getpeercert()) + finally: + s.close() + # Same with a server hostname + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=REMOTE_HOST) + s.connect((REMOTE_HOST, 443)) + s.close() + # This should fail because we have no verification certs + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, (REMOTE_HOST, 443)) + s.close() + # This should succeed because we specify the root cert + ctx.load_verify_locations(REMOTE_ROOT_CERT) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + + def test_connect_cadata(self): + with open(REMOTE_ROOT_CERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect((REMOTE_HOST, 443)) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + with support.transient_internet(REMOTE_HOST): + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss.connect((REMOTE_HOST, 443)) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + with support.transient_internet(REMOTE_HOST): + s = socket.socket(socket.AF_INET) + s.connect((REMOTE_HOST, 443)) + s.setblocking(False) + s = ssl.wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + s.close() + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + def _test_get_server_certificate(host, port, cert=None): + with support.transient_internet(host): + pem = ssl.get_server_certificate((host, port)) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + + try: + pem = ssl.get_server_certificate((host, port), + ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + pem = ssl.get_server_certificate((host, port), + ca_certs=cert) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + + _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) + if support.IPV6_ENABLED: + _test_get_server_certificate('ipv6.google.com', 443) + + def test_ciphers(self): + remote = (REMOTE_HOST, 443) + with support.transient_internet(remote[0]): + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(remote) + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(remote) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(remote) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + # sha256.tbs-internet.com needs SNI to use the correct certificate + if not ssl.HAS_SNI: + self.skipTest("SNI needed for this test") + # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with support.transient_internet("sha256.tbs-internet.com"): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(sha256_cert) + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="sha256.tbs-internet.com") + try: + s.connect(remote) + if support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + with support.transient_internet(REMOTE_HOST): + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + s.connect((REMOTE_HOST, 443)) + try: + cert = s.getpeercert() + self.assertTrue(cert) + finally: + s.close() + self.assertEqual(len(ctx.get_ca_certs()), 1) + + @needs_sni + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + with support.transient_internet(REMOTE_HOST): + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s) as ss: + ss.connect((REMOTE_HOST, 443)) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + + +class NetworkedBIOTests(unittest.TestCase): + + def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): + # A simple IO loop. Call func(*args) depending on the error we get + # (WANT_READ or WANT_WRITE) move data between the socket and the BIOs. + timeout = kwargs.get('timeout', 10) + count = 0 + while True: + errno = None + count += 1 + try: + ret = func(*args) + except ssl.SSLError as e: + if e.errno not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + raise + errno = e.errno + # Get any data from the outgoing BIO irrespective of any error, and + # send it to the socket. + buf = outgoing.read() + sock.sendall(buf) + # If there's no error, we're done. For WANT_READ, we need to get + # data from the socket and put it in the incoming BIO. + if errno is None: + break + elif errno == ssl.SSL_ERROR_WANT_READ: + buf = sock.recv(32768) + if buf: + incoming.write(buf) + else: + incoming.write_eof() + if support.verbose: + sys.stdout.write("Needed %d calls to complete %s().\n" + % (count, func.__name__)) + return ret + + def test_handshake(self): + # NOTE: this test has been modified, CPython in newer versions + # removed the ability to get the shared ciphers of the session, but + # they always return the cipher of the ssl context. This test is fully + # removed in later versions + with support.transient_internet(REMOTE_HOST): + sock = socket.socket(socket.AF_INET) + sock.connect((REMOTE_HOST, 443)) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(REMOTE_ROOT_CERT) + ctx.check_hostname = True + sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST) + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # self-signed.pythontest.net probably shuts down the TCP + # connection without sending a secure shutdown message, and + # this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + sock.close() + + def test_read_write_data(self): + with support.transient_internet(REMOTE_HOST): + sock = socket.socket(socket.AF_INET) + sock.connect((REMOTE_HOST, 443)) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'GET / HTTP/1.0\r\n\r\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf[:5], b'HTTP/') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + sock.close() + + +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + + from test.ssl_servers import make_https_server + + class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(1) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) + self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) + except (ssl.SSLError, ConnectionResetError) as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + self.server.conn_errors.append(e) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.server.stop() + self.close() + return False + else: + self.server.shared_ciphers.append(self.sslconn.shared_ciphers()) + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + sys.stdout.write(" cert binary is " + str(len(cert_binary)) + " bytes\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + sys.stdout.write(" server: selected protocol is now " + + str(self.sslconn.selected_npn_protocol()) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except OSError: + if self.server.chatty: + handle_error("Test server failure:\n") + self.close() + self.running = False + # normally, we'd just stop here, but for the test + # harness, we want to stop the server + self.server.stop() + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + npn_protocols=None, alpn_protocols=None, + ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLSv1) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if npn_protocols: + self.context.set_npn_protocols(npn_protocols) + if alpn_protocols: + self.context.set_alpn_protocols(alpn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = support.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_npn_protocols = [] + self.selected_alpn_protocols = [] + self.shared_ciphers = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.sock.settimeout(0.05) + self.sock.listen() + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except socket.timeout: + pass + except KeyboardInterrupt: + self.stop() + self.sock.close() + + def stop(self): + self.active = False + + class AsyncoreEchoServer(threading.Thread): + + # this one's based on asyncore.dispatcher + + class EchoServer (asyncore.dispatcher): + + class ConnectionHandler (asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = ssl.wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accepted(self, sock_obj, addr): + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + + def start (self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + + def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=sni_name) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_alpn_protocol': s.selected_alpn_protocol(), + 'client_npn_protocol': s.selected_npn_protocol(), + 'version': s.version(), + }) + s.close() + stats['server_alpn_protocols'] = server.selected_alpn_protocols + stats['server_npn_protocols'] = server.selected_npn_protocols + stats['server_shared_ciphers'] = server.shared_ciphers + return stats + + def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_SSLv23: + client_context.set_ciphers("ALL") + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(CERTFILE) + ctx.load_verify_locations(CERTFILE) + try: + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except OSError as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) + + + class ThreadedTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + for protocol in PROTOCOLS: + with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): + context = ssl.SSLContext(protocol) + context.load_cert_chain(CERTFILE) + server_params_test(context, context, + chatty=True, connectionchatty=True) + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + s = context.wrap_socket(socket.socket(), + do_handshake_on_connect=False) + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + s.close() + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(SIGNING_CA) + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + with self.assertRaisesRegex(ssl.SSLError, + "certificate verify failed"): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="localhost") as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: + with self.assertRaisesRegex(ssl.CertificateError, + "hostname 'invalid' doesn't match 'localhost'"): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with socket.socket() as s: + with self.assertRaisesRegex(ValueError, + "check_hostname requires server_hostname"): + context.wrap_socket(s) + + def test_wrong_cert(self): + """Connecting when the server rejects the client's certificate + + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with a wrong client certificate fails. + """ + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + "wrongcert.pem") + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_REQUIRED, + cacerts=CERTFILE, chatty=False, + connectionchatty=False) + with server, \ + socket.socket() as sock, \ + ssl.wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) as s: + try: + # Expect either an SSL error about the server rejecting + # the connection, or a low-level connection reset (which + # sometimes happens on Windows) + s.connect((HOST, server.port)) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = support.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen() + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with socket.socket() as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = ssl.wrap_socket(c) + except OSError: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") + def test_protocol_sslv2(self): + """Connecting to an SSLv2 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) + # SSLv23 client with specific SSL options + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv2) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv23(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) + except OSError as x: + # this fails on some older versions of OpenSSL (0.9.7l, for instance) + if support.verbose: + sys.stdout.write( + " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" + % str(x)) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + + # Server with specific SSL options + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), + "OpenSSL is compiled without SSLv3 support") + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, + False, client_options=ssl.OP_NO_SSLv2) + + @skip_if_broken_ubuntu_ssl + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), + "TLS version 1.1 not supported.") + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), + "TLS version 1.2 not supported.") + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(1) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = ssl.wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using socketserver to create and manage SSL connections.""" + server = make_https_server(self, certfile=CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + with open(CERTFILE, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = 'https://localhost:%d/%s' % ( + server.port, os.path.split(CERTFILE)[1]) + context = ssl.create_default_context(cafile=CERTFILE) + f = urllib.request.urlopen(url, context=context) + try: + dlen = f.info().get("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = ssl.wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, whether to expect success, *args) + send_methods = [ + ('send', s.send, True, []), + ('sendto', s.sendto, False, ["some.address"]), + ('sendall', s.sendall, True, []), + ] + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = "PREFIX_" + + for meth_name, send_meth, expect_success, args in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + send_meth(indata, *args) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # read(-1, buffer) is supported, even though read(-1) is not + data = b"data" + s.send(data) + buffer = bytearray(len(data)) + self.assertEqual(s.read(-1, buffer), len(data)) + self.assertEqual(buffer, data) + + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, bytearray(100)) + + s.write(b"over\n") + + self.assertRaises(ValueError, s.recv, -1) + self.assertRaises(ValueError, s.read, -1) + + s.close() + + def test_recv_zero(self): + server = ThreadedEchoServer(CERTFILE) + server.__enter__() + self.addCleanup(server.__exit__, None, None) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = ssl.wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + # recv/read(0) should return no data + s.send(b"data") + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.read(0), b"") + self.assertEqual(s.read(), b"data") + + # Should not block if the other end sends no data + s.setblocking(False) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.recv_into(bytearray()), 0) + + def test_nonblocking_send(self): + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + s.setblocking(False) + + # If we keep sending data, at some point the buffers + # will be full and the call will block + buf = bytearray(8192) + def fill_buffer(): + while True: + s.send(buf) + self.assertRaises((ssl.SSLWantWriteError, + ssl.SSLWantReadError), fill_buffer) + + # Now read all the output and discard it + s.setblocking(True) + s.close() + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen() + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + ssl.wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = ssl.wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + server = context.wrap_socket(server, server_side=True) + + evt = threading.Event() + remote = None + peer = None + def serve(): + nonlocal remote, peer + server.listen() + # Block on the accept and wait on the connection to close. + evt.set() + remote, peer = server.accept() + remote.recv(1) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = context.wrap_socket(socket.socket()) + client.connect((host, port)) + client_addr = client.getsockname() + client.close() + t.join() + remote.close() + server.close() + # Sanity checks. + self.assertIsInstance(remote, ssl.SSLSocket) + self.assertEqual(peer, client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_default_ciphers(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + try: + # Force a set of weak ciphers on our client context + context.set_ciphers("DES") + except ssl.SSLError: + self.skipTest("no DES cipher available") + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_SSLv23, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) + self.assertIn("no shared cipher", str(server.conn_errors[0])) + + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + self.assertIs(s.version(), None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1') + self.assertIs(s.version(), None) + + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain(CERTFILE) + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + if ssl.OPENSSL_VERSION_INFO < (1, 0, 0): + context.set_ciphers("ECCdraft:ECDH") + with ThreadedEchoServer(context=context) as server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got channel binding data: {0!r}\n" + .format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + s.close() + + # now, again + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got another channel binding data: {0!r}\n" + .format(new_cb_data)) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + s.close() + + def test_compression(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['compression'], None) + + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.load_dh_params(DHFILE) + context.set_ciphers("kEDH") + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: + self.fail("Non-DH cipher: " + cipher[0]) + + def test_selected_alpn_protocol(self): + # selected_alpn_protocol() is None unless ALPN is used. + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support required") + def test_selected_alpn_protocol_if_server_uses_alpn(self): + # selected_alpn_protocol() is None unless ALPN is used by the client. + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_verify_locations(CERTFILE) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(['foo', 'bar']) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support needed for this test") + def test_alpn_protocols(self): + server_protocols = ['foo', 'bar', 'milkshake'] + protocol_tests = [ + (['foo', 'bar'], 'foo'), + (['bar', 'foo'], 'foo'), + (['milkshake'], 'milkshake'), + (['http/3.0', 'http/4.0'], None) + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + client_context.load_cert_chain(CERTFILE) + client_context.set_alpn_protocols(client_protocols) + + try: + stats = server_params_test(client_context, + server_context, + chatty=True, + connectionchatty=True) + except ssl.SSLError as e: + stats = e + + if expected is None and IS_OPENSSL_1_1: + # OpenSSL 1.1.0 raises handshake error + self.assertIsInstance(stats, ssl.SSLError) + else: + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_alpn_protocol'] + self.assertEqual(client_result, expected, + msg % (client_result, "client")) + server_result = stats['server_alpn_protocols'][-1] \ + if len(stats['server_alpn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, + msg % (server_result, "server")) + + def test_selected_npn_protocol(self): + # selected_npn_protocol() is None unless NPN is used + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_npn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test") + def test_npn_protocols(self): + server_protocols = ['http/1.1', 'spdy/2'] + protocol_tests = [ + (['http/1.1', 'spdy/2'], 'http/1.1'), + (['spdy/2', 'http/1.1'], 'http/1.1'), + (['spdy/2', 'test'], 'spdy/2'), + (['abc', 'def'], 'abc') + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_npn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_cert_chain(CERTFILE) + client_context.set_npn_protocols(client_protocols) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_npn_protocol'] + self.assertEqual(client_result, expected, msg % (client_result, "client")) + server_result = stats['server_npn_protocols'][-1] \ + if len(stats['server_npn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, msg % (server_result, "server")) + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + @needs_sni + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, 'localhost') + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, 'localhost') + self.assertEqual(calls, []) + + @needs_sni + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @needs_sni + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1/0 + server_context.set_servername_callback(cb_raising) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertIn("ZeroDivisionError", stderr.getvalue()) + + @needs_sni + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertIn("TypeError", stderr.getvalue()) + + def test_shared_ciphers(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): + client_context.set_ciphers("AES128:AES256") + server_context.set_ciphers("AES256") + alg1 = "AES256" + alg2 = "AES-256" + else: + client_context.set_ciphers("AES:3DES") + server_context.set_ciphers("3DES") + alg1 = "3DES" + alg2 = "DES-CBC3" + + stats = server_params_test(client_context, server_context) + ciphers = stats['server_shared_ciphers'][0] + self.assertGreater(len(ciphers), 0) + for name, tls_version, bits in ciphers: + if not alg1 in name.split("-") and alg2 not in name: + self.fail(name) + + def test_read_write_after_close_raises_valuerror(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + + with server: + s = context.wrap_socket(socket.socket()) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + def test_sendfile(self): + TEST_DATA = b"x" * 512 + with open(support.TESTFN, 'wb') as f: + f.write(TEST_DATA) + self.addCleanup(support.unlink, support.TESTFN) + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + with open(support.TESTFN, 'rb') as file: + s.sendfile(file) + self.assertEqual(s.recv(1024), TEST_DATA) + + +def test_main(verbose=False): + if support.verbose: + import warnings + plats = { + 'Linux': platform.linux_distribution, + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', + 'dist\(\) and linux_distribution\(\) ' + 'functions are deprecated .*', + PendingDeprecationWarning, + ) + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + tests = [ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests] + + if support.is_resource_enabled('network'): + tests.append(NetworkedTests) + tests.append(NetworkedBIOTests) + + if _have_threads: + thread_info = support.threading_setup() + if thread_info: + tests.append(ThreadedTests) + + try: + support.run_unittest(*tests) + finally: + if _have_threads: + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.5pypy/test_subprocess.py b/src/greentest/3.5pypy/test_subprocess.py new file mode 100644 index 0000000..b7c8b19 --- /dev/null +++ b/src/greentest/3.5pypy/test_subprocess.py @@ -0,0 +1,2754 @@ +import unittest +from unittest import mock +from test.support import script_helper +from test import support +import subprocess +import sys +import signal +import io +import locale +import os +import errno +import tempfile +import time +import re +import selectors +import sysconfig +import warnings +import select +import shutil +import gc +import textwrap + +try: + import threading +except ImportError: + threading = None + +mswindows = (sys.platform == "win32") + +# +# Depends on the following external programs: Python +# + +if mswindows: + SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), ' + 'os.O_BINARY);') +else: + SETBINARY = '' + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + # Try to minimize the number of children we have so this test + # doesn't crash on some buildbots (Alphas in particular). + support.reap_children() + + def tearDown(self): + for inst in subprocess._active: + inst.wait() + subprocess._cleanup() + self.assertFalse(subprocess._active, "subprocess._active not empty") + + def assertStderrEqual(self, stderr, expected, msg=None): + # In a debug build, stuff like "[6580 refs]" is printed to stderr at + # shutdown time. That frustrates tests trying to check stderr produced + # from a spawned Python process. + actual = support.strip_python_stderr(stderr) + # strip_python_stderr also strips whitespace, so we do too. + expected = expected.strip() + self.assertEqual(actual, expected, msg) + + +class PopenTestException(Exception): + pass + + +class PopenExecuteChildRaises(subprocess.Popen): + """Popen subclass for testing cleanup of subprocess.PIPE filehandles when + _execute_child fails. + """ + def _execute_child(self, *args, **kwargs): + raise PopenTestException("Forced Exception for Test") + + +class ProcessTestCase(BaseTestCase): + + def test_io_buffered_by_default(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + self.assertIsInstance(p.stdin, io.BufferedIOBase) + self.assertIsInstance(p.stdout, io.BufferedIOBase) + self.assertIsInstance(p.stderr, io.BufferedIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_io_unbuffered_works(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0) + try: + self.assertIsInstance(p.stdin, io.RawIOBase) + self.assertIsInstance(p.stdout, io.RawIOBase) + self.assertIsInstance(p.stderr, io.RawIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_call_seq(self): + # call() function with sequence argument + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(rc, 47) + + def test_call_timeout(self): + # call() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.call waits for the + # child. + self.assertRaises(subprocess.TimeoutExpired, subprocess.call, + [sys.executable, "-c", "while True: pass"], + timeout=0.1) + + def test_check_call_zero(self): + # check_call() function with zero return code + rc = subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(0)"]) + self.assertEqual(rc, 0) + + def test_check_call_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(c.exception.returncode, 47) + + def test_check_output(self): + # check_output() function with zero return code + output = subprocess.check_output( + [sys.executable, "-c", "print('BDFL')"]) + self.assertIn(b'BDFL', output) + + def test_check_output_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_output( + [sys.executable, "-c", "import sys; sys.exit(5)"]) + self.assertEqual(c.exception.returncode, 5) + + def test_check_output_stderr(self): + # check_output() function stderr redirected to stdout + output = subprocess.check_output( + [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], + stderr=subprocess.STDOUT) + self.assertIn(b'BDFL', output) + + def test_check_output_stdin_arg(self): + # check_output() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + stdin=tf) + self.assertIn(b'PEAR', output) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + input=b'pear') + self.assertIn(b'PEAR', output) + + def test_check_output_stdout_arg(self): + # check_output() refuses to accept 'stdout' argument + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdout=sys.stdout) + self.fail("Expected ValueError when stdout arg supplied.") + self.assertIn('stdout', c.exception.args[0]) + + def test_check_output_stdin_with_input_arg(self): + # check_output() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdin=tf, input=b'hare') + self.fail("Expected ValueError when stdin and input args supplied.") + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + # check_output() function with timeout arg + with self.assertRaises(subprocess.TimeoutExpired) as c: + output = subprocess.check_output( + [sys.executable, "-c", + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"], + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3) + self.fail("Expected TimeoutExpired.") + self.assertEqual(c.exception.output, b'BDFL') + + def test_call_kwargs(self): + # call() function with keyword args + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + rc = subprocess.call([sys.executable, "-c", + 'import sys, os;' + 'sys.exit(os.getenv("FRUIT")=="banana")'], + env=newenv) + self.assertEqual(rc, 1) + + def test_invalid_args(self): + # Popen() called with invalid arguments should raise TypeError + # but Popen.__del__ should not complain (issue #12085) + with support.captured_stderr() as s: + self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) + argcount = subprocess.Popen.__init__.__code__.co_argcount + too_many_args = [0] * (argcount + 1) + self.assertRaises(TypeError, subprocess.Popen, *too_many_args) + self.assertEqual(s.getvalue(), '') + + def test_stdin_none(self): + # .stdin is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + p.wait() + self.assertEqual(p.stdin, None) + + def test_stdout_none(self): + # .stdout is None when not redirected, and the child's stdout will + # be inherited from the parent. In order to test this we run a + # subprocess in a subprocess: + # this_test + # \-- subprocess created by this test (parent) + # \-- subprocess created by the parent subprocess (child) + # The parent doesn't specify stdout, so the child will use the + # parent's stdout. This test checks that the message printed by the + # child goes to the parent stdout. The parent also checks that the + # child's stdout is None. See #11963. + code = ('import sys; from subprocess import Popen, PIPE;' + 'p = Popen([sys.executable, "-c", "print(\'test_stdout_none\')"],' + ' stdin=PIPE, stderr=PIPE);' + 'p.wait(); assert p.stdout is None;') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test_stdout_none') + + def test_stderr_none(self): + # .stderr is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + p.wait() + self.assertEqual(p.stderr, None) + + def _assert_python(self, pre_args, **kwargs): + # We include sys.exit() to prevent the test runner from hanging + # whenever python is found. + args = pre_args + ["import sys; sys.exit(47)"] + p = subprocess.Popen(args, **kwargs) + p.wait() + self.assertEqual(47, p.returncode) + + def test_executable(self): + # Check that the executable argument works. + # + # On Unix (non-Mac and non-Windows), Python looks at args[0] to + # determine where its standard library is, so we need the directory + # of args[0] to be valid for the Popen() call to Python to succeed. + # See also issue #16170 and issue #7774. + doesnotexist = os.path.join(os.path.dirname(sys.executable), + "doesnotexist") + self._assert_python([doesnotexist, "-c"], executable=sys.executable) + + def test_executable_takes_precedence(self): + # Check that the executable argument takes precedence over args[0]. + # + # Verify first that the call succeeds without the executable arg. + pre_args = [sys.executable, "-c"] + self._assert_python(pre_args) + self.assertRaises(FileNotFoundError, self._assert_python, pre_args, + executable="doesnotexist") + + @unittest.skipIf(mswindows, "executable argument replaces shell") + def test_executable_replaces_shell(self): + # Check that the executable argument replaces the default shell + # when shell=True. + self._assert_python([], executable=sys.executable, shell=True) + + # For use in the test_cwd* tests below. + def _normalize_cwd(self, cwd): + # Normalize an expected cwd (for Tru64 support). + # We can't use os.path.realpath since it doesn't expand Tru64 {memb} + # strings. See bug #1063571. + with support.change_cwd(cwd): + return os.getcwd() + + # For use in the test_cwd* tests below. + def _split_python_path(self): + # Return normalized (python_dir, python_base). + python_path = os.path.realpath(sys.executable) + return os.path.split(python_path) + + # For use in the test_cwd* tests below. + def _assert_cwd(self, expected_cwd, python_arg, **kwargs): + # Invoke Python via Popen, and assert that (1) the call succeeds, + # and that (2) the current working directory of the child process + # matches *expected_cwd*. + p = subprocess.Popen([python_arg, "-c", + "import os, sys; " + "sys.stdout.write(os.getcwd()); " + "sys.exit(47)"], + stdout=subprocess.PIPE, + **kwargs) + self.addCleanup(p.stdout.close) + p.wait() + self.assertEqual(47, p.returncode) + normcase = os.path.normcase + self.assertEqual(normcase(expected_cwd), + normcase(p.stdout.read().decode("utf-8"))) + + def test_cwd(self): + # Check that cwd changes the cwd for the child process. + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_arg(self): + # Check that Popen looks for args[0] relative to cwd if args[0] + # is relative. + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + with support.temp_cwd() as wrong_dir: + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python]) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, rel_python, cwd=python_dir) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_executable(self): + # Check that Popen looks for executable relative to cwd if executable + # is relative (and that executable takes precedence over args[0]). + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + doesntexist = "somethingyoudonthave" + with support.temp_cwd() as wrong_dir: + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python, + cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, doesntexist, executable=rel_python, + cwd=python_dir) + + def test_cwd_with_absolute_arg(self): + # Check that Popen can find the executable when the cwd is wrong + # if args[0] is an absolute path. + python_dir, python_base = self._split_python_path() + abs_python = os.path.join(python_dir, python_base) + rel_python = os.path.join(os.curdir, python_base) + with support.temp_dir() as wrong_dir: + # Before calling with an absolute path, confirm that using a + # relative path fails. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + wrong_dir = self._normalize_cwd(wrong_dir) + self._assert_cwd(wrong_dir, abs_python, cwd=wrong_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + def test_executable_with_cwd(self): + python_dir, python_base = self._split_python_path() + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, "somethingyoudonthave", + executable=sys.executable, cwd=python_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + @unittest.skipIf(sysconfig.is_python_build(), + "need an installed Python. See #7774") + def test_executable_without_cwd(self): + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + self._assert_cwd(os.getcwd(), "somethingyoudonthave", + executable=sys.executable) + + def test_stdin_pipe(self): + # stdin redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.stdin.write(b"pear") + p.stdin.close() + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_filedes(self): + # stdin is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + os.write(d, b"pear") + os.lseek(d, 0, 0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=d) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_fileobj(self): + # stdin is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b"pear") + tf.seek(0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=tf) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdout_pipe(self): + # stdout redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), b"orange") + + def test_stdout_filedes(self): + # stdout is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=d) + p.wait() + os.lseek(d, 0, 0) + self.assertEqual(os.read(d, 1024), b"orange") + + def test_stdout_fileobj(self): + # stdout is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=tf) + p.wait() + tf.seek(0) + self.assertEqual(tf.read(), b"orange") + + def test_stderr_pipe(self): + # stderr redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=subprocess.PIPE) + self.addCleanup(p.stderr.close) + self.assertStderrEqual(p.stderr.read(), b"strawberry") + + def test_stderr_filedes(self): + # stderr is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=d) + p.wait() + os.lseek(d, 0, 0) + self.assertStderrEqual(os.read(d, 1024), b"strawberry") + + def test_stderr_fileobj(self): + # stderr is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"strawberry") + + def test_stderr_redirect_with_no_stdout_redirect(self): + # test stderr=STDOUT while stdout=None (not set) + + # - grandchild prints to stderr + # - child redirects grandchild's stderr to its stdout + # - the parent should get grandchild's stderr in child's stdout + p = subprocess.Popen([sys.executable, "-c", + 'import sys, subprocess;' + 'rc = subprocess.call([sys.executable, "-c",' + ' "import sys;"' + ' "sys.stderr.write(\'42\')"],' + ' stderr=subprocess.STDOUT);' + 'sys.exit(rc)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + #NOTE: stdout should get stderr from grandchild + self.assertStderrEqual(stdout, b'42') + self.assertStderrEqual(stderr, b'') # should be empty + self.assertEqual(p.returncode, 0) + + def test_stdout_stderr_pipe(self): + # capture stdout and stderr to the same pipe + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + self.addCleanup(p.stdout.close) + self.assertStderrEqual(p.stdout.read(), b"appleorange") + + def test_stdout_stderr_file(self): + # capture stdout and stderr to the same open file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=tf, + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"appleorange") + + def test_stdout_filedes_of_stdout(self): + # stdout is set to 1 (#1531862). + # To avoid printing the text on stdout, we do something similar to + # test_stdout_none (see above). The parent subprocess calls the child + # subprocess passing stdout=1, and this test uses stdout=PIPE in + # order to capture and check the output of the parent. See #11963. + code = ('import sys, subprocess; ' + 'rc = subprocess.call([sys.executable, "-c", ' + ' "import os, sys; sys.exit(os.write(sys.stdout.fileno(), ' + 'b\'test with stdout=1\'))"], stdout=1); ' + 'assert rc == 18') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test with stdout=1') + + def test_stdout_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'for i in range(10240):' + 'print("x" * 1024)'], + stdout=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdout, None) + + def test_stderr_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys\n' + 'for i in range(10240):' + 'sys.stderr.write("x" * 1024)'], + stderr=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stderr, None) + + def test_stdin_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdin.read(1)'], + stdin=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdin, None) + + def test_env(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "orange" + with subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange") + + # Windows requires at least the SYSTEMROOT environment variable to start + # Python + @unittest.skipIf(sys.platform == 'win32', + 'cannot test an empty env on Windows') + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') is not None, + 'the python library cannot be loaded ' + 'with an empty environment') + def test_empty_env(self): + with subprocess.Popen([sys.executable, "-c", + 'import os; ' + 'print(list(os.environ.keys()))'], + stdout=subprocess.PIPE, + env={}) as p: + stdout, stderr = p.communicate() + self.assertIn(stdout.strip(), + (b"[]", + # Mac OS X adds __CF_USER_TEXT_ENCODING variable to an empty + # environment + b"['__CF_USER_TEXT_ENCODING']")) + + def test_communicate_stdin(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.communicate(b"pear") + self.assertEqual(p.returncode, 1) + + def test_communicate_stdout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("pineapple")'], + stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, b"pineapple") + self.assertEqual(stderr, None) + + def test_communicate_stderr(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("pineapple")'], + stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple");' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(b"banana") + self.assertEqual(stdout, b"banana") + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate_timeout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stderr.write("pineapple\\n");' + 'time.sleep(1);' + 'sys.stderr.write("pear\\n");' + 'sys.stdout.write(sys.stdin.read())'], + universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, "banana", + timeout=0.3) + # Make sure we can keep waiting for it, and that we get the whole output + # after it completes. + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr.encode(), b"pineapple\npear\n") + + def test_communicate_timeout_large_output(self): + # Test an expiring timeout while the child is outputting lots of data. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));'], + stdout=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, timeout=0.4) + (stdout, _) = p.communicate() + self.assertEqual(len(stdout), 4 * 64 * 1024) + + # Test for the fd leak reported in http://bugs.python.org/issue2791. + def test_communicate_pipe_fd_leak(self): + for stdin_pipe in (False, True): + for stdout_pipe in (False, True): + for stderr_pipe in (False, True): + options = {} + if stdin_pipe: + options['stdin'] = subprocess.PIPE + if stdout_pipe: + options['stdout'] = subprocess.PIPE + if stderr_pipe: + options['stderr'] = subprocess.PIPE + if not options: + continue + p = subprocess.Popen((sys.executable, "-c", "pass"), **options) + p.communicate() + if p.stdin is not None: + self.assertTrue(p.stdin.closed) + if p.stdout is not None: + self.assertTrue(p.stdout.closed) + if p.stderr is not None: + self.assertTrue(p.stderr.closed) + + def test_communicate_returns(self): + # communicate() should return None if no redirection is active + p = subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(47)"]) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertEqual(stderr, None) + + def test_communicate_pipe_buf(self): + # communicate() with writes larger than pipe_buf + # This test will probably deadlock rather than fail, if + # communicate() does not work properly. + x, y = os.pipe() + os.close(x) + os.close(y) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read(47));' + 'sys.stderr.write("x" * %d);' + 'sys.stdout.write(sys.stdin.read())' % + support.PIPE_MAX_SIZE], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + string_to_write = b"a" * support.PIPE_MAX_SIZE + (stdout, stderr) = p.communicate(string_to_write) + self.assertEqual(stdout, string_to_write) + + def test_writes_before_communicate(self): + # stdin.write before communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.stdin.write(b"banana") + (stdout, stderr) = p.communicate(b"split") + self.assertEqual(stdout, b"bananasplit") + self.assertStderrEqual(stderr, b"") + + def test_universal_newlines(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(sys.stdin.readline().encode());' + 'buf.flush();' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(sys.stdin.read().encode());' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + p.stdin.write("line1\n") + p.stdin.flush() + self.assertEqual(p.stdout.readline(), "line1\n") + p.stdin.write("line3\n") + p.stdin.close() + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.readline(), + "line2\n") + self.assertEqual(p.stdout.read(6), + "line3\n") + self.assertEqual(p.stdout.read(), + "line4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate(self): + # universal newlines through communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, + "line2\nline4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate_stdin(self): + # universal newlines through communicate(), with only stdin + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.readline() + assert s == "line1\\n", repr(s) + s = sys.stdin.read() + assert s == "line3\\n", repr(s) + ''')], + stdin=subprocess.PIPE, + universal_newlines=1) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_input_none(self): + # Test communicate(input=None) with universal newlines. + # + # We set stdout to PIPE because, as of this writing, a different + # code path is tested when the number of pipes is zero or one. + p = subprocess.Popen([sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + p.communicate() + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_stdin_stdout_stderr(self): + # universal newlines through communicate(), with stdin, stdout, stderr + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.buffer.readline() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line2\\r") + sys.stderr.buffer.write(b"eline2\\n") + s = sys.stdin.buffer.read() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line4\\n") + sys.stdout.buffer.write(b"line5\\r\\n") + sys.stderr.buffer.write(b"eline6\\r") + sys.stderr.buffer.write(b"eline7\\r\\nz") + ''')], + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) + # Python debug build push something like "[42442 refs]\n" + # to stderr at exit of subprocess. + # Don't use assertStderrEqual because it strips CR and LF from output. + self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + + def test_universal_newlines_communicate_encodings(self): + # Check that universal newlines mode works for various encodings, + # in particular for encodings in the UTF-16 and UTF-32 families. + # See issue #15595. + # + # UTF-16 and UTF-32-BE are sufficient to check both with BOM and + # without, and UTF-16 and UTF-32. + import _bootlocale + for encoding in ['utf-16', 'utf-32-be']: + old_getpreferredencoding = _bootlocale.getpreferredencoding + # Indirectly via io.TextIOWrapper, Popen() defaults to + # locale.getpreferredencoding(False) and earlier in Python 3.2 to + # locale.getpreferredencoding(). + def getpreferredencoding(do_setlocale=True): + return encoding + code = ("import sys; " + r"sys.stdout.buffer.write('1\r\n2\r3\n4'.encode('%s'))" % + encoding) + args = [sys.executable, '-c', code] + try: + _bootlocale.getpreferredencoding = getpreferredencoding + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout, stderr = popen.communicate(input='') + finally: + _bootlocale.getpreferredencoding = old_getpreferredencoding + self.assertEqual(stdout, '1\n2\n3\n4') + + def test_no_leaking(self): + # Make sure we leak no resources + if not mswindows: + max_handles = 1026 # too much for most UNIX systems + else: + max_handles = 2050 # too much for (at least some) Windows setups + handles = [] + tmpdir = tempfile.mkdtemp() + try: + for i in range(max_handles): + try: + tmpfile = os.path.join(tmpdir, support.TESTFN) + handles.append(os.open(tmpfile, os.O_WRONLY|os.O_CREAT)) + except OSError as e: + if e.errno != errno.EMFILE: + raise + break + else: + self.skipTest("failed to reach the file descriptor limit " + "(tried %d)" % max_handles) + # Close a couple of them (should be enough for a subprocess) + for i in range(10): + os.close(handles.pop()) + # Loop creating some subprocesses. If one of them leaks some fds, + # the next loop iteration will fail by reaching the max fd limit. + for i in range(15): + p = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write(sys.stdin.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + data = p.communicate(b"lime")[0] + self.assertEqual(data, b"lime") + finally: + for h in handles: + os.close(h) + shutil.rmtree(tmpdir) + + def test_list2cmdline(self): + self.assertEqual(subprocess.list2cmdline(['a b c', 'd', 'e']), + '"a b c" d e') + self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']), + 'ab\\"c \\ d') + self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']), + 'ab\\"c " \\\\" d') + self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']), + 'a\\\\\\b "de fg" h') + self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']), + 'a\\\\\\"b c d') + self.assertEqual(subprocess.list2cmdline(['a\\\\b c', 'd', 'e']), + '"a\\\\b c" d e') + self.assertEqual(subprocess.list2cmdline(['a\\\\b\\ c', 'd', 'e']), + '"a\\\\b\\ c" d e') + self.assertEqual(subprocess.list2cmdline(['ab', '']), + 'ab ""') + + def test_poll(self): + p = subprocess.Popen([sys.executable, "-c", + "import os; os.read(0, 1)"], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + self.assertIsNone(p.poll()) + os.write(p.stdin.fileno(), b'A') + p.wait() + # Subsequent invocations should just return the returncode + self.assertEqual(p.poll(), 0) + + def test_wait(self): + p = subprocess.Popen([sys.executable, "-c", "pass"]) + self.assertEqual(p.wait(), 0) + # Subsequent invocations should just return the returncode + self.assertEqual(p.wait(), 0) + + def test_wait_timeout(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(0.3)"]) + with self.assertRaises(subprocess.TimeoutExpired) as c: + p.wait(timeout=0.0001) + self.assertIn("0.0001", str(c.exception)) # For coverage of __str__. + # Some heavily loaded buildbots (sparc Debian 3.x) require this much + # time to start. + self.assertEqual(p.wait(timeout=3), 0) + + def test_invalid_bufsize(self): + # an invalid type of the bufsize argument should raise + # TypeError. + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], "orange") + + def test_bufsize_is_none(self): + # bufsize=None should be the same as bufsize=0. + p = subprocess.Popen([sys.executable, "-c", "pass"], None) + self.assertEqual(p.wait(), 0) + # Again with keyword arg + p = subprocess.Popen([sys.executable, "-c", "pass"], bufsize=None) + self.assertEqual(p.wait(), 0) + + def _test_bufsize_equal_one(self, line, expected, universal_newlines): + # subprocess may deadlock with bufsize=1, see issue #21332 + with subprocess.Popen([sys.executable, "-c", "import sys;" + "sys.stdout.write(sys.stdin.readline());" + "sys.stdout.flush()"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + bufsize=1, + universal_newlines=universal_newlines) as p: + p.stdin.write(line) # expect that it flushes the line in text mode + os.close(p.stdin.fileno()) # close it without flushing the buffer + read_line = p.stdout.readline() + try: + p.stdin.close() + except OSError: + pass + p.stdin = None + self.assertEqual(p.returncode, 0) + self.assertEqual(read_line, expected) + + def test_bufsize_equal_one_text_mode(self): + # line is flushed in text mode with bufsize=1. + # we should get the full line in return + line = "line\n" + self._test_bufsize_equal_one(line, line, universal_newlines=True) + + def test_bufsize_equal_one_binary_mode(self): + # line is not flushed in binary mode with bufsize=1. + # we should get empty response + line = b'line' + os.linesep.encode() # assume ascii-based locale + self._test_bufsize_equal_one(line, b'', universal_newlines=False) + + def test_leaking_fds_on_error(self): + # see bug #5179: Popen leaks file descriptors to PIPEs if + # the child fails to execute; this will eventually exhaust + # the maximum number of open fds. 1024 seems a very common + # value for that limit, but Windows has 2048, so we loop + # 1024 times (each call leaked two fds). + for i in range(1024): + with self.assertRaises(OSError) as c: + subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # ignore errors that indicate the command was not found + if c.exception.errno not in (errno.ENOENT, errno.EACCES): + raise c.exception + + @unittest.skipIf(threading is None, "threading required") + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(['nonexisting_i_hope'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + + @unittest.skipIf(threading is None, "threading required") + def test_threadsafe_wait(self): + """Issue21291: Popen.wait() needs to be threadsafe for returncode.""" + proc = subprocess.Popen([sys.executable, '-c', + 'import time; time.sleep(12)']) + self.assertEqual(proc.returncode, None) + results = [] + + def kill_proc_timer_thread(): + results.append(('thread-start-poll-result', proc.poll())) + # terminate it from the thread and wait for the result. + proc.kill() + proc.wait() + results.append(('thread-after-kill-and-wait', proc.returncode)) + # this wait should be a no-op given the above. + proc.wait() + results.append(('thread-after-second-wait', proc.returncode)) + + # This is a timing sensitive test, the failure mode is + # triggered when both the main thread and this thread are in + # the wait() call at once. The delay here is to allow the + # main thread to most likely be blocked in its wait() call. + t = threading.Timer(0.2, kill_proc_timer_thread) + t.start() + + if mswindows: + expected_errorcode = 1 + else: + # Should be -9 because of the proc.kill() from the thread. + expected_errorcode = -9 + + # Wait for the process to finish; the thread should kill it + # long before it finishes on its own. Supplying a timeout + # triggers a different code path for better coverage. + proc.wait(timeout=20) + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in wait from main thread") + + # This should be a no-op with no change in returncode. + proc.wait() + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in second main wait.") + + t.join() + # Ensure that all of the thread results are as expected. + # When a race condition occurs in wait(), the returncode could + # be set by the wrong thread that doesn't actually have it + # leading to an incorrect value. + self.assertEqual([('thread-start-poll-result', None), + ('thread-after-kill-and-wait', expected_errorcode), + ('thread-after-second-wait', expected_errorcode)], + results) + + def test_issue8780(self): + # Ensure that stdout is inherited from the parent + # if stdout=PIPE is not used + code = ';'.join(( + 'import subprocess, sys', + 'retcode = subprocess.call(' + "[sys.executable, '-c', 'print(\"Hello World!\")'])", + 'assert retcode == 0')) + output = subprocess.check_output([sys.executable, '-c', code]) + self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + + def test_handles_closed_on_exception(self): + # If CreateProcess exits with an error, ensure the + # duplicate output handles are released + ifhandle, ifname = tempfile.mkstemp() + ofhandle, ofname = tempfile.mkstemp() + efhandle, efname = tempfile.mkstemp() + try: + subprocess.Popen (["*"], stdin=ifhandle, stdout=ofhandle, + stderr=efhandle) + except OSError: + os.close(ifhandle) + os.remove(ifname) + os.close(ofhandle) + os.remove(ofname) + os.close(efhandle) + os.remove(efname) + self.assertFalse(os.path.exists(ifname)) + self.assertFalse(os.path.exists(ofname)) + self.assertFalse(os.path.exists(efname)) + + def test_communicate_epipe(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.communicate(b"x" * 2**20) + + def test_communicate_epipe_only_stdin(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + p.wait() + p.communicate(b"x" * 2**20) + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), + "Requires signal.SIGUSR1") + @unittest.skipUnless(hasattr(os, 'kill'), + "Requires os.kill") + @unittest.skipUnless(hasattr(os, 'getppid'), + "Requires os.getppid") + def test_communicate_eintr(self): + # Issue #12493: communicate() should handle EINTR + def handler(signum, frame): + pass + old_handler = signal.signal(signal.SIGUSR1, handler) + self.addCleanup(signal.signal, signal.SIGUSR1, old_handler) + + args = [sys.executable, "-c", + 'import os, signal;' + 'os.kill(os.getppid(), signal.SIGUSR1)'] + for stream in ('stdout', 'stderr'): + kw = {stream: subprocess.PIPE} + with subprocess.Popen(args, **kw) as process: + # communicate() will be interrupted by SIGUSR1 + process.communicate() + + + # This test is Linux-ish specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + def test_failed_child_execute_fd_leak(self): + """Test for the fork() failure fd leak reported in issue16327.""" + fd_directory = '/proc/%d/fd' % os.getpid() + fds_before_popen = os.listdir(fd_directory) + with self.assertRaises(PopenTestException): + PopenExecuteChildRaises( + [sys.executable, '-c', 'pass'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # NOTE: This test doesn't verify that the real _execute_child + # does not close the file descriptors itself on the way out + # during an exception. Code inspection has confirmed that. + + fds_after_exception = os.listdir(fd_directory) + self.assertEqual(fds_before_popen, fds_after_exception) + + +class RunFuncTestCase(BaseTestCase): + def run_python(self, code, **kwargs): + """Run Python code in a subprocess using subprocess.run""" + argv = [sys.executable, "-c", code] + return subprocess.run(argv, **kwargs) + + def test_returncode(self): + # call() function with sequence argument + cp = self.run_python("import sys; sys.exit(47)") + self.assertEqual(cp.returncode, 47) + with self.assertRaises(subprocess.CalledProcessError): + cp.check_returncode() + + def test_check(self): + with self.assertRaises(subprocess.CalledProcessError) as c: + self.run_python("import sys; sys.exit(47)", check=True) + self.assertEqual(c.exception.returncode, 47) + + def test_check_zero(self): + # check_returncode shouldn't raise when returncode is zero + cp = self.run_python("import sys; sys.exit(0)", check=True) + self.assertEqual(cp.returncode, 0) + + def test_timeout(self): + # run() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.run waits for the + # child. + with self.assertRaises(subprocess.TimeoutExpired): + self.run_python("while True: pass", timeout=0.0001) + + def test_capture_stdout(self): + # capture stdout with zero return code + cp = self.run_python("print('BDFL')", stdout=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stdout) + + def test_capture_stderr(self): + cp = self.run_python("import sys; sys.stderr.write('BDFL')", + stderr=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stderr) + + def test_check_output_stdin_arg(self): + # run() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + stdin=tf, stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + input=b'pear', stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_stdin_with_input_arg(self): + # run() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError, + msg="Expected ValueError when stdin and input args supplied.") as c: + output = self.run_python("print('will not be run')", + stdin=tf, input=b'hare') + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + with self.assertRaises(subprocess.TimeoutExpired) as c: + cp = self.run_python(( + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"), + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3, stdout=subprocess.PIPE) + self.assertEqual(c.exception.output, b'BDFL') + # output is aliased to stdout + self.assertEqual(c.exception.stdout, b'BDFL') + + def test_run_kwargs(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + cp = self.run_python(('import sys, os;' + 'sys.exit(33 if os.getenv("FRUIT")=="banana" else 31)'), + env=newenv) + self.assertEqual(cp.returncode, 33) + + +@unittest.skipIf(mswindows, "POSIX specific tests") +class POSIXProcessTestCase(BaseTestCase): + + def setUp(self): + super().setUp() + self._nonexistent_dir = "/_this/pa.th/does/not/exist" + + def _get_chdir_exception(self): + try: + os.chdir(self._nonexistent_dir) + except OSError as e: + # This avoids hard coding the errno value or the OS perror() + # string and instead capture the exception that we want to see + # below for comparison. + desired_exception = e + desired_exception.strerror += ': ' + repr(self._nonexistent_dir) + else: + self.fail("chdir to nonexistent directory %s succeeded." % + self._nonexistent_dir) + return desired_exception + + def test_exception_cwd(self): + """Test error in the child raised in the parent for a bad cwd.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + cwd=self._nonexistent_dir) + except OSError as e: + # Test that the child process chdir failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_executable(self): + """Test error in the child raised in the parent for a bad executable.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + executable=self._nonexistent_dir) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_args_0(self): + """Test error in the child raised in the parent for a bad args[0].""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([self._nonexistent_dir, "-c", ""]) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_restore_signals(self): + # Code coverage for both values of restore_signals to make sure it + # at least does not blow up. + # A test for behavior would be complex. Contributions welcome. + subprocess.call([sys.executable, "-c", ""], restore_signals=True) + subprocess.call([sys.executable, "-c", ""], restore_signals=False) + + def test_start_new_session(self): + # For code coverage of calling setsid(). We don't care if we get an + # EPERM error from it depending on the test execution environment, that + # still indicates that it was called. + try: + output = subprocess.check_output( + [sys.executable, "-c", + "import os; print(os.getpgid(os.getpid()))"], + start_new_session=True) + except OSError as e: + if e.errno != errno.EPERM: + raise + else: + parent_pgid = os.getpgid(os.getpid()) + child_pgid = int(output) + self.assertNotEqual(parent_pgid, child_pgid) + + def test_run_abort(self): + # returncode handles signal termination + with support.SuppressCrashReport(): + p = subprocess.Popen([sys.executable, "-c", + 'import os; os.abort()']) + p.wait() + self.assertEqual(-p.returncode, signal.SIGABRT) + + def test_preexec(self): + # DISCLAIMER: Setting environment variables is *not* a good use + # of a preexec_fn. This is merely a test. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + preexec_fn=lambda: os.putenv("FRUIT", "apple")) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read(), b"apple") + + def test_preexec_exception(self): + def raise_it(): + raise ValueError("What if two swallows carried a coconut?") + try: + p = subprocess.Popen([sys.executable, "-c", ""], + preexec_fn=raise_it) + except subprocess.SubprocessError as e: + self.assertTrue( + subprocess._posixsubprocess, + "Expected a ValueError from the preexec_fn") + except ValueError as e: + self.assertIn("coconut", e.args[0]) + else: + self.fail("Exception raised by preexec_fn did not make it " + "to the parent process.") + + class _TestExecuteChildPopen(subprocess.Popen): + """Used to test behavior at the end of _execute_child.""" + def __init__(self, testcase, *args, **kwargs): + self._testcase = testcase + subprocess.Popen.__init__(self, *args, **kwargs) + + def _execute_child(self, *args, **kwargs): + try: + subprocess.Popen._execute_child(self, *args, **kwargs) + finally: + # Open a bunch of file descriptors and verify that + # none of them are the same as the ones the Popen + # instance is using for stdin/stdout/stderr. + devzero_fds = [os.open("/dev/zero", os.O_RDONLY) + for _ in range(8)] + try: + for fd in devzero_fds: + self._testcase.assertNotIn( + fd, (self.stdin.fileno(), self.stdout.fileno(), + self.stderr.fileno()), + msg="At least one fd was closed early.") + finally: + for fd in devzero_fds: + os.close(fd) + + @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") + def test_preexec_errpipe_does_not_double_close_pipes(self): + """Issue16140: Don't double close pipes on preexec error.""" + + def raise_it(): + raise subprocess.SubprocessError( + "force the _execute_child() errpipe_data path.") + + with self.assertRaises(subprocess.SubprocessError): + self._TestExecuteChildPopen( + self, [sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=raise_it) + + @support.impl_detail("PyPy's _posixsubprocess doesn't have to disable gc") + def test_preexec_gc_module_failure(self): + # This tests the code that disables garbage collection if the child + # process will execute any Python. + def raise_runtime_error(): + raise RuntimeError("this shouldn't escape") + enabled = gc.isenabled() + orig_gc_disable = gc.disable + orig_gc_isenabled = gc.isenabled + try: + gc.disable() + self.assertFalse(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertFalse(gc.isenabled(), + "Popen enabled gc when it shouldn't.") + + gc.enable() + self.assertTrue(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertTrue(gc.isenabled(), "Popen left gc disabled.") + + gc.disable = raise_runtime_error + self.assertRaises(RuntimeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + + del gc.isenabled # force an AttributeError + self.assertRaises(AttributeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + finally: + gc.disable = orig_gc_disable + gc.isenabled = orig_gc_isenabled + if not enabled: + gc.disable() + + @unittest.skipIf( + sys.platform == 'darwin', 'setrlimit() seems to fail on OS X') + def test_preexec_fork_failure(self): + # The internal code did not preserve the previous exception when + # re-enabling garbage collection + try: + from resource import getrlimit, setrlimit, RLIMIT_NPROC + except ImportError as err: + self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD + limits = getrlimit(RLIMIT_NPROC) + [_, hard] = limits + setrlimit(RLIMIT_NPROC, (0, hard)) + self.addCleanup(setrlimit, RLIMIT_NPROC, limits) + try: + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + except BlockingIOError: + # Forking should raise EAGAIN, translated to BlockingIOError + pass + else: + self.skipTest('RLIMIT_NPROC had no effect; probably superuser') + + def test_args_string(self): + # args is a string + fd, fname = tempfile.mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!/bin/sh\n") + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + p = subprocess.Popen(fname) + p.wait() + os.remove(fname) + self.assertEqual(p.returncode, 47) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + startupinfo=47) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + creationflags=47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen(["echo $FRUIT"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen("echo $FRUIT", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_call_string(self): + # call() function with string argument on UNIX + fd, fname = tempfile.mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!/bin/sh\n") + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + rc = subprocess.call(fname) + os.remove(fname) + self.assertEqual(rc, 47) + + def test_specific_shell(self): + # Issue #9265: Incorrect name passed as arg[0]. + shells = [] + for prefix in ['/bin', '/usr/bin/', '/usr/local/bin']: + for name in ['bash', 'ksh']: + sh = os.path.join(prefix, name) + if os.path.isfile(sh): + shells.append(sh) + if not shells: # Will probably work for any shell but csh. + self.skipTest("bash or ksh required for this test") + sh = '/bin/sh' + if os.path.isfile(sh) and not os.path.islink(sh): + # Test will fail if /bin/sh is a symlink to csh. + shells.append(sh) + for sh in shells: + p = subprocess.Popen("echo $0", executable=sh, shell=True, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) + + def _kill_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + # Also set the SIGINT handler to the default to make sure it's not + # being ignored (some tests rely on that.) + old_handler = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + signal.signal(signal.SIGINT, old_handler) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + return p + + @unittest.skipIf(sys.platform.startswith(('netbsd', 'openbsd')), + "Due to known OS bug (issue #16762)") + def _kill_dead_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + p.communicate() + + def test_send_signal(self): + p = self._kill_process('send_signal', signal.SIGINT) + _, stderr = p.communicate() + self.assertIn(b'KeyboardInterrupt', stderr) + self.assertNotEqual(p.wait(), 0) + + def test_kill(self): + p = self._kill_process('kill') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGKILL) + + def test_terminate(self): + p = self._kill_process('terminate') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGTERM) + + def test_send_signal_dead(self): + # Sending a signal to a dead process + self._kill_dead_process('send_signal', signal.SIGINT) + + def test_kill_dead(self): + # Killing a dead process + self._kill_dead_process('kill') + + def test_terminate_dead(self): + # Terminating a dead process + self._kill_dead_process('terminate') + + def _save_fds(self, save_fds): + fds = [] + for fd in save_fds: + inheritable = os.get_inheritable(fd) + saved = os.dup(fd) + fds.append((fd, saved, inheritable)) + return fds + + def _restore_fds(self, fds): + for fd, saved, inheritable in fds: + os.dup2(saved, fd, inheritable=inheritable) + os.close(saved) + + def check_close_std_fds(self, fds): + # Issue #9905: test that subprocess pipes still work properly with + # some standard fds closed + stdin = 0 + saved_fds = self._save_fds(fds) + for fd, saved, inheritable in saved_fds: + if fd == 0: + stdin = saved + break + try: + for fd in fds: + os.close(fd) + out, err = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + err = support.strip_python_stderr(err) + self.assertEqual((out, err), (b'apple', b'orange')) + finally: + self._restore_fds(saved_fds) + + def test_close_fd_0(self): + self.check_close_std_fds([0]) + + def test_close_fd_1(self): + self.check_close_std_fds([1]) + + def test_close_fd_2(self): + self.check_close_std_fds([2]) + + def test_close_fds_0_1(self): + self.check_close_std_fds([0, 1]) + + def test_close_fds_0_2(self): + self.check_close_std_fds([0, 2]) + + def test_close_fds_1_2(self): + self.check_close_std_fds([1, 2]) + + def test_close_fds_0_1_2(self): + # Issue #10806: test that subprocess pipes still work properly with + # all standard fds closed. + self.check_close_std_fds([0, 1, 2]) + + def test_small_errpipe_write_fd(self): + """Issue #15798: Popen should work when stdio fds are available.""" + new_stdin = os.dup(0) + new_stdout = os.dup(1) + try: + os.close(0) + os.close(1) + + # Side test: if errpipe_write fails to have its CLOEXEC + # flag set this should cause the parent to think the exec + # failed. Extremely unlikely: everyone supports CLOEXEC. + subprocess.Popen([ + sys.executable, "-c", + "print('AssertionError:0:CLOEXEC failure.')"]).wait() + finally: + # Restore original stdin and stdout + os.dup2(new_stdin, 0) + os.dup2(new_stdout, 1) + os.close(new_stdin) + os.close(new_stdout) + + def test_remapping_std_fds(self): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + try: + temp_fds = [fd for fd, fname in temps] + + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # write some data to what will become stdin, and rewind + os.write(temp_fds[1], b"STDIN") + os.lseek(temp_fds[1], 0, 0) + + # move the standard file descriptors out of the way + saved_fds = self._save_fds(range(3)) + try: + # duplicate the file objects over the standard fd's + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # now use those files in the "wrong" order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=temp_fds[1], + stdout=temp_fds[2], + stderr=temp_fds[0]) + p.wait() + finally: + self._restore_fds(saved_fds) + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(temp_fds[2], 1024) + err = support.strip_python_stderr(os.read(temp_fds[0], 1024)) + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + def check_swap_fds(self, stdin_no, stdout_no, stderr_no): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + temp_fds = [fd for fd, fname in temps] + try: + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # save a copy of the standard file descriptors + saved_fds = self._save_fds(range(3)) + try: + # duplicate the temp files over the standard fd's 0, 1, 2 + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # write some data to what will become stdin, and rewind + os.write(stdin_no, b"STDIN") + os.lseek(stdin_no, 0, 0) + + # now use those files in the given order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=stdin_no, + stdout=stdout_no, + stderr=stderr_no) + p.wait() + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(stdout_no, 1024) + err = support.strip_python_stderr(os.read(stderr_no, 1024)) + finally: + self._restore_fds(saved_fds) + + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + # When duping fds, if there arises a situation where one of the fds is + # either 0, 1 or 2, it is possible that it is overwritten (#12607). + # This tests all combinations of this. + def test_swap_fds(self): + self.check_swap_fds(0, 1, 2) + self.check_swap_fds(0, 2, 1) + self.check_swap_fds(1, 0, 2) + self.check_swap_fds(1, 2, 0) + self.check_swap_fds(2, 0, 1) + self.check_swap_fds(2, 1, 0) + + def test_surrogates_error_message(self): + def prepare(): + raise ValueError("surrogate:\uDCff") + + try: + subprocess.call( + [sys.executable, "-c", "pass"], + preexec_fn=prepare) + except ValueError as err: + # Pure Python implementations keeps the message + self.assertIsNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "surrogate:\uDCff") + except subprocess.SubprocessError as err: + # _posixsubprocess uses a default message + self.assertIsNotNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "Exception occurred in preexec_fn.") + else: + self.fail("Expected ValueError or subprocess.SubprocessError") + + def test_undecodable_env(self): + for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): + encoded_value = value.encode("ascii", "surrogateescape") + + # test str with surrogates + script = "import os; print(ascii(os.getenv(%s)))" % repr(key) + env = os.environ.copy() + env[key] = value + # Use C locale to get ASCII for the locale encoding to force + # surrogate-escaping of \xFF in the child process; otherwise it can + # be decoded as-is if the default locale is latin-1. + env['LC_ALL'] = 'C' + if sys.platform.startswith("aix"): + # On AIX, the C locale uses the Latin1 encoding + decoded_value = encoded_value.decode("latin1", "surrogateescape") + else: + # On other UNIXes, the C locale uses the ASCII encoding + decoded_value = value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(decoded_value)) + + # test bytes + key = key.encode("ascii", "surrogateescape") + script = "import os; print(ascii(os.getenvb(%s)))" % repr(key) + env = os.environ.copy() + env[key] = encoded_value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(encoded_value)) + + def test_bytes_program(self): + abs_program = os.fsencode(sys.executable) + path, program = os.path.split(sys.executable) + program = os.fsencode(program) + + # absolute bytes path + exitcode = subprocess.call([abs_program, "-c", "pass"]) + self.assertEqual(exitcode, 0) + + # absolute bytes path as a string + cmd = b"'" + abs_program + b"' -c pass" + exitcode = subprocess.call(cmd, shell=True) + self.assertEqual(exitcode, 0) + + # bytes program, unicode PATH + env = os.environ.copy() + env["PATH"] = path + exitcode = subprocess.call([program, "-c", "pass"], env=env) + self.assertEqual(exitcode, 0) + + # bytes program, bytes PATH + envb = os.environb.copy() + envb[b"PATH"] = os.fsencode(path) + exitcode = subprocess.call([program, "-c", "pass"], env=envb) + self.assertEqual(exitcode, 0) + + def test_pipe_cloexec(self): + sleeper = support.findfile("input_reader.py", subdir="subprocessdata") + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + p1 = subprocess.Popen([sys.executable, sleeper], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=False) + + self.addCleanup(p1.communicate, b'') + + p2 = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + + output, error = p2.communicate() + result_fds = set(map(int, output.split(b','))) + unwanted_fds = set([p1.stdin.fileno(), p1.stdout.fileno(), + p1.stderr.fileno()]) + + self.assertFalse(result_fds & unwanted_fds, + "Expected no fds from %r to be open in child, " + "found %r" % + (unwanted_fds, result_fds & unwanted_fds)) + + def test_pipe_cloexec_real_tools(self): + qcat = support.findfile("qcat.py", subdir="subprocessdata") + qgrep = support.findfile("qgrep.py", subdir="subprocessdata") + + subdata = b'zxcvbn' + data = subdata * 4 + b'\n' + + p1 = subprocess.Popen([sys.executable, qcat], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + close_fds=False) + + p2 = subprocess.Popen([sys.executable, qgrep, subdata], + stdin=p1.stdout, stdout=subprocess.PIPE, + close_fds=False) + + self.addCleanup(p1.wait) + self.addCleanup(p2.wait) + def kill_p1(): + try: + p1.terminate() + except ProcessLookupError: + pass + def kill_p2(): + try: + p2.terminate() + except ProcessLookupError: + pass + self.addCleanup(kill_p1) + self.addCleanup(kill_p2) + + p1.stdin.write(data) + p1.stdin.close() + + readfiles, ignored1, ignored2 = select.select([p2.stdout], [], [], 10) + + self.assertTrue(readfiles, "The child hung") + self.assertEqual(p2.stdout.read(), data) + + p1.stdout.close() + p2.stdout.close() + + def test_close_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + + open_fds = set(fds) + # add a bunch more fds + for _ in range(9): + fd = os.open(os.devnull, os.O_RDONLY) + self.addCleanup(os.close, fd) + open_fds.add(fd) + + for fd in open_fds: + os.set_inheritable(fd, True) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertEqual(remaining_fds & open_fds, open_fds, + "Some fds were closed") + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & open_fds, + "Some fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + # Keep some of the fd's we opened open in the subprocess. + # This tests _posixsubprocess.c's proper handling of fds_to_keep. + fds_to_keep = set(open_fds.pop() for _ in range(8)) + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=()) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & fds_to_keep & open_fds, + "Some fds not in pass_fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD.") + def test_close_fds_when_max_fd_is_lowered(self): + """Confirm that issue21618 is fixed (may fail under valgrind).""" + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # This launches the meat of the test in a child process to + # avoid messing with the larger unittest processes maximum + # number of file descriptors. + # This process launches: + # +--> Process that lowers its RLIMIT_NOFILE aftr setting up + # a bunch of high open fds above the new lower rlimit. + # Those are reported via stdout before launching a new + # process with close_fds=False to run the actual test: + # +--> The TEST: This one launches a fd_status.py + # subprocess with close_fds=True so we can find out if + # any of the fds above the lowered rlimit are still open. + p = subprocess.Popen([sys.executable, '-c', textwrap.dedent( + ''' + import os, resource, subprocess, sys, textwrap + open_fds = set() + # Add a bunch more fds to pass down. + for _ in range(40): + fd = os.open(os.devnull, os.O_RDONLY) + open_fds.add(fd) + + # Leave a two pairs of low ones available for use by the + # internal child error pipe and the stdout pipe. + # We also leave 10 more open as some Python buildbots run into + # "too many open files" errors during the test if we do not. + for fd in sorted(open_fds)[:14]: + os.close(fd) + open_fds.remove(fd) + + for fd in open_fds: + #self.addCleanup(os.close, fd) + os.set_inheritable(fd, True) + + max_fd_open = max(open_fds) + + # Communicate the open_fds to the parent unittest.TestCase process. + print(','.join(map(str, sorted(open_fds)))) + sys.stdout.flush() + + rlim_cur, rlim_max = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + # 29 is lower than the highest fds we are leaving open. + resource.setrlimit(resource.RLIMIT_NOFILE, (29, rlim_max)) + # Launch a new Python interpreter with our low fd rlim_cur that + # inherits open fds above that limit. It then uses subprocess + # with close_fds=True to get a report of open fds in the child. + # An explicit list of fds to check is passed to fd_status.py as + # letting fd_status rely on its default logic would miss the + # fds above rlim_cur as it normally only checks up to that limit. + subprocess.Popen( + [sys.executable, '-c', + textwrap.dedent(""" + import subprocess, sys + subprocess.Popen([sys.executable, %r] + + [str(x) for x in range({max_fd})], + close_fds=True).wait() + """.format(max_fd=max_fd_open+1))], + close_fds=False).wait() + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, (rlim_cur, rlim_max)) + ''' % fd_status)], stdout=subprocess.PIPE) + + output, unused_stderr = p.communicate() + output_lines = output.splitlines() + self.assertEqual(len(output_lines), 2, + msg="expected exactly two lines of output:\n%r" % output) + opened_fds = set(map(int, output_lines[0].strip().split(b','))) + remaining_fds = set(map(int, output_lines[1].strip().split(b','))) + + self.assertFalse(remaining_fds & opened_fds, + msg="Some fds were left open.") + + + # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file + # descriptor of a pipe closed in the parent process is valid in the + # child process according to fstat(), but the mode of the file + # descriptor is invalid, and read or write raise an error. + @support.requires_mac_ver(10, 5) + def test_pass_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + open_fds = set() + + for x in range(5): + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + os.set_inheritable(fds[0], True) + os.set_inheritable(fds[1], True) + open_fds.update(fds) + + for fd in open_fds: + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=(fd, )) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + to_be_closed = open_fds - {fd} + + self.assertIn(fd, remaining_fds, "fd to be passed not passed") + self.assertFalse(remaining_fds & to_be_closed, + "fd to be closed passed") + + # pass_fds overrides close_fds with a warning. + with self.assertWarns(RuntimeWarning) as context: + self.assertFalse(subprocess.call( + [sys.executable, "-c", "import sys; sys.exit(0)"], + close_fds=False, pass_fds=(fd, ))) + self.assertIn('overriding close_fds', str(context.warning)) + + def test_pass_fds_inheritable(self): + script = support.findfile("fd_status.py", subdir="subprocessdata") + + inheritable, non_inheritable = os.pipe() + self.addCleanup(os.close, inheritable) + self.addCleanup(os.close, non_inheritable) + os.set_inheritable(inheritable, True) + os.set_inheritable(non_inheritable, False) + pass_fds = (inheritable, non_inheritable) + args = [sys.executable, script] + args += list(map(str, pass_fds)) + + p = subprocess.Popen(args, + stdout=subprocess.PIPE, close_fds=True, + pass_fds=pass_fds) + output, ignored = p.communicate() + fds = set(map(int, output.split(b','))) + + # the inheritable file descriptor must be inherited, so its inheritable + # flag must be set in the child process after fork() and before exec() + self.assertEqual(fds, set(pass_fds), "output=%a" % output) + + # inheritable flag must not be changed in the parent process + self.assertEqual(os.get_inheritable(inheritable), True) + self.assertEqual(os.get_inheritable(non_inheritable), False) + + def test_stdout_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stdin=inout) + p.wait() + + def test_stdout_stderr_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stderr=inout) + p.wait() + + def test_stderr_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stderr=inout, stdin=inout) + p.wait() + + def test_wait_when_sigchild_ignored(self): + # NOTE: sigchild_ignore.py may not be an effective test on all OSes. + sigchild_ignore = support.findfile("sigchild_ignore.py", + subdir="subprocessdata") + p = subprocess.Popen([sys.executable, sigchild_ignore], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" + " non-zero with this error:\n%s" % + stderr.decode('utf-8')) + + def test_select_unbuffered(self): + # Issue #11459: bufsize=0 should really set the pipes as + # unbuffered (and therefore let select() work properly). + select = support.import_module("select") + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple")'], + stdout=subprocess.PIPE, + bufsize=0) + f = p.stdout + self.addCleanup(f.close) + try: + self.assertEqual(f.read(4), b"appl") + self.assertIn(f, select.select([f], [], [], 0.0)[0]) + finally: + p.wait() + + def test_zombie_fast_process_del(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, it wouldn't be added to subprocess._active, and would + # remain a zombie. + # spawn a Popen, and delete its reference before it exits + p = subprocess.Popen([sys.executable, "-c", + 'import sys, time;' + 'time.sleep(0.2)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + support.gc_collect() + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + def test_leak_fast_process_del_killed(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, and the process got killed by a signal, it would never + # be removed from subprocess._active, which triggered a FD and memory + # leak. + # spawn a Popen, delete its reference and kill it + p = subprocess.Popen([sys.executable, "-c", + 'import time;' + 'time.sleep(3)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + del p + support.gc_collect() + os.kill(pid, signal.SIGKILL) + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + # let some time for the process to exit, and create a new Popen: this + # should trigger the wait() of p + time.sleep(0.2) + with self.assertRaises(OSError) as c: + with subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + # p should have been wait()ed on, and removed from the _active list + self.assertRaises(OSError, os.waitpid, pid, 0) + self.assertNotIn(ident, [id(o) for o in subprocess._active]) + + def test_close_fds_after_preexec(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # this FD is used as dup2() target by preexec_fn, and should be closed + # in the child process + fd = os.dup(1) + self.addCleanup(os.close, fd) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + preexec_fn=lambda: os.dup2(1, fd)) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + + self.assertNotIn(fd, remaining_fds) + + @support.cpython_only + def test_fork_exec(self): + # Issue #22290: fork_exec() must not crash on memory allocation failure + # or other errors + import _posixsubprocess + gc_enabled = gc.isenabled() + try: + # Use a preexec function and enable the garbage collector + # to force fork_exec() to re-enable the garbage collector + # on error. + func = lambda: None + gc.enable() + + for args, exe_list, cwd, env_list in ( + (123, [b"exe"], None, [b"env"]), + ([b"arg"], 123, None, [b"env"]), + ([b"arg"], [b"exe"], 123, [b"env"]), + ([b"arg"], [b"exe"], None, 123), + ): + with self.assertRaises(TypeError): + _posixsubprocess.fork_exec( + args, exe_list, + True, [], cwd, env_list, + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, func) + finally: + if not gc_enabled: + gc.disable() + + @support.cpython_only + def test_fork_exec_sorted_fd_sanity_check(self): + # Issue #23564: sanity check the fork_exec() fds_to_keep sanity check. + import _posixsubprocess + gc_enabled = gc.isenabled() + try: + gc.enable() + + for fds_to_keep in ( + (-1, 2, 3, 4, 5), # Negative number. + ('str', 4), # Not an int. + (18, 23, 42, 2**63), # Out of range. + (5, 4), # Not sorted. + (6, 7, 7, 8), # Duplicate. + ): + with self.assertRaises( + ValueError, + msg='fds_to_keep={}'.format(fds_to_keep)) as c: + _posixsubprocess.fork_exec( + [b"false"], [b"false"], + True, fds_to_keep, None, [b"env"], + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, None) + self.assertIn('fds_to_keep', str(c.exception)) + finally: + if not gc_enabled: + gc.disable() + + def test_communicate_BrokenPipeError_stdin_close(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + proc.communicate() # Should swallow BrokenPipeError from close. + mock_proc_stdin.close.assert_called_with() + + def test_communicate_BrokenPipeError_stdin_write(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.write.side_effect = BrokenPipeError + proc.communicate(b'stuff') # Should swallow the BrokenPipeError. + mock_proc_stdin.write.assert_called_once_with(b'stuff') + mock_proc_stdin.close.assert_called_once_with() + + def test_communicate_BrokenPipeError_stdin_flush(self): + # Setting stdin and stdout forces the ._communicate() code path. + # python -h exits faster than python -c pass (but spams stdout). + proc = subprocess.Popen([sys.executable, '-h'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin, \ + open(os.devnull, 'wb') as dev_null: + mock_proc_stdin.flush.side_effect = BrokenPipeError + # because _communicate registers a selector using proc.stdin... + mock_proc_stdin.fileno.return_value = dev_null.fileno() + # _communicate() should swallow BrokenPipeError from flush. + proc.communicate(b'stuff') + mock_proc_stdin.flush.assert_called_once_with() + + def test_communicate_BrokenPipeError_stdin_close_with_timeout(self): + # Setting stdin and stdout forces the ._communicate() code path. + # python -h exits faster than python -c pass (but spams stdout). + proc = subprocess.Popen([sys.executable, '-h'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + # _communicate() should swallow BrokenPipeError from close. + proc.communicate(timeout=999) + mock_proc_stdin.close.assert_called_once_with() + + +@unittest.skipUnless(mswindows, "Windows specific tests") +class Win32ProcessTestCase(BaseTestCase): + + def test_startupinfo(self): + # startupinfo argument + # We uses hardcoded constants, because we do not want to + # depend on win32all. + STARTF_USESHOWWINDOW = 1 + SW_MAXIMIZE = 3 + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = STARTF_USESHOWWINDOW + startupinfo.wShowWindow = SW_MAXIMIZE + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_creationflags(self): + # creationflags argument + CREATE_NEW_CONSOLE = 16 + sys.stderr.write(" a DOS box should flash briefly ...\n") + subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + preexec_fn=lambda: 1) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + stdout=subprocess.PIPE, + close_fds=True) + + def test_close_fds(self): + # close file descriptors + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"], + close_fds=True) + self.assertEqual(rc, 47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen(["set"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn(b"physalis", p.stdout.read()) + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv) + self.addCleanup(p.stdout.close) + self.assertIn(b"physalis", p.stdout.read()) + + def test_call_string(self): + # call() function with string argument on Windows + rc = subprocess.call(sys.executable + + ' -c "import sys; sys.exit(47)"') + self.assertEqual(rc, 47) + + def _kill_process(self, method, *args): + # Some win32 buildbot raises EOFError if stdin is inherited + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + returncode = p.wait() + self.assertNotEqual(returncode, 0) + + def _kill_dead_process(self, method, *args): + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + sys.exit(42) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() + self.assertEqual(rc, 42) + + def test_send_signal(self): + self._kill_process('send_signal', signal.SIGTERM) + + def test_kill(self): + self._kill_process('kill') + + def test_terminate(self): + self._kill_process('terminate') + + def test_send_signal_dead(self): + self._kill_dead_process('send_signal', signal.SIGTERM) + + def test_kill_dead(self): + self._kill_dead_process('kill') + + def test_terminate_dead(self): + self._kill_dead_process('terminate') + +class MiscTests(unittest.TestCase): + def test_getoutput(self): + self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') + self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), + (0, 'xyzzy')) + + # we use mkdtemp in the next line to create an empty directory + # under our exclusive control; from that, we can invent a pathname + # that we _know_ won't exist. This is guaranteed to fail. + dir = None + try: + dir = tempfile.mkdtemp() + name = os.path.join(dir, "foo") + status, output = subprocess.getstatusoutput( + ("type " if mswindows else "cat ") + name) + self.assertNotEqual(status, 0) + finally: + if dir is not None: + os.rmdir(dir) + + def test__all__(self): + """Ensure that __all__ is populated properly.""" + # STARTUPINFO added to __all__ in 3.6 + intentionally_excluded = {"list2cmdline", "STARTUPINFO", "Handle"} + exported = set(subprocess.__all__) + possible_exports = set() + import types + for name, value in subprocess.__dict__.items(): + if name.startswith('_'): + continue + if isinstance(value, (types.ModuleType,)): + continue + possible_exports.add(name) + self.assertEqual(exported, possible_exports - intentionally_excluded) + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class ProcessTestCaseNoPoll(ProcessTestCase): + def setUp(self): + self.orig_selector = subprocess._PopenSelector + subprocess._PopenSelector = selectors.SelectSelector + ProcessTestCase.setUp(self) + + def tearDown(self): + subprocess._PopenSelector = self.orig_selector + ProcessTestCase.tearDown(self) + + +@unittest.skipUnless(mswindows, "Windows-specific tests") +class CommandsWithSpaces (BaseTestCase): + + def setUp(self): + super().setUp() + f, fname = tempfile.mkstemp(".py", "te st") + self.fname = fname.lower () + os.write(f, b"import sys;" + b"sys.stdout.write('%d %s' % (len(sys.argv), [a.lower () for a in sys.argv]))" + ) + os.close(f) + + def tearDown(self): + os.remove(self.fname) + super().tearDown() + + def with_spaces(self, *args, **kwargs): + kwargs['stdout'] = subprocess.PIPE + p = subprocess.Popen(*args, **kwargs) + self.addCleanup(p.stdout.close) + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) + + def test_shell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd"), shell=1) + + def test_shell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) + + def test_noshell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd")) + + def test_noshell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"]) + + +class ContextManagerTests(BaseTestCase): + + def test_pipe(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write('stdout');" + "sys.stderr.write('stderr');"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + self.assertEqual(proc.stdout.read(), b"stdout") + self.assertStderrEqual(proc.stderr.read(), b"stderr") + + self.assertTrue(proc.stdout.closed) + self.assertTrue(proc.stderr.closed) + + def test_returncode(self): + with subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(100)"]) as proc: + pass + # __exit__ calls wait(), so the returncode should be set + self.assertEqual(proc.returncode, 100) + + def test_communicate_stdin(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.exit(sys.stdin.read() == 'context')"], + stdin=subprocess.PIPE) as proc: + proc.communicate(b"context") + self.assertEqual(proc.returncode, 1) + + def test_invalid_args(self): + with self.assertRaises(FileNotFoundError) as c: + with subprocess.Popen(['nonexisting_i_hope'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + + def test_broken_pipe_cleanup(self): + """Broken pipe error should not prevent wait() (Issue 21619)""" + proc = subprocess.Popen([sys.executable, '-c', 'pass'], + stdin=subprocess.PIPE, + bufsize=support.PIPE_MAX_SIZE*2) + proc = proc.__enter__() + # Prepare to send enough data to overflow any OS pipe buffering and + # guarantee a broken pipe error. Data is held in BufferedWriter + # buffer until closed. + proc.stdin.write(b'x' * support.PIPE_MAX_SIZE) + self.assertIsNone(proc.returncode) + # EPIPE expected under POSIX; EINVAL under Windows + self.assertRaises(OSError, proc.__exit__, None, None, None) + self.assertEqual(proc.returncode, 0) + self.assertTrue(proc.stdin.closed) + + +def test_main(): + unit_tests = (ProcessTestCase, + POSIXProcessTestCase, + Win32ProcessTestCase, + MiscTests, + ProcessTestCaseNoPoll, + CommandsWithSpaces, + ContextManagerTests, + RunFuncTestCase, + ) + + support.run_unittest(*unit_tests) + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_telnetlib.py b/src/greentest/3.5pypy/test_telnetlib.py new file mode 100644 index 0000000..4122931 --- /dev/null +++ b/src/greentest/3.5pypy/test_telnetlib.py @@ -0,0 +1,398 @@ +import socket +import selectors +import telnetlib +import time +import contextlib + +from test import support +import unittest +threading = support.import_module('threading') + +HOST = support.HOST + +def server(evt, serv): + serv.listen() + evt.set() + try: + conn, addr = serv.accept() + conn.close() + except socket.timeout: + pass + finally: + serv.close() + +class GeneralTests(unittest.TestCase): + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(60) # Safety net. Look issue 11812 + self.port = support.bind_port(self.sock) + self.thread = threading.Thread(target=server, args=(self.evt,self.sock)) + self.thread.setDaemon(True) + self.thread.start() + self.evt.wait() + + def tearDown(self): + self.thread.join() + del self.thread # Clear out any dangling Thread objects. + + def testBasic(self): + # connects + telnet = telnetlib.Telnet(HOST, self.port) + telnet.sock.close() + + def testTimeoutDefault(self): + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + telnet = telnetlib.Telnet(HOST, self.port) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testTimeoutNone(self): + # None, having other default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + telnet = telnetlib.Telnet(HOST, self.port, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertTrue(telnet.sock.gettimeout() is None) + telnet.sock.close() + + def testTimeoutValue(self): + telnet = telnetlib.Telnet(HOST, self.port, timeout=30) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testTimeoutOpen(self): + telnet = telnetlib.Telnet() + telnet.open(HOST, self.port, timeout=30) + self.assertEqual(telnet.sock.gettimeout(), 30) + telnet.sock.close() + + def testGetters(self): + # Test telnet getter methods + telnet = telnetlib.Telnet(HOST, self.port, timeout=30) + t_sock = telnet.sock + self.assertEqual(telnet.get_socket(), t_sock) + self.assertEqual(telnet.fileno(), t_sock.fileno()) + telnet.sock.close() + +class SocketStub(object): + ''' a socket proxy that re-defines sendall() ''' + def __init__(self, reads=()): + self.reads = list(reads) # Intentionally make a copy. + self.writes = [] + self.block = False + def sendall(self, data): + self.writes.append(data) + def recv(self, size): + out = b'' + while self.reads and len(out) < size: + out += self.reads.pop(0) + if len(out) > size: + self.reads.insert(0, out[size:]) + out = out[:size] + return out + +class TelnetAlike(telnetlib.Telnet): + def fileno(self): + raise NotImplementedError() + def close(self): pass + def sock_avail(self): + return (not self.sock.block) + def msg(self, msg, *args): + with support.captured_stdout() as out: + telnetlib.Telnet.msg(self, msg, *args) + self._messages += out.getvalue() + return + +class MockSelector(selectors.BaseSelector): + + def __init__(self): + self.keys = {} + + @property + def resolution(self): + return 1e-3 + + def register(self, fileobj, events, data=None): + key = selectors.SelectorKey(fileobj, 0, events, data) + self.keys[fileobj] = key + return key + + def unregister(self, fileobj): + return self.keys.pop(fileobj) + + def select(self, timeout=None): + block = False + for fileobj in self.keys: + if isinstance(fileobj, TelnetAlike): + block = fileobj.sock.block + break + if block: + return [] + else: + return [(key, key.events) for key in self.keys.values()] + + def get_map(self): + return self.keys + + +@contextlib.contextmanager +def test_socket(reads): + def new_conn(*ignored): + return SocketStub(reads) + try: + old_conn = socket.create_connection + socket.create_connection = new_conn + yield None + finally: + socket.create_connection = old_conn + return + +def test_telnet(reads=(), cls=TelnetAlike): + ''' return a telnetlib.Telnet object that uses a SocketStub with + reads queued up to be read ''' + for x in reads: + assert type(x) is bytes, x + with test_socket(reads): + telnet = cls('dummy', 0) + telnet._messages = '' # debuglevel output + return telnet + +class ExpectAndReadTestCase(unittest.TestCase): + def setUp(self): + self.old_selector = telnetlib._TelnetSelector + telnetlib._TelnetSelector = MockSelector + def tearDown(self): + telnetlib._TelnetSelector = self.old_selector + +class ReadTests(ExpectAndReadTestCase): + def test_read_until(self): + """ + read_until(expected, timeout=None) + test the blocking version of read_util + """ + want = [b'xxxmatchyyy'] + telnet = test_telnet(want) + data = telnet.read_until(b'match') + self.assertEqual(data, b'xxxmatch', msg=(telnet.cookedq, telnet.rawq, telnet.sock.reads)) + + reads = [b'x' * 50, b'match', b'y' * 50] + expect = b''.join(reads[:-1]) + telnet = test_telnet(reads) + data = telnet.read_until(b'match') + self.assertEqual(data, expect) + + + def test_read_all(self): + """ + read_all() + Read all data until EOF; may block. + """ + reads = [b'x' * 500, b'y' * 500, b'z' * 500] + expect = b''.join(reads) + telnet = test_telnet(reads) + data = telnet.read_all() + self.assertEqual(data, expect) + return + + def test_read_some(self): + """ + read_some() + Read at least one byte or EOF; may block. + """ + # test 'at least one byte' + telnet = test_telnet([b'x' * 500]) + data = telnet.read_some() + self.assertTrue(len(data) >= 1) + # test EOF + telnet = test_telnet() + data = telnet.read_some() + self.assertEqual(b'', data) + + def _read_eager(self, func_name): + """ + read_*_eager() + Read all data available already queued or on the socket, + without blocking. + """ + want = b'x' * 100 + telnet = test_telnet([want]) + func = getattr(telnet, func_name) + telnet.sock.block = True + self.assertEqual(b'', func()) + telnet.sock.block = False + data = b'' + while True: + try: + data += func() + except EOFError: + break + self.assertEqual(data, want) + + def test_read_eager(self): + # read_eager and read_very_eager make the same guarantees + # (they behave differently but we only test the guarantees) + self._read_eager('read_eager') + self._read_eager('read_very_eager') + # NB -- we need to test the IAC block which is mentioned in the + # docstring but not in the module docs + + def read_very_lazy(self): + want = b'x' * 100 + telnet = test_telnet([want]) + self.assertEqual(b'', telnet.read_very_lazy()) + while telnet.sock.reads: + telnet.fill_rawq() + data = telnet.read_very_lazy() + self.assertEqual(want, data) + self.assertRaises(EOFError, telnet.read_very_lazy) + + def test_read_lazy(self): + want = b'x' * 100 + telnet = test_telnet([want]) + self.assertEqual(b'', telnet.read_lazy()) + data = b'' + while True: + try: + read_data = telnet.read_lazy() + data += read_data + if not read_data: + telnet.fill_rawq() + except EOFError: + break + self.assertTrue(want.startswith(data)) + self.assertEqual(data, want) + +class nego_collector(object): + def __init__(self, sb_getter=None): + self.seen = b'' + self.sb_getter = sb_getter + self.sb_seen = b'' + + def do_nego(self, sock, cmd, opt): + self.seen += cmd + opt + if cmd == tl.SE and self.sb_getter: + sb_data = self.sb_getter() + self.sb_seen += sb_data + +tl = telnetlib + +class WriteTests(unittest.TestCase): + '''The only thing that write does is replace each tl.IAC for + tl.IAC+tl.IAC''' + + def test_write(self): + data_sample = [b'data sample without IAC', + b'data sample with' + tl.IAC + b' one IAC', + b'a few' + tl.IAC + tl.IAC + b' iacs' + tl.IAC, + tl.IAC, + b''] + for data in data_sample: + telnet = test_telnet() + telnet.write(data) + written = b''.join(telnet.sock.writes) + self.assertEqual(data.replace(tl.IAC,tl.IAC+tl.IAC), written) + +class OptionTests(unittest.TestCase): + # RFC 854 commands + cmds = [tl.AO, tl.AYT, tl.BRK, tl.EC, tl.EL, tl.GA, tl.IP, tl.NOP] + + def _test_command(self, data): + """ helper for testing IAC + cmd """ + telnet = test_telnet(data) + data_len = len(b''.join(data)) + nego = nego_collector() + telnet.set_option_negotiation_callback(nego.do_nego) + txt = telnet.read_all() + cmd = nego.seen + self.assertTrue(len(cmd) > 0) # we expect at least one command + self.assertIn(cmd[:1], self.cmds) + self.assertEqual(cmd[1:2], tl.NOOPT) + self.assertEqual(data_len, len(txt + cmd)) + nego.sb_getter = None # break the nego => telnet cycle + + def test_IAC_commands(self): + for cmd in self.cmds: + self._test_command([tl.IAC, cmd]) + self._test_command([b'x' * 100, tl.IAC, cmd, b'y'*100]) + self._test_command([b'x' * 10, tl.IAC, cmd, b'y'*10]) + # all at once + self._test_command([tl.IAC + cmd for (cmd) in self.cmds]) + + def test_SB_commands(self): + # RFC 855, subnegotiations portion + send = [tl.IAC + tl.SB + tl.IAC + tl.SE, + tl.IAC + tl.SB + tl.IAC + tl.IAC + tl.IAC + tl.SE, + tl.IAC + tl.SB + tl.IAC + tl.IAC + b'aa' + tl.IAC + tl.SE, + tl.IAC + tl.SB + b'bb' + tl.IAC + tl.IAC + tl.IAC + tl.SE, + tl.IAC + tl.SB + b'cc' + tl.IAC + tl.IAC + b'dd' + tl.IAC + tl.SE, + ] + telnet = test_telnet(send) + nego = nego_collector(telnet.read_sb_data) + telnet.set_option_negotiation_callback(nego.do_nego) + txt = telnet.read_all() + self.assertEqual(txt, b'') + want_sb_data = tl.IAC + tl.IAC + b'aabb' + tl.IAC + b'cc' + tl.IAC + b'dd' + self.assertEqual(nego.sb_seen, want_sb_data) + self.assertEqual(b'', telnet.read_sb_data()) + nego.sb_getter = None # break the nego => telnet cycle + + def test_debuglevel_reads(self): + # test all the various places that self.msg(...) is called + given_a_expect_b = [ + # Telnet.fill_rawq + (b'a', ": recv b''\n"), + # Telnet.process_rawq + (tl.IAC + bytes([88]), ": IAC 88 not recognized\n"), + (tl.IAC + tl.DO + bytes([1]), ": IAC DO 1\n"), + (tl.IAC + tl.DONT + bytes([1]), ": IAC DONT 1\n"), + (tl.IAC + tl.WILL + bytes([1]), ": IAC WILL 1\n"), + (tl.IAC + tl.WONT + bytes([1]), ": IAC WONT 1\n"), + ] + for a, b in given_a_expect_b: + telnet = test_telnet([a]) + telnet.set_debuglevel(1) + txt = telnet.read_all() + self.assertIn(b, telnet._messages) + return + + def test_debuglevel_write(self): + telnet = test_telnet() + telnet.set_debuglevel(1) + telnet.write(b'xxx') + expected = "send b'xxx'\n" + self.assertIn(expected, telnet._messages) + + def test_debug_accepts_str_port(self): + # Issue 10695 + with test_socket([]): + telnet = TelnetAlike('dummy', '0') + telnet._messages = '' + telnet.set_debuglevel(1) + telnet.msg('test') + self.assertRegex(telnet._messages, r'0.*test') + + +class ExpectTests(ExpectAndReadTestCase): + def test_expect(self): + """ + expect(expected, [timeout]) + Read until the expected string has been seen, or a timeout is + hit (default is no timeout); may block. + """ + want = [b'x' * 10, b'match', b'y' * 10] + telnet = test_telnet(want) + (_,_,data) = telnet.expect([b'match']) + self.assertEqual(data, b''.join(want[:-1])) + + +if __name__ == '__main__': + import unittest + unittest.main() diff --git a/src/greentest/3.5pypy/test_thread.py b/src/greentest/3.5pypy/test_thread.py new file mode 100644 index 0000000..a5380ba --- /dev/null +++ b/src/greentest/3.5pypy/test_thread.py @@ -0,0 +1,264 @@ +import os +import unittest +import random +from test import support +thread = support.import_module('_thread') +import time +import sys +import weakref + +from test import lock_tests + +NUMTASKS = 10 +NUMTRIPS = 3 + +_print_mutex = thread.allocate_lock() + +def verbose_print(arg): + """Helper function for printing out debugging output.""" + if support.verbose: + with _print_mutex: + print(arg) + +class BasicThreadTest(unittest.TestCase): + + def setUp(self): + self.done_mutex = thread.allocate_lock() + self.done_mutex.acquire() + self.running_mutex = thread.allocate_lock() + self.random_mutex = thread.allocate_lock() + self.created = 0 + self.running = 0 + self.next_ident = 0 + + +class ThreadRunningTests(BasicThreadTest): + + def newtask(self): + with self.running_mutex: + self.next_ident += 1 + verbose_print("creating task %s" % self.next_ident) + thread.start_new_thread(self.task, (self.next_ident,)) + self.created += 1 + self.running += 1 + + def task(self, ident): + with self.random_mutex: + delay = random.random() / 10000.0 + verbose_print("task %s will run for %sus" % (ident, round(delay*1e6))) + time.sleep(delay) + verbose_print("task %s done" % ident) + with self.running_mutex: + self.running -= 1 + if self.created == NUMTASKS and self.running == 0: + self.done_mutex.release() + + def test_starting_threads(self): + # Basic test for thread creation. + for i in range(NUMTASKS): + self.newtask() + verbose_print("waiting for tasks to complete...") + self.done_mutex.acquire() + verbose_print("all tasks done") + + def test_stack_size(self): + # Various stack size tests. + self.assertEqual(thread.stack_size(), 0, "initial stack size is not 0") + + thread.stack_size(0) + self.assertEqual(thread.stack_size(), 0, "stack_size not reset to default") + + @unittest.skipIf(os.name not in ("nt", "posix"), 'test meant for nt and posix') + def test_nt_and_posix_stack_size(self): + try: + thread.stack_size(4096) + except ValueError: + verbose_print("caught expected ValueError setting " + "stack_size(4096)") + except thread.error: + self.skipTest("platform does not support changing thread stack " + "size") + + fail_msg = "stack_size(%d) failed - should succeed" + for tss in (262144, 0x100000, 0): + thread.stack_size(tss) + self.assertEqual(thread.stack_size(), tss, fail_msg % tss) + verbose_print("successfully set stack_size(%d)" % tss) + + for tss in (262144, 0x100000): + verbose_print("trying stack_size = (%d)" % tss) + self.next_ident = 0 + self.created = 0 + for i in range(NUMTASKS): + self.newtask() + + verbose_print("waiting for all tasks to complete") + self.done_mutex.acquire() + verbose_print("all tasks done") + + thread.stack_size(0) + + def test__count(self): + # Test the _count() function. + orig = thread._count() + mut = thread.allocate_lock() + mut.acquire() + started = [] + def task(): + started.append(None) + mut.acquire() + mut.release() + thread.start_new_thread(task, ()) + while not started: + time.sleep(0.01) + self.assertEqual(thread._count(), orig + 1) + # Allow the task to finish. + mut.release() + # The only reliable way to be sure that the thread ended from the + # interpreter's point of view is to wait for the function object to be + # destroyed. + done = [] + wr = weakref.ref(task, lambda _: done.append(None)) + del task + while not done: + time.sleep(0.01) + support.gc_collect() + self.assertEqual(thread._count(), orig) + + def test_save_exception_state_on_error(self): + # See issue #14474 + def task(): + started.release() + raise SyntaxError + def mywrite(self, *args): + try: + raise ValueError + except ValueError: + pass + real_write(self, *args) + c = thread._count() + started = thread.allocate_lock() + with support.captured_output("stderr") as stderr: + real_write = stderr.write + stderr.write = mywrite + started.acquire() + thread.start_new_thread(task, ()) + started.acquire() + while thread._count() > c: + time.sleep(0.01) + self.assertIn("Traceback", stderr.getvalue()) + + +class Barrier: + def __init__(self, num_threads): + self.num_threads = num_threads + self.waiting = 0 + self.checkin_mutex = thread.allocate_lock() + self.checkout_mutex = thread.allocate_lock() + self.checkout_mutex.acquire() + + def enter(self): + self.checkin_mutex.acquire() + self.waiting = self.waiting + 1 + if self.waiting == self.num_threads: + self.waiting = self.num_threads - 1 + self.checkout_mutex.release() + return + self.checkin_mutex.release() + + self.checkout_mutex.acquire() + self.waiting = self.waiting - 1 + if self.waiting == 0: + self.checkin_mutex.release() + return + self.checkout_mutex.release() + + +class BarrierTest(BasicThreadTest): + + def test_barrier(self): + self.bar = Barrier(NUMTASKS) + self.running = NUMTASKS + for i in range(NUMTASKS): + thread.start_new_thread(self.task2, (i,)) + verbose_print("waiting for tasks to end") + self.done_mutex.acquire() + verbose_print("tasks done") + + def task2(self, ident): + for i in range(NUMTRIPS): + if ident == 0: + # give it a good chance to enter the next + # barrier before the others are all out + # of the current one + delay = 0 + else: + with self.random_mutex: + delay = random.random() / 10000.0 + verbose_print("task %s will run for %sus" % + (ident, round(delay * 1e6))) + time.sleep(delay) + verbose_print("task %s entering %s" % (ident, i)) + self.bar.enter() + verbose_print("task %s leaving barrier" % ident) + with self.running_mutex: + self.running -= 1 + # Must release mutex before releasing done, else the main thread can + # exit and set mutex to None as part of global teardown; then + # mutex.release() raises AttributeError. + finished = self.running == 0 + if finished: + self.done_mutex.release() + +class LockTests(lock_tests.LockTests): + locktype = thread.allocate_lock + + @unittest.skip("not on gevent") + def test_locked_repr(self): + pass + + @unittest.skip("not on gevent") + def test_repr(self): + pass + +class TestForkInThread(unittest.TestCase): + def setUp(self): + self.read_fd, self.write_fd = os.pipe() + + @unittest.skipIf(sys.platform.startswith('win'), + "This test is only appropriate for POSIX-like systems.") + @support.reap_threads + def test_forkinthread(self): + def thread1(): + try: + pid = os.fork() # fork in a thread + except RuntimeError: + os._exit(1) # exit the child + + if pid == 0: # child + try: + os.close(self.read_fd) + os.write(self.write_fd, b"OK") + finally: + os._exit(0) + else: # parent + os.close(self.write_fd) + + thread.start_new_thread(thread1, ()) + self.assertEqual(os.read(self.read_fd, 2), b"OK", + "Unable to fork() in thread") + + def tearDown(self): + try: + os.close(self.read_fd) + except OSError: + pass + + try: + os.close(self.write_fd) + except OSError: + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_threading.py b/src/greentest/3.5pypy/test_threading.py new file mode 100644 index 0000000..c55642a --- /dev/null +++ b/src/greentest/3.5pypy/test_threading.py @@ -0,0 +1,1151 @@ +""" +Tests for the threading module. +""" + +import test.support +from test.support import (verbose, import_module, cpython_only, + requires_type_collecting) +from test.support.script_helper import assert_python_ok, assert_python_failure + +import random +import re +import sys +_thread = import_module('_thread') +threading = import_module('threading') +import time +import unittest +import weakref +import os +import subprocess + +from test import lock_tests + + +# Between fork() and exec(), only async-safe functions are allowed (issues +# #12316 and #11870), and fork() from a worker thread is known to trigger +# problems with some operating systems (issue #3863): skip problematic tests +# on platforms known to behave badly. +platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', + 'hp-ux11') + + +# A trivial mutable counter. +class Counter(object): + def __init__(self): + self.value = 0 + def inc(self): + self.value += 1 + def dec(self): + self.value -= 1 + def get(self): + return self.value + +class TestThread(threading.Thread): + def __init__(self, name, testcase, sema, mutex, nrunning): + threading.Thread.__init__(self, name=name) + self.testcase = testcase + self.sema = sema + self.mutex = mutex + self.nrunning = nrunning + + def run(self): + delay = random.random() / 10000.0 + if verbose: + print('task %s will run for %.1f usec' % + (self.name, delay * 1e6)) + + with self.sema: + with self.mutex: + self.nrunning.inc() + if verbose: + print(self.nrunning.get(), 'tasks are running') + self.testcase.assertLessEqual(self.nrunning.get(), 3) + + time.sleep(delay) + if verbose: + print('task', self.name, 'done') + + with self.mutex: + self.nrunning.dec() + self.testcase.assertGreaterEqual(self.nrunning.get(), 0) + if verbose: + print('%s is finished. %d tasks are running' % + (self.name, self.nrunning.get())) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test.support.threading_setup() + + def tearDown(self): + test.support.threading_cleanup(*self._threads) + test.support.reap_children() + + +class ThreadTests(BaseTestCase): + + # Create a bunch of threads, let each do some work, wait until all are + # done. + def test_various_ops(self): + # This takes about n/3 seconds to run (about n/3 clumps of tasks, + # times about 1 second per clump). + NUMTASKS = 10 + + # no more than 3 of the 10 can run at once + sema = threading.BoundedSemaphore(value=3) + mutex = threading.RLock() + numrunning = Counter() + + threads = [] + + for i in range(NUMTASKS): + t = TestThread(""%i, self, sema, mutex, numrunning) + threads.append(t) + self.assertIsNone(t.ident) + self.assertRegex(repr(t), r'^$') + t.start() + + if verbose: + print('waiting for all tasks to complete') + for t in threads: + t.join() + self.assertFalse(t.is_alive()) + self.assertNotEqual(t.ident, 0) + self.assertIsNotNone(t.ident) + self.assertRegex(repr(t), r'^$') + if verbose: + print('all tasks done') + self.assertEqual(numrunning.get(), 0) + + def test_ident_of_no_threading_threads(self): + # The ident still must work for the main thread and dummy threads. + self.assertIsNotNone(threading.currentThread().ident) + def f(): + ident.append(threading.currentThread().ident) + done.set() + done = threading.Event() + ident = [] + _thread.start_new_thread(f, ()) + done.wait() + self.assertIsNotNone(ident[0]) + # Kill the "immortal" _DummyThread + del threading._active[ident[0]] + + # run with a small(ish) thread stack size (256kB) + def test_various_ops_small_stack(self): + if verbose: + print('with 256kB thread stack size...') + try: + threading.stack_size(262144) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1MB) + def test_various_ops_large_stack(self): + if verbose: + print('with 1MB thread stack size...') + try: + threading.stack_size(0x100000) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + def test_foreign_thread(self): + # Check that a "foreign" thread can use the threading module. + def f(mutex): + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + threading.current_thread() + mutex.release() + + mutex = threading.Lock() + mutex.acquire() + tid = _thread.start_new_thread(f, (mutex,)) + # Wait for the thread to finish. + mutex.acquire() + self.assertIn(tid, threading._active) + self.assertIsInstance(threading._active[tid], threading._DummyThread) + del threading._active[tid] + + # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) + # exposed at the Python level. This test relies on ctypes to get at it. + @test.support.cpython_only + def test_PyThreadState_SetAsyncExc(self): + ctypes = import_module("ctypes") + + set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc + + class AsyncExc(Exception): + pass + + exception = ctypes.py_object(AsyncExc) + + # First check it works when setting the exception from the same thread. + tid = threading.get_ident() + + try: + result = set_async_exc(ctypes.c_long(tid), exception) + # The exception is async, so we might have to keep the VM busy until + # it notices. + while True: + pass + except AsyncExc: + pass + else: + # This code is unreachable but it reflects the intent. If we wanted + # to be smarter the above loop wouldn't be infinite. + self.fail("AsyncExc not raised") + try: + self.assertEqual(result, 1) # one thread state modified + except UnboundLocalError: + # The exception was raised too quickly for us to get the result. + pass + + # `worker_started` is set by the thread when it's inside a try/except + # block waiting to catch the asynchronously set AsyncExc exception. + # `worker_saw_exception` is set by the thread upon catching that + # exception. + worker_started = threading.Event() + worker_saw_exception = threading.Event() + + class Worker(threading.Thread): + def run(self): + self.id = threading.get_ident() + self.finished = False + + try: + while True: + worker_started.set() + time.sleep(0.1) + except AsyncExc: + self.finished = True + worker_saw_exception.set() + + t = Worker() + t.daemon = True # so if this fails, we don't hang Python at shutdown + t.start() + if verbose: + print(" started worker thread") + + # Try a thread id that doesn't make sense. + if verbose: + print(" trying nonsensical thread id") + result = set_async_exc(ctypes.c_long(-1), exception) + self.assertEqual(result, 0) # no thread states modified + + # Now raise an exception in the worker thread. + if verbose: + print(" waiting for worker thread to get started") + ret = worker_started.wait() + self.assertTrue(ret) + if verbose: + print(" verifying worker hasn't exited") + self.assertFalse(t.finished) + if verbose: + print(" attempting to raise asynch exception in worker") + result = set_async_exc(ctypes.c_long(t.id), exception) + self.assertEqual(result, 1) # one thread state modified + if verbose: + print(" waiting for worker to say it caught the exception") + worker_saw_exception.wait(timeout=10) + self.assertTrue(t.finished) + if verbose: + print(" all OK -- joining worker") + if t.finished: + t.join() + # else the thread is still running, and we have no way to kill it + + def test_limbo_cleanup(self): + # Issue 7481: Failure to start thread should cleanup the limbo map. + def fail_new_thread(*args): + raise threading.ThreadError() + _start_new_thread = threading._start_new_thread + threading._start_new_thread = fail_new_thread + try: + t = threading.Thread(target=lambda: None) + self.assertRaises(threading.ThreadError, t.start) + self.assertFalse( + t in threading._limbo, + "Failed to cleanup _limbo map on failure of Thread.start().") + finally: + threading._start_new_thread = _start_new_thread + + @test.support.cpython_only + def test_finalize_runnning_thread(self): + # Issue 1402: the PyGILState_Ensure / _Release functions may be called + # very late on python exit: on deallocation of a running thread for + # example. + import_module("ctypes") + + rc, out, err = assert_python_failure("-c", """if 1: + import ctypes, sys, time, _thread + + # This lock is used as a simple event variable. + ready = _thread.allocate_lock() + ready.acquire() + + # Module globals are cleared before __del__ is run + # So we save the functions in class dict + class C: + ensure = ctypes.pythonapi.PyGILState_Ensure + release = ctypes.pythonapi.PyGILState_Release + def __del__(self): + state = self.ensure() + self.release(state) + + def waitingThread(): + x = C() + ready.release() + time.sleep(100) + + _thread.start_new_thread(waitingThread, ()) + ready.acquire() # Be sure the other thread is waiting. + sys.exit(42) + """) + self.assertEqual(rc, 42) + + def test_finalize_with_trace(self): + # Issue1733757 + # Avoid a deadlock when sys.settrace steps into threading._shutdown + assert_python_ok("-c", """if 1: + import sys, threading + + # A deadlock-killer, to prevent the + # testsuite to hang forever + def killer(): + import os, time + time.sleep(2) + print('program blocked; aborting') + os._exit(2) + t = threading.Thread(target=killer) + t.daemon = True + t.start() + + # This is the trace function + def func(frame, event, arg): + threading.current_thread() + return func + + sys.settrace(func) + """) + + def test_join_nondaemon_on_shutdown(self): + # Issue 1722344 + # Raising SystemExit skipped threading._shutdown + rc, out, err = assert_python_ok("-c", """if 1: + import threading + from time import sleep + + def child(): + sleep(1) + # As a non-daemon thread we SHOULD wake up and nothing + # should be torn down yet + print("Woke up, sleep function is:", sleep) + + threading.Thread(target=child).start() + raise SystemExit + """) + self.assertEqual(out.strip(), + b"Woke up, sleep function is: ") + self.assertEqual(err, b"") + + def test_enumerate_after_join(self): + # Try hard to trigger #1703448: a thread is still returned in + # threading.enumerate() after it has been join()ed. + enum = threading.enumerate + newgil = hasattr(sys, 'getswitchinterval') + if newgil: + geti, seti = sys.getswitchinterval, sys.setswitchinterval + else: + geti, seti = sys.getcheckinterval, sys.setcheckinterval + old_interval = geti() + try: + for i in range(1, 100): + seti(i * 0.0002 if newgil else i // 5) + t = threading.Thread(target=lambda: None) + t.start() + t.join() + l = enum() + self.assertNotIn(t, l, + "#1703448 triggered after %d trials: %s" % (i, l)) + finally: + seti(old_interval) + + @test.support.cpython_only + def test_no_refcycle_through_target(self): + class RunSelfFunction(object): + def __init__(self, should_raise): + # The links in this refcycle from Thread back to self + # should be cleaned up when the thread completes. + self.should_raise = should_raise + self.thread = threading.Thread(target=self._run, + args=(self,), + kwargs={'yet_another':self}) + self.thread.start() + + def _run(self, other_ref, yet_another): + if self.should_raise: + raise SystemExit + + cyclic_object = RunSelfFunction(should_raise=False) + weak_cyclic_object = weakref.ref(cyclic_object) + cyclic_object.thread.join() + del cyclic_object + self.assertIsNone(weak_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_cyclic_object()))) + + raising_cyclic_object = RunSelfFunction(should_raise=True) + weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) + raising_cyclic_object.thread.join() + del raising_cyclic_object + self.assertIsNone(weak_raising_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_raising_cyclic_object()))) + + def test_old_threading_api(self): + # Just a quick sanity check to make sure the old method names are + # still present + t = threading.Thread() + t.isDaemon() + t.setDaemon(True) + t.getName() + t.setName("name") + t.isAlive() + e = threading.Event() + e.isSet() + threading.activeCount() + + def test_repr_daemon(self): + t = threading.Thread() + self.assertNotIn('daemon', repr(t)) + t.daemon = True + self.assertIn('daemon', repr(t)) + + def test_deamon_param(self): + t = threading.Thread() + self.assertFalse(t.daemon) + t = threading.Thread(daemon=False) + self.assertFalse(t.daemon) + t = threading.Thread(daemon=True) + self.assertTrue(t.daemon) + + @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + def test_dummy_thread_after_fork(self): + # Issue #14308: a dummy thread in the active list doesn't mess up + # the after-fork mechanism. + code = """if 1: + import _thread, threading, os, time + + def background_thread(evt): + # Creates and registers the _DummyThread instance + threading.current_thread() + evt.set() + time.sleep(10) + + evt = threading.Event() + _thread.start_new_thread(background_thread, (evt,)) + evt.wait() + assert threading.active_count() == 2, threading.active_count() + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + os.wait() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + newgil = hasattr(sys, 'getswitchinterval') + if newgil: + geti, seti = sys.getswitchinterval, sys.setswitchinterval + else: + geti, seti = sys.getcheckinterval, sys.setcheckinterval + old_interval = geti() + self.addCleanup(seti, old_interval) + + # Make the bug more likely to manifest. + seti(1e-6 if newgil else 1) + + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + self.addCleanup(t.join) + pid = os.fork() + if pid == 0: + os._exit(1 if t.is_alive() else 0) + else: + pid, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + def test_main_thread(self): + main = threading.main_thread() + self.assertEqual(main.name, 'MainThread') + self.assertEqual(main.ident, threading.current_thread().ident) + self.assertEqual(main.ident, threading.get_ident()) + + def f(): + self.assertNotEqual(threading.main_thread().ident, + threading.current_thread().ident) + th = threading.Thread(target=f) + th.start() + th.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork(self): + code = """if 1: + import os, threading + + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + else: + os.waitpid(pid, 0) + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "MainThread\nTrue\nTrue\n") + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_nonmain_thread(self): + code = """if 1: + import os, threading, sys + + def f(): + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + os.waitpid(pid, 0) + + th = threading.Thread(target=f) + th.start() + th.join() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertIsNone(t._tstate_lock) + + def test_repr_stopped(self): + # Verify that "stopped" shows up in repr(Thread) appropriately. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + t = threading.Thread(target=f) + t.start() + started.acquire() + self.assertIn("started", repr(t)) + finish.release() + # "stopped" should appear in the repr in a reasonable amount of time. + # Implementation detail: as of this writing, that's trivially true + # if .join() is called, and almost trivially true if .is_alive() is + # called. The detail we're testing here is that "stopped" shows up + # "all on its own". + LOOKING_FOR = "stopped" + for i in range(500): + if LOOKING_FOR in repr(t): + break + time.sleep(0.01) + self.assertIn(LOOKING_FOR, repr(t)) # we waited at least 5 seconds + + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) + + @cpython_only + def test_frame_tstate_tracing(self): + # Issue #14432: Crash when a generator is created in a C thread that is + # destroyed while the generator is still used. The issue was that a + # generator contains a frame, and the frame kept a reference to the + # Python state of the destroyed C thread. The crash occurs when a trace + # function is setup. + + def noop_trace(frame, event, arg): + # no operation + return noop_trace + + def generator(): + while 1: + yield "generator" + + def callback(): + if callback.gen is None: + callback.gen = generator() + return next(callback.gen) + callback.gen = None + + old_trace = sys.gettrace() + sys.settrace(noop_trace) + try: + # Install a trace function + threading.settrace(noop_trace) + + # Create a generator in a C thread which exits after the call + import _testcapi + _testcapi.call_in_temporary_c_thread(callback) + + # Call the generator in a different Python thread, check that the + # generator didn't keep a reference to the destroyed thread state + for test in range(3): + # The trace function is still called here + callback() + finally: + sys.settrace(old_trace) + + +class ThreadJoinOnShutdown(BaseTestCase): + + def _run_and_join(self, script): + script = """if 1: + import sys, os, time, threading + + # a thread, which waits for the main program to terminate + def joiningfunc(mainthread): + mainthread.join() + print('end of thread') + # stdout is fully buffered because not a tty, we have to flush + # before exit. + sys.stdout.flush() + \n""" + script + + rc, out, err = assert_python_ok("-c", script) + data = out.decode().replace('\r', '') + self.assertEqual(data, "end of main\nend of thread\n") + + def test_1_join_on_shutdown(self): + # The usual case: on exit, wait for a non-daemon thread + script = """if 1: + import os + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + time.sleep(0.1) + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_2_join_in_forked_process(self): + # Like the test above, but from a forked interpreter + script = """if 1: + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_3_join_in_forked_from_thread(self): + # Like the test above, but fork() was called from a worker thread + # In the forked process, the main Thread object must be marked as stopped. + + script = """if 1: + main_thread = threading.current_thread() + def worker(): + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(main_thread,)) + print('end of main') + t.start() + t.join() # Should not block: main_thread is already stopped + + w = threading.Thread(target=worker) + w.start() + """ + self._run_and_join(script) + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_4_daemon_threads(self): + # Check that a daemon thread cannot crash the interpreter on shutdown + # by manipulating internal structures that are being disposed of in + # the main thread. + script = """if True: + import os + import random + import sys + import time + import threading + + thread_has_run = set() + + def random_io(): + '''Loop for a while sleeping random tiny amounts and doing some I/O.''' + while True: + in_f = open(os.__file__, 'rb') + stuff = in_f.read(200) + null_f = open(os.devnull, 'wb') + null_f.write(stuff) + time.sleep(random.random() / 1995) + null_f.close() + in_f.close() + thread_has_run.add(threading.current_thread()) + + def main(): + count = 0 + for _ in range(40): + new_thread = threading.Thread(target=random_io) + new_thread.daemon = True + new_thread.start() + count += 1 + while len(thread_has_run) < count: + time.sleep(0.001) + # Trigger process shutdown + sys.exit(0) + + main() + """ + rc, out, err = assert_python_ok('-c', script) + self.assertFalse(err) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_reinit_tls_after_fork(self): + # Issue #13817: fork() would deadlock in a multithreaded program with + # the ad-hoc TLS implementation. + + def do_fork_and_wait(): + # just fork a child process and wait it + pid = os.fork() + if pid > 0: + os.waitpid(pid, 0) + else: + os._exit(0) + + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() + + for t in threads: + t.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_clear_threads_states_after_fork(self): + # Issue #17094: check that threads states are cleared after fork() + + # start a bunch of threads + threads = [] + for i in range(16): + t = threading.Thread(target=lambda : time.sleep(0.3)) + threads.append(t) + t.start() + + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(0) + else: + os._exit(1) + else: + _, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + for t in threads: + t.join() + + +class SubinterpThreadingTests(BaseTestCase): + + @cpython_only + def test_threads_join(self): + # Non-daemon threads should be joined at subinterpreter shutdown + # (issue #18808) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + @cpython_only + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + @cpython_only + def test_daemon_threads_fatal_error(self): + subinterp_code = r"""if 1: + import os + import threading + import time + + def f(): + # Make sure the daemon thread is still running when + # Py_EndInterpreter is called. + time.sleep(10) + threading.Thread(target=f, daemon=True).start() + """ + script = r"""if 1: + import _testcapi + + _testcapi.run_in_subinterp(%r) + """ % (subinterp_code,) + with test.support.SuppressCrashReport(): + rc, out, err = assert_python_failure("-c", script) + self.assertIn("Fatal Python error: Py_EndInterpreter: " + "not the last thread", err.decode()) + + +class ThreadingExceptionTests(BaseTestCase): + # A RuntimeError should be raised if Thread.start() is called + # multiple times. + def test_start_thread_again(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, thread.start) + + def test_joining_current_thread(self): + current_thread = threading.current_thread() + self.assertRaises(RuntimeError, current_thread.join); + + def test_joining_inactive_thread(self): + thread = threading.Thread() + self.assertRaises(RuntimeError, thread.join) + + def test_daemonize_active_thread(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + + def test_releasing_unacquired_lock(self): + lock = threading.Lock() + self.assertRaises(RuntimeError, lock.release) + + @unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(), + 'test macosx problem') + def test_recursion_limit(self): + # Issue 9670 + # test that excessive recursion within a non-main thread causes + # an exception rather than crashing the interpreter on platforms + # like Mac OS X or FreeBSD which have small default stack sizes + # for threads + script = """if True: + import threading + + def recurse(): + return recurse() + + def outer(): + try: + recurse() + except RecursionError: + pass + + w = threading.Thread(target=outer) + w.start() + w.join() + print('end of main thread') + """ + expected_output = "end of main thread\n" + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + data = stdout.decode().replace('\r', '') + self.assertEqual(p.returncode, 0, "Unexpected error: " + stderr.decode()) + self.assertEqual(data, expected_output) + + def test_print_exception(self): + script = r"""if True: + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + @requires_type_collecting + def test_print_exception_stderr_is_none_1(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + sys.stderr = None + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_2(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + sys.stderr = None + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + self.assertNotIn("Unhandled exception", err.decode()) + + def test_bare_raise_in_brand_new_thread(self): + def bare_raise(): + raise + + class Issue27558(threading.Thread): + exc = None + + def run(self): + try: + bare_raise() + except Exception as exc: + self.exc = exc + + thread = Issue27558() + thread.start() + thread.join() + self.assertIsNotNone(thread.exc) + self.assertIsInstance(thread.exc, RuntimeError) + +class TimerTests(BaseTestCase): + + def setUp(self): + BaseTestCase.setUp(self) + self.callback_args = [] + self.callback_event = threading.Event() + + def test_init_immutable_default_args(self): + # Issue 17435: constructor defaults were mutable objects, they could be + # mutated via the object attributes and affect other Timer objects. + timer1 = threading.Timer(0.01, self._callback_spy) + timer1.start() + self.callback_event.wait() + timer1.args.append("blah") + timer1.kwargs["foo"] = "bar" + self.callback_event.clear() + timer2 = threading.Timer(0.01, self._callback_spy) + timer2.start() + self.callback_event.wait() + self.assertEqual(len(self.callback_args), 2) + self.assertEqual(self.callback_args, [((), {}), ((), {})]) + + def _callback_spy(self, *args, **kwargs): + self.callback_args.append((args[:], kwargs.copy())) + self.callback_event.set() + +class LockTests(lock_tests.LockTests): + locktype = staticmethod(threading.Lock) + + + @unittest.skip("not on gevent") + def test_locked_repr(self): + pass + + @unittest.skip("not on gevent") + def test_repr(self): + pass + + +class PyRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._PyRLock) + +@unittest.skipIf(threading._CRLock is None, 'RLock not implemented in C') +class CRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._CRLock) + +class EventTests(lock_tests.EventTests): + eventtype = staticmethod(threading.Event) + + @unittest.skip("not on gevent") + def test_reset_internal_locks(self): + pass + + +class ConditionAsRLockTests(lock_tests.RLockTests): + # Condition uses an RLock by default and exports its API. + locktype = staticmethod(threading.Condition) + +class ConditionTests(lock_tests.ConditionTests): + condtype = staticmethod(threading.Condition) + +class SemaphoreTests(lock_tests.SemaphoreTests): + semtype = staticmethod(threading.Semaphore) + +class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): + semtype = staticmethod(threading.BoundedSemaphore) + +class BarrierTests(lock_tests.BarrierTests): + barriertype = staticmethod(threading.Barrier) + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_threading_local.py b/src/greentest/3.5pypy/test_threading_local.py new file mode 100644 index 0000000..0e0c11f --- /dev/null +++ b/src/greentest/3.5pypy/test_threading_local.py @@ -0,0 +1,220 @@ +import unittest +from doctest import DocTestSuite +from test import support +import weakref +import gc + +# Modules under test +_thread = support.import_module('_thread') +threading = support.import_module('threading') +import _threading_local + + +class Weak(object): + pass + +def target(local, weaklist): + weak = Weak() + local.weak = weak + weaklist.append(weakref.ref(weak)) + + +class BaseLocalTest: + + def test_local_refs(self): + self._local_refs(20) + self._local_refs(50) + self._local_refs(100) + + def _local_refs(self, n): + local = self._local() + weaklist = [] + for i in range(n): + t = threading.Thread(target=target, args=(local, weaklist)) + t.start() + t.join() + del t + + support.gc_collect() + self.assertEqual(len(weaklist), n) + + # XXX _threading_local keeps the local of the last stopped thread alive. + deadlist = [weak for weak in weaklist if weak() is None] + self.assertIn(len(deadlist), (n-1, n)) + + # Assignment to the same thread local frees it sometimes (!) + local.someothervar = None + support.gc_collect() + deadlist = [weak for weak in weaklist if weak() is None] + self.assertIn(len(deadlist), (n-1, n), (n, len(deadlist))) + + def test_derived(self): + # Issue 3088: if there is a threads switch inside the __init__ + # of a threading.local derived class, the per-thread dictionary + # is created but not correctly set on the object. + # The first member set may be bogus. + import time + class Local(self._local): + def __init__(self): + time.sleep(0.01) + local = Local() + + def f(i): + local.x = i + # Simply check that the variable is correctly set + self.assertEqual(local.x, i) + + with support.start_threads(threading.Thread(target=f, args=(i,)) + for i in range(10)): + pass + + def test_derived_cycle_dealloc(self): + # http://bugs.python.org/issue6990 + class Local(self._local): + pass + locals = None + passed = False + e1 = threading.Event() + e2 = threading.Event() + + def f(): + nonlocal passed + # 1) Involve Local in a cycle + cycle = [Local()] + cycle.append(cycle) + cycle[0].foo = 'bar' + + # 2) GC the cycle (triggers threadmodule.c::local_clear + # before local_dealloc) + del cycle + support.gc_collect() + e1.set() + e2.wait() + + # 4) New Locals should be empty + passed = all(not hasattr(local, 'foo') for local in locals) + + t = threading.Thread(target=f) + t.start() + e1.wait() + + # 3) New Locals should recycle the original's address. Creating + # them in the thread overwrites the thread state and avoids the + # bug + locals = [Local() for i in range(10)] + e2.set() + t.join() + + self.assertTrue(passed) + + def test_arguments(self): + # Issue 1522237 + class MyLocal(self._local): + def __init__(self, *args, **kwargs): + pass + + MyLocal(a=1) + MyLocal(1) + self.assertRaises(TypeError, self._local, a=1) + self.assertRaises(TypeError, self._local, 1) + + def _test_one_class(self, c): + self._failed = "No error message set or cleared." + obj = c() + e1 = threading.Event() + e2 = threading.Event() + + def f1(): + obj.x = 'foo' + obj.y = 'bar' + del obj.y + e1.set() + e2.wait() + + def f2(): + try: + foo = obj.x + except AttributeError: + # This is expected -- we haven't set obj.x in this thread yet! + self._failed = "" # passed + else: + self._failed = ('Incorrectly got value %r from class %r\n' % + (foo, c)) + sys.stderr.write(self._failed) + + t1 = threading.Thread(target=f1) + t1.start() + e1.wait() + t2 = threading.Thread(target=f2) + t2.start() + t2.join() + # The test is done; just let t1 know it can exit, and wait for it. + e2.set() + t1.join() + + self.assertFalse(self._failed, self._failed) + + def test_threading_local(self): + self._test_one_class(self._local) + + def test_threading_local_subclass(self): + class LocalSubclass(self._local): + """To test that subclasses behave properly.""" + self._test_one_class(LocalSubclass) + + def _test_dict_attribute(self, cls): + obj = cls() + obj.x = 5 + self.assertEqual(obj.__dict__, {'x': 5}) + with self.assertRaises(AttributeError): + obj.__dict__ = {} + with self.assertRaises(AttributeError): + del obj.__dict__ + + def test_dict_attribute(self): + self._test_dict_attribute(self._local) + + def test_dict_attribute_subclass(self): + class LocalSubclass(self._local): + """To test that subclasses behave properly.""" + self._test_dict_attribute(LocalSubclass) + + def test_cycle_collection(self): + class X: + pass + + x = X() + x.local = self._local() + x.local.x = x + wr = weakref.ref(x) + del x + support.gc_collect() + self.assertIsNone(wr()) + + +class ThreadLocalTest(unittest.TestCase, BaseLocalTest): + _local = _thread._local + +class PyThreadingLocalTest(unittest.TestCase, BaseLocalTest): + _local = _threading_local.local + + +def test_main(): + suite = unittest.TestSuite() + suite.addTest(DocTestSuite('_threading_local')) + suite.addTest(unittest.makeSuite(ThreadLocalTest)) + suite.addTest(unittest.makeSuite(PyThreadingLocalTest)) + + local_orig = _threading_local.local + def setUp(test): + _threading_local.local = _thread._local + def tearDown(test): + _threading_local.local = local_orig + suite.addTest(DocTestSuite('_threading_local', + setUp=setUp, tearDown=tearDown) + ) + + support.run_unittest(suite) + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/3.5pypy/test_timeout.py b/src/greentest/3.5pypy/test_timeout.py new file mode 100644 index 0000000..3c75dcc --- /dev/null +++ b/src/greentest/3.5pypy/test_timeout.py @@ -0,0 +1,303 @@ +"""Unit tests for socket timeout feature.""" + +import functools +import unittest +from test import support + +# This requires the 'network' resource as given on the regrtest command line. +skip_expected = not support.is_resource_enabled('network') + +import time +import errno +import socket + + +@functools.lru_cache() +def resolve_address(host, port): + """Resolve an (host, port) to an address. + + We must perform name resolution before timeout tests, otherwise it will be + performed by connect(). + """ + with support.transient_internet(host): + return socket.getaddrinfo(host, port, socket.AF_INET, + socket.SOCK_STREAM)[0][4] + + +class CreationTestCase(unittest.TestCase): + """Test case for socket.gettimeout() and socket.settimeout()""" + + def setUp(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def tearDown(self): + self.sock.close() + + def testObjectCreation(self): + # Test Socket creation + self.assertEqual(self.sock.gettimeout(), None, + "timeout not disabled by default") + + def testFloatReturnValue(self): + # Test return value of gettimeout() + self.sock.settimeout(7.345) + self.assertEqual(self.sock.gettimeout(), 7.345) + + self.sock.settimeout(3) + self.assertEqual(self.sock.gettimeout(), 3) + + self.sock.settimeout(None) + self.assertEqual(self.sock.gettimeout(), None) + + def testReturnType(self): + # Test return type of gettimeout() + self.sock.settimeout(1) + self.assertEqual(type(self.sock.gettimeout()), type(1.0)) + + self.sock.settimeout(3.9) + self.assertEqual(type(self.sock.gettimeout()), type(1.0)) + + def testTypeCheck(self): + # Test type checking by settimeout() + self.sock.settimeout(0) + self.sock.settimeout(0) + self.sock.settimeout(0.0) + self.sock.settimeout(None) + self.assertRaises(TypeError, self.sock.settimeout, "") + self.assertRaises(TypeError, self.sock.settimeout, "") + self.assertRaises(TypeError, self.sock.settimeout, ()) + self.assertRaises(TypeError, self.sock.settimeout, []) + self.assertRaises(TypeError, self.sock.settimeout, {}) + self.assertRaises(TypeError, self.sock.settimeout, 0j) + + def testRangeCheck(self): + # Test range checking by settimeout() + self.assertRaises(ValueError, self.sock.settimeout, -1) + self.assertRaises(ValueError, self.sock.settimeout, -1) + self.assertRaises(ValueError, self.sock.settimeout, -1.0) + + def testTimeoutThenBlocking(self): + # Test settimeout() followed by setblocking() + self.sock.settimeout(10) + self.sock.setblocking(1) + self.assertEqual(self.sock.gettimeout(), None) + self.sock.setblocking(0) + self.assertEqual(self.sock.gettimeout(), 0.0) + + self.sock.settimeout(10) + self.sock.setblocking(0) + self.assertEqual(self.sock.gettimeout(), 0.0) + self.sock.setblocking(1) + self.assertEqual(self.sock.gettimeout(), None) + + def testBlockingThenTimeout(self): + # Test setblocking() followed by settimeout() + self.sock.setblocking(0) + self.sock.settimeout(1) + self.assertEqual(self.sock.gettimeout(), 1) + + self.sock.setblocking(1) + self.sock.settimeout(1) + self.assertEqual(self.sock.gettimeout(), 1) + + +class TimeoutTestCase(unittest.TestCase): + # There are a number of tests here trying to make sure that an operation + # doesn't take too much longer than expected. But competing machine + # activity makes it inevitable that such tests will fail at times. + # When fuzz was at 1.0, I (tim) routinely saw bogus failures on Win2K + # and Win98SE. Boosting it to 2.0 helped a lot, but isn't a real + # solution. + fuzz = 2.0 + + localhost = support.HOST + + def setUp(self): + raise NotImplementedError() + + tearDown = setUp + + def _sock_operation(self, count, timeout, method, *args): + """ + Test the specified socket method. + + The method is run at most `count` times and must raise a socket.timeout + within `timeout` + self.fuzz seconds. + """ + self.sock.settimeout(timeout) + method = getattr(self.sock, method) + for i in range(count): + t1 = time.time() + try: + method(*args) + except socket.timeout as e: + delta = time.time() - t1 + break + else: + self.fail('socket.timeout was not raised') + # These checks should account for timing unprecision + self.assertLess(delta, timeout + self.fuzz) + self.assertGreater(delta, timeout - 1.0) + + +class TCPTimeoutTestCase(TimeoutTestCase): + """TCP test case for socket.socket() timeout functions""" + + def setUp(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addr_remote = resolve_address('www.python.org.', 80) + + def tearDown(self): + self.sock.close() + + def testConnectTimeout(self): + # Testing connect timeout is tricky: we need to have IP connectivity + # to a host that silently drops our packets. We can't simulate this + # from Python because it's a function of the underlying TCP/IP stack. + # So, the following Snakebite host has been defined: + blackhole = resolve_address('blackhole.snakebite.net', 56666) + + # Blackhole has been configured to silently drop any incoming packets. + # No RSTs (for TCP) or ICMP UNREACH (for UDP/ICMP) will be sent back + # to hosts that attempt to connect to this address: which is exactly + # what we need to confidently test connect timeout. + + # However, we want to prevent false positives. It's not unreasonable + # to expect certain hosts may not be able to reach the blackhole, due + # to firewalling or general network configuration. In order to improve + # our confidence in testing the blackhole, a corresponding 'whitehole' + # has also been set up using one port higher: + whitehole = resolve_address('whitehole.snakebite.net', 56667) + + # This address has been configured to immediately drop any incoming + # packets as well, but it does it respectfully with regards to the + # incoming protocol. RSTs are sent for TCP packets, and ICMP UNREACH + # is sent for UDP/ICMP packets. This means our attempts to connect to + # it should be met immediately with ECONNREFUSED. The test case has + # been structured around this premise: if we get an ECONNREFUSED from + # the whitehole, we proceed with testing connect timeout against the + # blackhole. If we don't, we skip the test (with a message about not + # getting the required RST from the whitehole within the required + # timeframe). + + # For the records, the whitehole/blackhole configuration has been set + # up using the 'pf' firewall (available on BSDs), using the following: + # + # ext_if="bge0" + # + # blackhole_ip="35.8.247.6" + # whitehole_ip="35.8.247.6" + # blackhole_port="56666" + # whitehole_port="56667" + # + # block return in log quick on $ext_if proto { tcp udp } \ + # from any to $whitehole_ip port $whitehole_port + # block drop in log quick on $ext_if proto { tcp udp } \ + # from any to $blackhole_ip port $blackhole_port + # + + skip = True + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Use a timeout of 3 seconds. Why 3? Because it's more than 1, and + # less than 5. i.e. no particular reason. Feel free to tweak it if + # you feel a different value would be more appropriate. + timeout = 3 + sock.settimeout(timeout) + try: + sock.connect((whitehole)) + except socket.timeout: + pass + except OSError as err: + if err.errno == errno.ECONNREFUSED: + skip = False + finally: + sock.close() + del sock + + if skip: + self.skipTest( + "We didn't receive a connection reset (RST) packet from " + "{}:{} within {} seconds, so we're unable to test connect " + "timeout against the corresponding {}:{} (which is " + "configured to silently drop packets)." + .format( + whitehole[0], + whitehole[1], + timeout, + blackhole[0], + blackhole[1], + ) + ) + + # All that hard work just to test if connect times out in 0.001s ;-) + self.addr_remote = blackhole + with support.transient_internet(self.addr_remote[0]): + self._sock_operation(1, 0.001, 'connect', self.addr_remote) + + def testRecvTimeout(self): + # Test recv() timeout + with support.transient_internet(self.addr_remote[0]): + self.sock.connect(self.addr_remote) + self._sock_operation(1, 1.5, 'recv', 1024) + + def testAcceptTimeout(self): + # Test accept() timeout + support.bind_port(self.sock, self.localhost) + self.sock.listen() + self._sock_operation(1, 1.5, 'accept') + + def testSend(self): + # Test send() timeout + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as serv: + support.bind_port(serv, self.localhost) + serv.listen() + self.sock.connect(serv.getsockname()) + # Send a lot of data in order to bypass buffering in the TCP stack. + self._sock_operation(100, 1.5, 'send', b"X" * 200000) + + def testSendto(self): + # Test sendto() timeout + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as serv: + support.bind_port(serv, self.localhost) + serv.listen() + self.sock.connect(serv.getsockname()) + # The address argument is ignored since we already connected. + self._sock_operation(100, 1.5, 'sendto', b"X" * 200000, + serv.getsockname()) + + def testSendall(self): + # Test sendall() timeout + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as serv: + support.bind_port(serv, self.localhost) + serv.listen() + self.sock.connect(serv.getsockname()) + # Send a lot of data in order to bypass buffering in the TCP stack. + self._sock_operation(100, 1.5, 'sendall', b"X" * 200000) + + +class UDPTimeoutTestCase(TimeoutTestCase): + """UDP test case for socket.socket() timeout functions""" + + def setUp(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + def tearDown(self): + self.sock.close() + + def testRecvfromTimeout(self): + # Test recvfrom() timeout + # Prevent "Address already in use" socket exceptions + support.bind_port(self.sock, self.localhost) + self._sock_operation(1, 1.5, 'recvfrom', 1024) + + +def test_main(): + support.requires('network') + support.run_unittest( + CreationTestCase, + TCPTimeoutTestCase, + UDPTimeoutTestCase, + ) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.5pypy/test_urllib.py b/src/greentest/3.5pypy/test_urllib.py new file mode 100644 index 0000000..247598a --- /dev/null +++ b/src/greentest/3.5pypy/test_urllib.py @@ -0,0 +1,1566 @@ +"""Regression tests for what was in Python 2's "urllib" module""" + +import urllib.parse +import urllib.request +import urllib.error +import http.client +import email.message +import io +import unittest +from unittest.mock import patch +from test import support +import os +try: + import ssl +except ImportError: + ssl = None +import sys +import tempfile +from nturl2path import url2pathname, pathname2url + +from base64 import b64encode +import collections + + +def hexescape(char): + """Escape char as RFC 2396 specifies""" + hex_repr = hex(ord(char))[2:].upper() + if len(hex_repr) == 1: + hex_repr = "0%s" % hex_repr + return "%" + hex_repr + +# Shortcut for testing FancyURLopener +_urlopener = None + + +def urlopen(url, data=None, proxies=None): + """urlopen(url [, data]) -> open file-like object""" + global _urlopener + if proxies is not None: + opener = urllib.request.FancyURLopener(proxies=proxies) + elif not _urlopener: + opener = FancyURLopener() + _urlopener = opener + else: + opener = _urlopener + if data is None: + return opener.open(url) + else: + return opener.open(url, data) + + +def FancyURLopener(): + with support.check_warnings( + ('FancyURLopener style of invoking requests is deprecated.', + DeprecationWarning)): + return urllib.request.FancyURLopener() + + +def fakehttp(fakedata): + class FakeSocket(io.BytesIO): + io_refs = 1 + + def sendall(self, data): + FakeHTTPConnection.buf = data + + def makefile(self, *args, **kwds): + self.io_refs += 1 + return self + + def read(self, amt=None): + if self.closed: + return b"" + return io.BytesIO.read(self, amt) + + def readline(self, length=None): + if self.closed: + return b"" + return io.BytesIO.readline(self, length) + + def close(self): + self.io_refs -= 1 + if self.io_refs == 0: + io.BytesIO.close(self) + + class FakeHTTPConnection(http.client.HTTPConnection): + + # buffer to store data for verification in urlopen tests. + buf = None + + def connect(self): + self.sock = FakeSocket(self.fakedata) + type(self).fakesock = self.sock + FakeHTTPConnection.fakedata = fakedata + + return FakeHTTPConnection + + +class FakeHTTPMixin(object): + def fakehttp(self, fakedata): + self._connection_class = http.client.HTTPConnection + http.client.HTTPConnection = fakehttp(fakedata) + + def unfakehttp(self): + http.client.HTTPConnection = self._connection_class + + +class FakeFTPMixin(object): + def fakeftp(self): + class FakeFtpWrapper(object): + def __init__(self, user, passwd, host, port, dirs, timeout=None, + persistent=True): + pass + + def retrfile(self, file, type): + return io.BytesIO(), 0 + + def close(self): + pass + + self._ftpwrapper_class = urllib.request.ftpwrapper + urllib.request.ftpwrapper = FakeFtpWrapper + + def unfakeftp(self): + urllib.request.ftpwrapper = self._ftpwrapper_class + + +class urlopen_FileTests(unittest.TestCase): + """Test urlopen() opening a temporary file. + + Try to test as much functionality as possible so as to cut down on reliance + on connecting to the Net for testing. + + """ + + def setUp(self): + # Create a temp file to use for testing + self.text = bytes("test_urllib: %s\n" % self.__class__.__name__, + "ascii") + f = open(support.TESTFN, 'wb') + try: + f.write(self.text) + finally: + f.close() + self.pathname = support.TESTFN + self.returned_obj = urlopen("file:%s" % self.pathname) + + def tearDown(self): + """Shut down the open object""" + self.returned_obj.close() + os.remove(support.TESTFN) + + def test_interface(self): + # Make sure object returned by urlopen() has the specified methods + for attr in ("read", "readline", "readlines", "fileno", + "close", "info", "geturl", "getcode", "__iter__"): + self.assertTrue(hasattr(self.returned_obj, attr), + "object returned by urlopen() lacks %s attribute" % + attr) + + def test_read(self): + self.assertEqual(self.text, self.returned_obj.read()) + + def test_readline(self): + self.assertEqual(self.text, self.returned_obj.readline()) + self.assertEqual(b'', self.returned_obj.readline(), + "calling readline() after exhausting the file did not" + " return an empty string") + + def test_readlines(self): + lines_list = self.returned_obj.readlines() + self.assertEqual(len(lines_list), 1, + "readlines() returned the wrong number of lines") + self.assertEqual(lines_list[0], self.text, + "readlines() returned improper text") + + def test_fileno(self): + file_num = self.returned_obj.fileno() + self.assertIsInstance(file_num, int, "fileno() did not return an int") + self.assertEqual(os.read(file_num, len(self.text)), self.text, + "Reading on the file descriptor returned by fileno() " + "did not return the expected text") + + def test_close(self): + # Test close() by calling it here and then having it be called again + # by the tearDown() method for the test + self.returned_obj.close() + + def test_info(self): + self.assertIsInstance(self.returned_obj.info(), email.message.Message) + + def test_geturl(self): + self.assertEqual(self.returned_obj.geturl(), self.pathname) + + def test_getcode(self): + self.assertIsNone(self.returned_obj.getcode()) + + def test_iter(self): + # Test iterator + # Don't need to count number of iterations since test would fail the + # instant it returned anything beyond the first line from the + # comparison. + # Use the iterator in the usual implicit way to test for ticket #4608. + for line in self.returned_obj: + self.assertEqual(line, self.text) + + def test_relativelocalfile(self): + self.assertRaises(ValueError,urllib.request.urlopen,'./' + self.pathname) + +class ProxyTests(unittest.TestCase): + + def setUp(self): + # Records changes to env vars + self.env = support.EnvironmentVarGuard() + # Delete all proxy related env vars + for k in list(os.environ): + if 'proxy' in k.lower(): + self.env.unset(k) + + def tearDown(self): + # Restore all proxy related env vars + self.env.__exit__() + del self.env + + def test_getproxies_environment_keep_no_proxies(self): + self.env.set('NO_PROXY', 'localhost') + proxies = urllib.request.getproxies_environment() + # getproxies_environment use lowered case truncated (no '_proxy') keys + self.assertEqual('localhost', proxies['no']) + # List of no_proxies with space. + self.env.set('NO_PROXY', 'localhost, anotherdomain.com, newdomain.com:1234') + self.assertTrue(urllib.request.proxy_bypass_environment('anotherdomain.com')) + self.assertTrue(urllib.request.proxy_bypass_environment('anotherdomain.com:8888')) + self.assertTrue(urllib.request.proxy_bypass_environment('newdomain.com:1234')) + + def test_proxy_cgi_ignore(self): + try: + self.env.set('HTTP_PROXY', 'http://somewhere:3128') + proxies = urllib.request.getproxies_environment() + self.assertEqual('http://somewhere:3128', proxies['http']) + self.env.set('REQUEST_METHOD', 'GET') + proxies = urllib.request.getproxies_environment() + self.assertNotIn('http', proxies) + finally: + self.env.unset('REQUEST_METHOD') + self.env.unset('HTTP_PROXY') + + def test_proxy_bypass_environment_host_match(self): + bypass = urllib.request.proxy_bypass_environment + self.env.set('NO_PROXY', + 'localhost, anotherdomain.com, newdomain.com:1234') + self.assertTrue(bypass('localhost')) + self.assertTrue(bypass('LocalHost')) # MixedCase + self.assertTrue(bypass('LOCALHOST')) # UPPERCASE + self.assertTrue(bypass('newdomain.com:1234')) + self.assertTrue(bypass('anotherdomain.com:8888')) + self.assertTrue(bypass('www.newdomain.com:1234')) + self.assertFalse(bypass('prelocalhost')) + self.assertFalse(bypass('newdomain.com')) # no port + self.assertFalse(bypass('newdomain.com:1235')) # wrong port + +class ProxyTests_withOrderedEnv(unittest.TestCase): + + def setUp(self): + # We need to test conditions, where variable order _is_ significant + self._saved_env = os.environ + # Monkey patch os.environ, start with empty fake environment + os.environ = collections.OrderedDict() + + def tearDown(self): + os.environ = self._saved_env + + def test_getproxies_environment_prefer_lowercase(self): + # Test lowercase preference with removal + os.environ['no_proxy'] = '' + os.environ['No_Proxy'] = 'localhost' + self.assertFalse(urllib.request.proxy_bypass_environment('localhost')) + self.assertFalse(urllib.request.proxy_bypass_environment('arbitrary')) + os.environ['http_proxy'] = '' + os.environ['HTTP_PROXY'] = 'http://somewhere:3128' + proxies = urllib.request.getproxies_environment() + self.assertEqual({}, proxies) + # Test lowercase preference of proxy bypass and correct matching including ports + os.environ['no_proxy'] = 'localhost, noproxy.com, my.proxy:1234' + os.environ['No_Proxy'] = 'xyz.com' + self.assertTrue(urllib.request.proxy_bypass_environment('localhost')) + self.assertTrue(urllib.request.proxy_bypass_environment('noproxy.com:5678')) + self.assertTrue(urllib.request.proxy_bypass_environment('my.proxy:1234')) + self.assertFalse(urllib.request.proxy_bypass_environment('my.proxy')) + self.assertFalse(urllib.request.proxy_bypass_environment('arbitrary')) + # Test lowercase preference with replacement + os.environ['http_proxy'] = 'http://somewhere:3128' + os.environ['Http_Proxy'] = 'http://somewhereelse:3128' + proxies = urllib.request.getproxies_environment() + self.assertEqual('http://somewhere:3128', proxies['http']) + +class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): + """Test urlopen() opening a fake http connection.""" + + def check_read(self, ver): + self.fakehttp(b"HTTP/" + ver + b" 200 OK\r\n\r\nHello!") + try: + fp = urlopen("http://python.org/") + self.assertEqual(fp.readline(), b"Hello!") + self.assertEqual(fp.readline(), b"") + self.assertEqual(fp.geturl(), 'http://python.org/') + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + def test_url_fragment(self): + # Issue #11703: geturl() omits fragments in the original URL. + url = 'http://docs.python.org/library/urllib.html#OK' + self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello!") + try: + fp = urllib.request.urlopen(url) + self.assertEqual(fp.geturl(), url) + finally: + self.unfakehttp() + + def test_willclose(self): + self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello!") + try: + resp = urlopen("http://www.python.org") + self.assertTrue(resp.fp.will_close) + finally: + self.unfakehttp() + + def test_read_0_9(self): + # "0.9" response accepted (but not "simple responses" without + # a status line) + self.check_read(b"0.9") + + def test_read_1_0(self): + self.check_read(b"1.0") + + def test_read_1_1(self): + self.check_read(b"1.1") + + def test_read_bogus(self): + # urlopen() should raise OSError for many error codes. + self.fakehttp(b'''HTTP/1.1 401 Authentication Required +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Type: text/html; charset=iso-8859-1 +''') + try: + self.assertRaises(OSError, urlopen, "http://python.org/") + finally: + self.unfakehttp() + + def test_invalid_redirect(self): + # urlopen() should raise OSError for many error codes. + self.fakehttp(b'''HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file://guidocomputer.athome.com:/python/license +Connection: close +Content-Type: text/html; charset=iso-8859-1 +''') + try: + msg = "Redirection to url 'file:" + with self.assertRaisesRegex(urllib.error.HTTPError, msg): + urlopen("http://python.org/") + finally: + self.unfakehttp() + + def test_redirect_limit_independent(self): + # Ticket #12923: make sure independent requests each use their + # own retry limit. + for i in range(FancyURLopener().maxtries): + self.fakehttp(b'''HTTP/1.1 302 Found +Location: file://guidocomputer.athome.com:/python/license +Connection: close +''') + try: + self.assertRaises(urllib.error.HTTPError, urlopen, + "http://something") + finally: + self.unfakehttp() + + def test_empty_socket(self): + # urlopen() raises OSError if the underlying socket does not send any + # data. (#1680230) + self.fakehttp(b'') + try: + self.assertRaises(OSError, urlopen, "http://something") + finally: + self.unfakehttp() + + def test_missing_localfile(self): + # Test for #10836 + with self.assertRaises(urllib.error.URLError) as e: + urlopen('file://localhost/a/file/which/doesnot/exists.py') + self.assertTrue(e.exception.filename) + self.assertTrue(e.exception.reason) + + def test_file_notexists(self): + fd, tmp_file = tempfile.mkstemp() + tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + try: + self.assertTrue(os.path.exists(tmp_file)) + with urlopen(tmp_fileurl) as fobj: + self.assertTrue(fobj) + finally: + os.close(fd) + os.unlink(tmp_file) + self.assertFalse(os.path.exists(tmp_file)) + with self.assertRaises(urllib.error.URLError): + urlopen(tmp_fileurl) + + def test_ftp_nohost(self): + test_ftp_url = 'ftp:///path' + with self.assertRaises(urllib.error.URLError) as e: + urlopen(test_ftp_url) + self.assertFalse(e.exception.filename) + self.assertTrue(e.exception.reason) + + def test_ftp_nonexisting(self): + with self.assertRaises(urllib.error.URLError) as e: + urlopen('ftp://localhost/a/file/which/doesnot/exists.py') + self.assertFalse(e.exception.filename) + self.assertTrue(e.exception.reason) + + @patch.object(urllib.request, 'MAXFTPCACHE', 0) + def test_ftp_cache_pruning(self): + self.fakeftp() + try: + urllib.request.ftpcache['test'] = urllib.request.ftpwrapper('user', 'pass', 'localhost', 21, []) + urlopen('ftp://localhost') + finally: + self.unfakeftp() + + + def test_userpass_inurl(self): + self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") + try: + fp = urlopen("http://user:pass@python.org/") + self.assertEqual(fp.readline(), b"Hello!") + self.assertEqual(fp.readline(), b"") + self.assertEqual(fp.geturl(), 'http://user:pass@python.org/') + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + def test_userpass_inurl_w_spaces(self): + self.fakehttp(b"HTTP/1.0 200 OK\r\n\r\nHello!") + try: + userpass = "a b:c d" + url = "http://{}@python.org/".format(userpass) + fakehttp_wrapper = http.client.HTTPConnection + authorization = ("Authorization: Basic %s\r\n" % + b64encode(userpass.encode("ASCII")).decode("ASCII")) + fp = urlopen(url) + # The authorization header must be in place + self.assertIn(authorization, fakehttp_wrapper.buf.decode("UTF-8")) + self.assertEqual(fp.readline(), b"Hello!") + self.assertEqual(fp.readline(), b"") + # the spaces are quoted in URL so no match + self.assertNotEqual(fp.geturl(), url) + self.assertEqual(fp.getcode(), 200) + finally: + self.unfakehttp() + + def test_URLopener_deprecation(self): + with support.check_warnings(('',DeprecationWarning)): + urllib.request.URLopener() + + @unittest.skipUnless(ssl, "ssl module required") + def test_cafile_and_context(self): + context = ssl.create_default_context() + with self.assertRaises(ValueError): + urllib.request.urlopen( + "https://localhost", cafile="/nonexistent/path", context=context + ) + +class urlopen_DataTests(unittest.TestCase): + """Test urlopen() opening a data URL.""" + + def setUp(self): + # text containing URL special- and unicode-characters + self.text = "test data URLs :;,%=& \u00f6 \u00c4 " + # 2x1 pixel RGB PNG image with one black and one white pixel + self.image = ( + b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x02\x00\x00\x00' + b'\x01\x08\x02\x00\x00\x00{@\xe8\xdd\x00\x00\x00\x01sRGB\x00\xae' + b'\xce\x1c\xe9\x00\x00\x00\x0fIDAT\x08\xd7c```\xf8\xff\xff?\x00' + b'\x06\x01\x02\xfe\no/\x1e\x00\x00\x00\x00IEND\xaeB`\x82') + + self.text_url = ( + "data:text/plain;charset=UTF-8,test%20data%20URLs%20%3A%3B%2C%25%3" + "D%26%20%C3%B6%20%C3%84%20") + self.text_url_base64 = ( + "data:text/plain;charset=ISO-8859-1;base64,dGVzdCBkYXRhIFVSTHMgOjs" + "sJT0mIPYgxCA%3D") + # base64 encoded data URL that contains ignorable spaces, + # such as "\n", " ", "%0A", and "%20". + self.image_url = ( + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAIAAAB7\n" + "QOjdAAAAAXNSR0IArs4c6QAAAA9JREFUCNdj%0AYGBg%2BP//PwAGAQL%2BCm8 " + "vHgAAAABJRU5ErkJggg%3D%3D%0A%20") + + self.text_url_resp = urllib.request.urlopen(self.text_url) + self.text_url_base64_resp = urllib.request.urlopen( + self.text_url_base64) + self.image_url_resp = urllib.request.urlopen(self.image_url) + + def test_interface(self): + # Make sure object returned by urlopen() has the specified methods + for attr in ("read", "readline", "readlines", + "close", "info", "geturl", "getcode", "__iter__"): + self.assertTrue(hasattr(self.text_url_resp, attr), + "object returned by urlopen() lacks %s attribute" % + attr) + + def test_info(self): + self.assertIsInstance(self.text_url_resp.info(), email.message.Message) + self.assertEqual(self.text_url_base64_resp.info().get_params(), + [('text/plain', ''), ('charset', 'ISO-8859-1')]) + self.assertEqual(self.image_url_resp.info()['content-length'], + str(len(self.image))) + self.assertEqual(urllib.request.urlopen("data:,").info().get_params(), + [('text/plain', ''), ('charset', 'US-ASCII')]) + + def test_geturl(self): + self.assertEqual(self.text_url_resp.geturl(), self.text_url) + self.assertEqual(self.text_url_base64_resp.geturl(), + self.text_url_base64) + self.assertEqual(self.image_url_resp.geturl(), self.image_url) + + def test_read_text(self): + self.assertEqual(self.text_url_resp.read().decode( + dict(self.text_url_resp.info().get_params())['charset']), self.text) + + def test_read_text_base64(self): + self.assertEqual(self.text_url_base64_resp.read().decode( + dict(self.text_url_base64_resp.info().get_params())['charset']), + self.text) + + def test_read_image(self): + self.assertEqual(self.image_url_resp.read(), self.image) + + def test_missing_comma(self): + self.assertRaises(ValueError,urllib.request.urlopen,'data:text/plain') + + def test_invalid_base64_data(self): + # missing padding character + self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + +class urlretrieve_FileTests(unittest.TestCase): + """Test urllib.urlretrieve() on local files""" + + def setUp(self): + # Create a list of temporary files. Each item in the list is a file + # name (absolute path or relative to the current working directory). + # All files in this list will be deleted in the tearDown method. Note, + # this only helps to makes sure temporary files get deleted, but it + # does nothing about trying to close files that may still be open. It + # is the responsibility of the developer to properly close files even + # when exceptional conditions occur. + self.tempFiles = [] + + # Create a temporary file. + self.registerFileForCleanUp(support.TESTFN) + self.text = b'testing urllib.urlretrieve' + try: + FILE = open(support.TESTFN, 'wb') + FILE.write(self.text) + FILE.close() + finally: + try: FILE.close() + except: pass + + def tearDown(self): + # Delete the temporary files. + for each in self.tempFiles: + try: os.remove(each) + except: pass + + def constructLocalFileUrl(self, filePath): + filePath = os.path.abspath(filePath) + try: + filePath.encode("utf-8") + except UnicodeEncodeError: + raise unittest.SkipTest("filePath is not encodable to utf8") + return "file://%s" % urllib.request.pathname2url(filePath) + + def createNewTempFile(self, data=b""): + """Creates a new temporary file containing the specified data, + registers the file for deletion during the test fixture tear down, and + returns the absolute path of the file.""" + + newFd, newFilePath = tempfile.mkstemp() + try: + self.registerFileForCleanUp(newFilePath) + newFile = os.fdopen(newFd, "wb") + newFile.write(data) + newFile.close() + finally: + try: newFile.close() + except: pass + return newFilePath + + def registerFileForCleanUp(self, fileName): + self.tempFiles.append(fileName) + + def test_basic(self): + # Make sure that a local file just gets its own location returned and + # a headers value is returned. + result = urllib.request.urlretrieve("file:%s" % support.TESTFN) + self.assertEqual(result[0], support.TESTFN) + self.assertIsInstance(result[1], email.message.Message, + "did not get an email.message.Message instance " + "as second returned value") + + def test_copy(self): + # Test that setting the filename argument works. + second_temp = "%s.2" % support.TESTFN + self.registerFileForCleanUp(second_temp) + result = urllib.request.urlretrieve(self.constructLocalFileUrl( + support.TESTFN), second_temp) + self.assertEqual(second_temp, result[0]) + self.assertTrue(os.path.exists(second_temp), "copy of the file was not " + "made") + FILE = open(second_temp, 'rb') + try: + text = FILE.read() + FILE.close() + finally: + try: FILE.close() + except: pass + self.assertEqual(self.text, text) + + def test_reporthook(self): + # Make sure that the reporthook works. + def hooktester(block_count, block_read_size, file_size, count_holder=[0]): + self.assertIsInstance(block_count, int) + self.assertIsInstance(block_read_size, int) + self.assertIsInstance(file_size, int) + self.assertEqual(block_count, count_holder[0]) + count_holder[0] = count_holder[0] + 1 + second_temp = "%s.2" % support.TESTFN + self.registerFileForCleanUp(second_temp) + urllib.request.urlretrieve( + self.constructLocalFileUrl(support.TESTFN), + second_temp, hooktester) + + def test_reporthook_0_bytes(self): + # Test on zero length file. Should call reporthook only 1 time. + report = [] + def hooktester(block_count, block_read_size, file_size, _report=report): + _report.append((block_count, block_read_size, file_size)) + srcFileName = self.createNewTempFile() + urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName), + support.TESTFN, hooktester) + self.assertEqual(len(report), 1) + self.assertEqual(report[0][2], 0) + + def test_reporthook_5_bytes(self): + # Test on 5 byte file. Should call reporthook only 2 times (once when + # the "network connection" is established and once when the block is + # read). + report = [] + def hooktester(block_count, block_read_size, file_size, _report=report): + _report.append((block_count, block_read_size, file_size)) + srcFileName = self.createNewTempFile(b"x" * 5) + urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName), + support.TESTFN, hooktester) + self.assertEqual(len(report), 2) + self.assertEqual(report[0][2], 5) + self.assertEqual(report[1][2], 5) + + def test_reporthook_8193_bytes(self): + # Test on 8193 byte file. Should call reporthook only 3 times (once + # when the "network connection" is established, once for the next 8192 + # bytes, and once for the last byte). + report = [] + def hooktester(block_count, block_read_size, file_size, _report=report): + _report.append((block_count, block_read_size, file_size)) + srcFileName = self.createNewTempFile(b"x" * 8193) + urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName), + support.TESTFN, hooktester) + self.assertEqual(len(report), 3) + self.assertEqual(report[0][2], 8193) + self.assertEqual(report[0][1], 8192) + self.assertEqual(report[1][1], 8192) + self.assertEqual(report[2][1], 8192) + + +class urlretrieve_HttpTests(unittest.TestCase, FakeHTTPMixin): + """Test urllib.urlretrieve() using fake http connections""" + + def test_short_content_raises_ContentTooShortError(self): + self.fakehttp(b'''HTTP/1.1 200 OK +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Length: 100 +Content-Type: text/html; charset=iso-8859-1 + +FF +''') + + def _reporthook(par1, par2, par3): + pass + + with self.assertRaises(urllib.error.ContentTooShortError): + try: + urllib.request.urlretrieve('http://example.com/', + reporthook=_reporthook) + finally: + self.unfakehttp() + + def test_short_content_raises_ContentTooShortError_without_reporthook(self): + self.fakehttp(b'''HTTP/1.1 200 OK +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Length: 100 +Content-Type: text/html; charset=iso-8859-1 + +FF +''') + with self.assertRaises(urllib.error.ContentTooShortError): + try: + urllib.request.urlretrieve('http://example.com/') + finally: + self.unfakehttp() + + +class QuotingTests(unittest.TestCase): + """Tests for urllib.quote() and urllib.quote_plus() + + According to RFC 2396 (Uniform Resource Identifiers), to escape a + character you write it as '%' + <2 character US-ASCII hex value>. + The Python code of ``'%' + hex(ord())[2:]`` escapes a + character properly. Case does not matter on the hex letters. + + The various character sets specified are: + + Reserved characters : ";/?:@&=+$," + Have special meaning in URIs and must be escaped if not being used for + their special meaning + Data characters : letters, digits, and "-_.!~*'()" + Unreserved and do not need to be escaped; can be, though, if desired + Control characters : 0x00 - 0x1F, 0x7F + Have no use in URIs so must be escaped + space : 0x20 + Must be escaped + Delimiters : '<>#%"' + Must be escaped + Unwise : "{}|\^[]`" + Must be escaped + + """ + + def test_never_quote(self): + # Make sure quote() does not quote letters, digits, and "_,.-" + do_not_quote = '' .join(["ABCDEFGHIJKLMNOPQRSTUVWXYZ", + "abcdefghijklmnopqrstuvwxyz", + "0123456789", + "_.-"]) + result = urllib.parse.quote(do_not_quote) + self.assertEqual(do_not_quote, result, + "using quote(): %r != %r" % (do_not_quote, result)) + result = urllib.parse.quote_plus(do_not_quote) + self.assertEqual(do_not_quote, result, + "using quote_plus(): %r != %r" % (do_not_quote, result)) + + def test_default_safe(self): + # Test '/' is default value for 'safe' parameter + self.assertEqual(urllib.parse.quote.__defaults__[0], '/') + + def test_safe(self): + # Test setting 'safe' parameter does what it should do + quote_by_default = "<>" + result = urllib.parse.quote(quote_by_default, safe=quote_by_default) + self.assertEqual(quote_by_default, result, + "using quote(): %r != %r" % (quote_by_default, result)) + result = urllib.parse.quote_plus(quote_by_default, + safe=quote_by_default) + self.assertEqual(quote_by_default, result, + "using quote_plus(): %r != %r" % + (quote_by_default, result)) + # Safe expressed as bytes rather than str + result = urllib.parse.quote(quote_by_default, safe=b"<>") + self.assertEqual(quote_by_default, result, + "using quote(): %r != %r" % (quote_by_default, result)) + # "Safe" non-ASCII characters should have no effect + # (Since URIs are not allowed to have non-ASCII characters) + result = urllib.parse.quote("a\xfcb", encoding="latin-1", safe="\xfc") + expect = urllib.parse.quote("a\xfcb", encoding="latin-1", safe="") + self.assertEqual(expect, result, + "using quote(): %r != %r" % + (expect, result)) + # Same as above, but using a bytes rather than str + result = urllib.parse.quote("a\xfcb", encoding="latin-1", safe=b"\xfc") + expect = urllib.parse.quote("a\xfcb", encoding="latin-1", safe="") + self.assertEqual(expect, result, + "using quote(): %r != %r" % + (expect, result)) + + def test_default_quoting(self): + # Make sure all characters that should be quoted are by default sans + # space (separate test for that). + should_quote = [chr(num) for num in range(32)] # For 0x00 - 0x1F + should_quote.append('<>#%"{}|\^[]`') + should_quote.append(chr(127)) # For 0x7F + should_quote = ''.join(should_quote) + for char in should_quote: + result = urllib.parse.quote(char) + self.assertEqual(hexescape(char), result, + "using quote(): " + "%s should be escaped to %s, not %s" % + (char, hexescape(char), result)) + result = urllib.parse.quote_plus(char) + self.assertEqual(hexescape(char), result, + "using quote_plus(): " + "%s should be escapes to %s, not %s" % + (char, hexescape(char), result)) + del should_quote + partial_quote = "ab[]cd" + expected = "ab%5B%5Dcd" + result = urllib.parse.quote(partial_quote) + self.assertEqual(expected, result, + "using quote(): %r != %r" % (expected, result)) + result = urllib.parse.quote_plus(partial_quote) + self.assertEqual(expected, result, + "using quote_plus(): %r != %r" % (expected, result)) + + def test_quoting_space(self): + # Make sure quote() and quote_plus() handle spaces as specified in + # their unique way + result = urllib.parse.quote(' ') + self.assertEqual(result, hexescape(' '), + "using quote(): %r != %r" % (result, hexescape(' '))) + result = urllib.parse.quote_plus(' ') + self.assertEqual(result, '+', + "using quote_plus(): %r != +" % result) + given = "a b cd e f" + expect = given.replace(' ', hexescape(' ')) + result = urllib.parse.quote(given) + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + expect = given.replace(' ', '+') + result = urllib.parse.quote_plus(given) + self.assertEqual(expect, result, + "using quote_plus(): %r != %r" % (expect, result)) + + def test_quoting_plus(self): + self.assertEqual(urllib.parse.quote_plus('alpha+beta gamma'), + 'alpha%2Bbeta+gamma') + self.assertEqual(urllib.parse.quote_plus('alpha+beta gamma', '+'), + 'alpha+beta+gamma') + # Test with bytes + self.assertEqual(urllib.parse.quote_plus(b'alpha+beta gamma'), + 'alpha%2Bbeta+gamma') + # Test with safe bytes + self.assertEqual(urllib.parse.quote_plus('alpha+beta gamma', b'+'), + 'alpha+beta+gamma') + + def test_quote_bytes(self): + # Bytes should quote directly to percent-encoded values + given = b"\xa2\xd8ab\xff" + expect = "%A2%D8ab%FF" + result = urllib.parse.quote(given) + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + # Encoding argument should raise type error on bytes input + self.assertRaises(TypeError, urllib.parse.quote, given, + encoding="latin-1") + # quote_from_bytes should work the same + result = urllib.parse.quote_from_bytes(given) + self.assertEqual(expect, result, + "using quote_from_bytes(): %r != %r" + % (expect, result)) + + def test_quote_with_unicode(self): + # Characters in Latin-1 range, encoded by default in UTF-8 + given = "\xa2\xd8ab\xff" + expect = "%C2%A2%C3%98ab%C3%BF" + result = urllib.parse.quote(given) + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + # Characters in Latin-1 range, encoded by with None (default) + result = urllib.parse.quote(given, encoding=None, errors=None) + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + # Characters in Latin-1 range, encoded with Latin-1 + given = "\xa2\xd8ab\xff" + expect = "%A2%D8ab%FF" + result = urllib.parse.quote(given, encoding="latin-1") + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + # Characters in BMP, encoded by default in UTF-8 + given = "\u6f22\u5b57" # "Kanji" + expect = "%E6%BC%A2%E5%AD%97" + result = urllib.parse.quote(given) + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + # Characters in BMP, encoded with Latin-1 + given = "\u6f22\u5b57" + self.assertRaises(UnicodeEncodeError, urllib.parse.quote, given, + encoding="latin-1") + # Characters in BMP, encoded with Latin-1, with replace error handling + given = "\u6f22\u5b57" + expect = "%3F%3F" # "??" + result = urllib.parse.quote(given, encoding="latin-1", + errors="replace") + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + # Characters in BMP, Latin-1, with xmlcharref error handling + given = "\u6f22\u5b57" + expect = "%26%2328450%3B%26%2323383%3B" # "漢字" + result = urllib.parse.quote(given, encoding="latin-1", + errors="xmlcharrefreplace") + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + + def test_quote_plus_with_unicode(self): + # Encoding (latin-1) test for quote_plus + given = "\xa2\xd8 \xff" + expect = "%A2%D8+%FF" + result = urllib.parse.quote_plus(given, encoding="latin-1") + self.assertEqual(expect, result, + "using quote_plus(): %r != %r" % (expect, result)) + # Errors test for quote_plus + given = "ab\u6f22\u5b57 cd" + expect = "ab%3F%3F+cd" + result = urllib.parse.quote_plus(given, encoding="latin-1", + errors="replace") + self.assertEqual(expect, result, + "using quote_plus(): %r != %r" % (expect, result)) + + +class UnquotingTests(unittest.TestCase): + """Tests for unquote() and unquote_plus() + + See the doc string for quoting_Tests for details on quoting and such. + + """ + + def test_unquoting(self): + # Make sure unquoting of all ASCII values works + escape_list = [] + for num in range(128): + given = hexescape(chr(num)) + expect = chr(num) + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + result = urllib.parse.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %r != %r" % + (expect, result)) + escape_list.append(given) + escape_string = ''.join(escape_list) + del escape_list + result = urllib.parse.unquote(escape_string) + self.assertEqual(result.count('%'), 1, + "using unquote(): not all characters escaped: " + "%s" % result) + self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, None) + self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, ()) + with support.check_warnings(('', BytesWarning), quiet=True): + self.assertRaises((TypeError, AttributeError), urllib.parse.unquote, b'') + + def test_unquoting_badpercent(self): + # Test unquoting on bad percent-escapes + given = '%xab' + expect = given + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + given = '%x' + expect = given + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + given = '%' + expect = given + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, "using unquote(): %r != %r" + % (expect, result)) + # unquote_to_bytes + given = '%xab' + expect = bytes(given, 'ascii') + result = urllib.parse.unquote_to_bytes(given) + self.assertEqual(expect, result, "using unquote_to_bytes(): %r != %r" + % (expect, result)) + given = '%x' + expect = bytes(given, 'ascii') + result = urllib.parse.unquote_to_bytes(given) + self.assertEqual(expect, result, "using unquote_to_bytes(): %r != %r" + % (expect, result)) + given = '%' + expect = bytes(given, 'ascii') + result = urllib.parse.unquote_to_bytes(given) + self.assertEqual(expect, result, "using unquote_to_bytes(): %r != %r" + % (expect, result)) + self.assertRaises((TypeError, AttributeError), urllib.parse.unquote_to_bytes, None) + self.assertRaises((TypeError, AttributeError), urllib.parse.unquote_to_bytes, ()) + + def test_unquoting_mixed_case(self): + # Test unquoting on mixed-case hex digits in the percent-escapes + given = '%Ab%eA' + expect = b'\xab\xea' + result = urllib.parse.unquote_to_bytes(given) + self.assertEqual(expect, result, + "using unquote_to_bytes(): %r != %r" + % (expect, result)) + + def test_unquoting_parts(self): + # Make sure unquoting works when have non-quoted characters + # interspersed + given = 'ab%sd' % hexescape('c') + expect = "abcd" + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using quote(): %r != %r" % (expect, result)) + result = urllib.parse.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %r != %r" % (expect, result)) + + def test_unquoting_plus(self): + # Test difference between unquote() and unquote_plus() + given = "are+there+spaces..." + expect = given + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + expect = given.replace('+', ' ') + result = urllib.parse.unquote_plus(given) + self.assertEqual(expect, result, + "using unquote_plus(): %r != %r" % (expect, result)) + + def test_unquote_to_bytes(self): + given = 'br%C3%BCckner_sapporo_20050930.doc' + expect = b'br\xc3\xbcckner_sapporo_20050930.doc' + result = urllib.parse.unquote_to_bytes(given) + self.assertEqual(expect, result, + "using unquote_to_bytes(): %r != %r" + % (expect, result)) + # Test on a string with unescaped non-ASCII characters + # (Technically an invalid URI; expect those characters to be UTF-8 + # encoded). + result = urllib.parse.unquote_to_bytes("\u6f22%C3%BC") + expect = b'\xe6\xbc\xa2\xc3\xbc' # UTF-8 for "\u6f22\u00fc" + self.assertEqual(expect, result, + "using unquote_to_bytes(): %r != %r" + % (expect, result)) + # Test with a bytes as input + given = b'%A2%D8ab%FF' + expect = b'\xa2\xd8ab\xff' + result = urllib.parse.unquote_to_bytes(given) + self.assertEqual(expect, result, + "using unquote_to_bytes(): %r != %r" + % (expect, result)) + # Test with a bytes as input, with unescaped non-ASCII bytes + # (Technically an invalid URI; expect those bytes to be preserved) + given = b'%A2\xd8ab%FF' + expect = b'\xa2\xd8ab\xff' + result = urllib.parse.unquote_to_bytes(given) + self.assertEqual(expect, result, + "using unquote_to_bytes(): %r != %r" + % (expect, result)) + + def test_unquote_with_unicode(self): + # Characters in the Latin-1 range, encoded with UTF-8 + given = 'br%C3%BCckner_sapporo_20050930.doc' + expect = 'br\u00fcckner_sapporo_20050930.doc' + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + # Characters in the Latin-1 range, encoded with None (default) + result = urllib.parse.unquote(given, encoding=None, errors=None) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + # Characters in the Latin-1 range, encoded with Latin-1 + result = urllib.parse.unquote('br%FCckner_sapporo_20050930.doc', + encoding="latin-1") + expect = 'br\u00fcckner_sapporo_20050930.doc' + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + # Characters in BMP, encoded with UTF-8 + given = "%E6%BC%A2%E5%AD%97" + expect = "\u6f22\u5b57" # "Kanji" + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + # Decode with UTF-8, invalid sequence + given = "%F3%B1" + expect = "\ufffd" # Replacement character + result = urllib.parse.unquote(given) + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + # Decode with UTF-8, invalid sequence, replace errors + result = urllib.parse.unquote(given, errors="replace") + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + # Decode with UTF-8, invalid sequence, ignoring errors + given = "%F3%B1" + expect = "" + result = urllib.parse.unquote(given, errors="ignore") + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + # A mix of non-ASCII and percent-encoded characters, UTF-8 + result = urllib.parse.unquote("\u6f22%C3%BC") + expect = '\u6f22\u00fc' + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + + # A mix of non-ASCII and percent-encoded characters, Latin-1 + # (Note, the string contains non-Latin-1-representable characters) + result = urllib.parse.unquote("\u6f22%FC", encoding="latin-1") + expect = '\u6f22\u00fc' + self.assertEqual(expect, result, + "using unquote(): %r != %r" % (expect, result)) + +class urlencode_Tests(unittest.TestCase): + """Tests for urlencode()""" + + def help_inputtype(self, given, test_type): + """Helper method for testing different input types. + + 'given' must lead to only the pairs: + * 1st, 1 + * 2nd, 2 + * 3rd, 3 + + Test cannot assume anything about order. Docs make no guarantee and + have possible dictionary input. + + """ + expect_somewhere = ["1st=1", "2nd=2", "3rd=3"] + result = urllib.parse.urlencode(given) + for expected in expect_somewhere: + self.assertIn(expected, result, + "testing %s: %s not found in %s" % + (test_type, expected, result)) + self.assertEqual(result.count('&'), 2, + "testing %s: expected 2 '&'s; got %s" % + (test_type, result.count('&'))) + amp_location = result.index('&') + on_amp_left = result[amp_location - 1] + on_amp_right = result[amp_location + 1] + self.assertTrue(on_amp_left.isdigit() and on_amp_right.isdigit(), + "testing %s: '&' not located in proper place in %s" % + (test_type, result)) + self.assertEqual(len(result), (5 * 3) + 2, #5 chars per thing and amps + "testing %s: " + "unexpected number of characters: %s != %s" % + (test_type, len(result), (5 * 3) + 2)) + + def test_using_mapping(self): + # Test passing in a mapping object as an argument. + self.help_inputtype({"1st":'1', "2nd":'2', "3rd":'3'}, + "using dict as input type") + + def test_using_sequence(self): + # Test passing in a sequence of two-item sequences as an argument. + self.help_inputtype([('1st', '1'), ('2nd', '2'), ('3rd', '3')], + "using sequence of two-item tuples as input") + + def test_quoting(self): + # Make sure keys and values are quoted using quote_plus() + given = {"&":"="} + expect = "%s=%s" % (hexescape('&'), hexescape('=')) + result = urllib.parse.urlencode(given) + self.assertEqual(expect, result) + given = {"key name":"A bunch of pluses"} + expect = "key+name=A+bunch+of+pluses" + result = urllib.parse.urlencode(given) + self.assertEqual(expect, result) + + def test_doseq(self): + # Test that passing True for 'doseq' parameter works correctly + given = {'sequence':['1', '2', '3']} + expect = "sequence=%s" % urllib.parse.quote_plus(str(['1', '2', '3'])) + result = urllib.parse.urlencode(given) + self.assertEqual(expect, result) + result = urllib.parse.urlencode(given, True) + for value in given["sequence"]: + expect = "sequence=%s" % value + self.assertIn(expect, result) + self.assertEqual(result.count('&'), 2, + "Expected 2 '&'s, got %s" % result.count('&')) + + def test_empty_sequence(self): + self.assertEqual("", urllib.parse.urlencode({})) + self.assertEqual("", urllib.parse.urlencode([])) + + def test_nonstring_values(self): + self.assertEqual("a=1", urllib.parse.urlencode({"a": 1})) + self.assertEqual("a=None", urllib.parse.urlencode({"a": None})) + + def test_nonstring_seq_values(self): + self.assertEqual("a=1&a=2", urllib.parse.urlencode({"a": [1, 2]}, True)) + self.assertEqual("a=None&a=a", + urllib.parse.urlencode({"a": [None, "a"]}, True)) + data = collections.OrderedDict([("a", 1), ("b", 1)]) + self.assertEqual("a=a&a=b", + urllib.parse.urlencode({"a": data}, True)) + + def test_urlencode_encoding(self): + # ASCII encoding. Expect %3F with errors="replace' + given = (('\u00a0', '\u00c1'),) + expect = '%3F=%3F' + result = urllib.parse.urlencode(given, encoding="ASCII", errors="replace") + self.assertEqual(expect, result) + + # Default is UTF-8 encoding. + given = (('\u00a0', '\u00c1'),) + expect = '%C2%A0=%C3%81' + result = urllib.parse.urlencode(given) + self.assertEqual(expect, result) + + # Latin-1 encoding. + given = (('\u00a0', '\u00c1'),) + expect = '%A0=%C1' + result = urllib.parse.urlencode(given, encoding="latin-1") + self.assertEqual(expect, result) + + def test_urlencode_encoding_doseq(self): + # ASCII Encoding. Expect %3F with errors="replace' + given = (('\u00a0', '\u00c1'),) + expect = '%3F=%3F' + result = urllib.parse.urlencode(given, doseq=True, + encoding="ASCII", errors="replace") + self.assertEqual(expect, result) + + # ASCII Encoding. On a sequence of values. + given = (("\u00a0", (1, "\u00c1")),) + expect = '%3F=1&%3F=%3F' + result = urllib.parse.urlencode(given, True, + encoding="ASCII", errors="replace") + self.assertEqual(expect, result) + + # Utf-8 + given = (("\u00a0", "\u00c1"),) + expect = '%C2%A0=%C3%81' + result = urllib.parse.urlencode(given, True) + self.assertEqual(expect, result) + + given = (("\u00a0", (42, "\u00c1")),) + expect = '%C2%A0=42&%C2%A0=%C3%81' + result = urllib.parse.urlencode(given, True) + self.assertEqual(expect, result) + + # latin-1 + given = (("\u00a0", "\u00c1"),) + expect = '%A0=%C1' + result = urllib.parse.urlencode(given, True, encoding="latin-1") + self.assertEqual(expect, result) + + given = (("\u00a0", (42, "\u00c1")),) + expect = '%A0=42&%A0=%C1' + result = urllib.parse.urlencode(given, True, encoding="latin-1") + self.assertEqual(expect, result) + + def test_urlencode_bytes(self): + given = ((b'\xa0\x24', b'\xc1\x24'),) + expect = '%A0%24=%C1%24' + result = urllib.parse.urlencode(given) + self.assertEqual(expect, result) + result = urllib.parse.urlencode(given, True) + self.assertEqual(expect, result) + + # Sequence of values + given = ((b'\xa0\x24', (42, b'\xc1\x24')),) + expect = '%A0%24=42&%A0%24=%C1%24' + result = urllib.parse.urlencode(given, True) + self.assertEqual(expect, result) + + def test_urlencode_encoding_safe_parameter(self): + + # Send '$' (\x24) as safe character + # Default utf-8 encoding + + given = ((b'\xa0\x24', b'\xc1\x24'),) + result = urllib.parse.urlencode(given, safe=":$") + expect = '%A0$=%C1$' + self.assertEqual(expect, result) + + given = ((b'\xa0\x24', b'\xc1\x24'),) + result = urllib.parse.urlencode(given, doseq=True, safe=":$") + expect = '%A0$=%C1$' + self.assertEqual(expect, result) + + # Safe parameter in sequence + given = ((b'\xa0\x24', (b'\xc1\x24', 0xd, 42)),) + expect = '%A0$=%C1$&%A0$=13&%A0$=42' + result = urllib.parse.urlencode(given, True, safe=":$") + self.assertEqual(expect, result) + + # Test all above in latin-1 encoding + + given = ((b'\xa0\x24', b'\xc1\x24'),) + result = urllib.parse.urlencode(given, safe=":$", + encoding="latin-1") + expect = '%A0$=%C1$' + self.assertEqual(expect, result) + + given = ((b'\xa0\x24', b'\xc1\x24'),) + expect = '%A0$=%C1$' + result = urllib.parse.urlencode(given, doseq=True, safe=":$", + encoding="latin-1") + + given = ((b'\xa0\x24', (b'\xc1\x24', 0xd, 42)),) + expect = '%A0$=%C1$&%A0$=13&%A0$=42' + result = urllib.parse.urlencode(given, True, safe=":$", + encoding="latin-1") + self.assertEqual(expect, result) + +class Pathname_Tests(unittest.TestCase): + """Test pathname2url() and url2pathname()""" + + def test_basic(self): + # Make sure simple tests pass + expected_path = os.path.join("parts", "of", "a", "path") + expected_url = "parts/of/a/path" + result = urllib.request.pathname2url(expected_path) + self.assertEqual(expected_url, result, + "pathname2url() failed; %s != %s" % + (result, expected_url)) + result = urllib.request.url2pathname(expected_url) + self.assertEqual(expected_path, result, + "url2pathame() failed; %s != %s" % + (result, expected_path)) + + def test_quoting(self): + # Test automatic quoting and unquoting works for pathnam2url() and + # url2pathname() respectively + given = os.path.join("needs", "quot=ing", "here") + expect = "needs/%s/here" % urllib.parse.quote("quot=ing") + result = urllib.request.pathname2url(given) + self.assertEqual(expect, result, + "pathname2url() failed; %s != %s" % + (expect, result)) + expect = given + result = urllib.request.url2pathname(result) + self.assertEqual(expect, result, + "url2pathname() failed; %s != %s" % + (expect, result)) + given = os.path.join("make sure", "using_quote") + expect = "%s/using_quote" % urllib.parse.quote("make sure") + result = urllib.request.pathname2url(given) + self.assertEqual(expect, result, + "pathname2url() failed; %s != %s" % + (expect, result)) + given = "make+sure/using_unquote" + expect = os.path.join("make+sure", "using_unquote") + result = urllib.request.url2pathname(given) + self.assertEqual(expect, result, + "url2pathname() failed; %s != %s" % + (expect, result)) + + @unittest.skipUnless(sys.platform == 'win32', + 'test specific to the urllib.url2path function.') + def test_ntpath(self): + given = ('/C:/', '///C:/', '/C|//') + expect = 'C:\\' + for url in given: + result = urllib.request.url2pathname(url) + self.assertEqual(expect, result, + 'urllib.request..url2pathname() failed; %s != %s' % + (expect, result)) + given = '///C|/path' + expect = 'C:\\path' + result = urllib.request.url2pathname(given) + self.assertEqual(expect, result, + 'urllib.request.url2pathname() failed; %s != %s' % + (expect, result)) + +class Utility_Tests(unittest.TestCase): + """Testcase to test the various utility functions in the urllib.""" + + def test_thishost(self): + """Test the urllib.request.thishost utility function returns a tuple""" + self.assertIsInstance(urllib.request.thishost(), tuple) + + +class URLopener_Tests(unittest.TestCase): + """Testcase to test the open method of URLopener class.""" + + def test_quoted_open(self): + class DummyURLopener(urllib.request.URLopener): + def open_spam(self, url): + return url + with support.check_warnings( + ('DummyURLopener style of invoking requests is deprecated.', + DeprecationWarning)): + self.assertEqual(DummyURLopener().open( + 'spam://example/ /'),'//example/%20/') + + # test the safe characters are not quoted by urlopen + self.assertEqual(DummyURLopener().open( + "spam://c:|windows%/:=&?~#+!$,;'@()*[]|/path/"), + "//c:|windows%/:=&?~#+!$,;'@()*[]|/path/") + +# Just commented them out. +# Can't really tell why keep failing in windows and sparc. +# Everywhere else they work ok, but on those machines, sometimes +# fail in one of the tests, sometimes in other. I have a linux, and +# the tests go ok. +# If anybody has one of the problematic environments, please help! +# . Facundo +# +# def server(evt): +# import socket, time +# serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +# serv.settimeout(3) +# serv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +# serv.bind(("", 9093)) +# serv.listen() +# try: +# conn, addr = serv.accept() +# conn.send("1 Hola mundo\n") +# cantdata = 0 +# while cantdata < 13: +# data = conn.recv(13-cantdata) +# cantdata += len(data) +# time.sleep(.3) +# conn.send("2 No more lines\n") +# conn.close() +# except socket.timeout: +# pass +# finally: +# serv.close() +# evt.set() +# +# class FTPWrapperTests(unittest.TestCase): +# +# def setUp(self): +# import ftplib, time, threading +# ftplib.FTP.port = 9093 +# self.evt = threading.Event() +# threading.Thread(target=server, args=(self.evt,)).start() +# time.sleep(.1) +# +# def tearDown(self): +# self.evt.wait() +# +# def testBasic(self): +# # connects +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# ftp.close() +# +# def testTimeoutNone(self): +# # global default timeout is ignored +# import socket +# self.assertIsNone(socket.getdefaulttimeout()) +# socket.setdefaulttimeout(30) +# try: +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# finally: +# socket.setdefaulttimeout(None) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() +# +# def testTimeoutDefault(self): +# # global default timeout is used +# import socket +# self.assertIsNone(socket.getdefaulttimeout()) +# socket.setdefaulttimeout(30) +# try: +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, []) +# finally: +# socket.setdefaulttimeout(None) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() +# +# def testTimeoutValue(self): +# ftp = urllib.ftpwrapper("myuser", "mypass", "localhost", 9093, [], +# timeout=30) +# self.assertEqual(ftp.ftp.sock.gettimeout(), 30) +# ftp.close() + + +class RequestTests(unittest.TestCase): + """Unit tests for urllib.request.Request.""" + + def test_default_values(self): + Request = urllib.request.Request + request = Request("http://www.python.org") + self.assertEqual(request.get_method(), 'GET') + request = Request("http://www.python.org", {}) + self.assertEqual(request.get_method(), 'POST') + + def test_with_method_arg(self): + Request = urllib.request.Request + request = Request("http://www.python.org", method='HEAD') + self.assertEqual(request.method, 'HEAD') + self.assertEqual(request.get_method(), 'HEAD') + request = Request("http://www.python.org", {}, method='HEAD') + self.assertEqual(request.method, 'HEAD') + self.assertEqual(request.get_method(), 'HEAD') + request = Request("http://www.python.org", method='GET') + self.assertEqual(request.get_method(), 'GET') + request.method = 'HEAD' + self.assertEqual(request.get_method(), 'HEAD') + + +class URL2PathNameTests(unittest.TestCase): + + def test_converting_drive_letter(self): + self.assertEqual(url2pathname("///C|"), 'C:') + self.assertEqual(url2pathname("///C:"), 'C:') + self.assertEqual(url2pathname("///C|/"), 'C:\\') + + def test_converting_when_no_drive_letter(self): + # cannot end a raw string in \ + self.assertEqual(url2pathname("///C/test/"), r'\\\C\test' '\\') + self.assertEqual(url2pathname("////C/test/"), r'\\C\test' '\\') + + def test_simple_compare(self): + self.assertEqual(url2pathname("///C|/foo/bar/spam.foo"), + r'C:\foo\bar\spam.foo') + + def test_non_ascii_drive_letter(self): + self.assertRaises(IOError, url2pathname, "///\u00e8|/") + + def test_roundtrip_url2pathname(self): + list_of_paths = ['C:', + r'\\\C\test\\', + r'C:\foo\bar\spam.foo' + ] + for path in list_of_paths: + self.assertEqual(url2pathname(pathname2url(path)), path) + +class PathName2URLTests(unittest.TestCase): + + def test_converting_drive_letter(self): + self.assertEqual(pathname2url("C:"), '///C:') + self.assertEqual(pathname2url("C:\\"), '///C:') + + def test_converting_when_no_drive_letter(self): + self.assertEqual(pathname2url(r"\\\folder\test" "\\"), + '/////folder/test/') + self.assertEqual(pathname2url(r"\\folder\test" "\\"), + '////folder/test/') + self.assertEqual(pathname2url(r"\folder\test" "\\"), + '/folder/test/') + + def test_simple_compare(self): + self.assertEqual(pathname2url(r'C:\foo\bar\spam.foo'), + "///C:/foo/bar/spam.foo" ) + + def test_long_drive_letter(self): + self.assertRaises(IOError, pathname2url, "XX:\\") + + def test_roundtrip_pathname2url(self): + list_of_paths = ['///C:', + '/////folder/test/', + '///C:/foo/bar/spam.foo'] + for path in list_of_paths: + self.assertEqual(pathname2url(url2pathname(path)), path) + +if __name__ == '__main__': + unittest.main() diff --git a/src/greentest/3.5pypy/test_urllib2.py b/src/greentest/3.5pypy/test_urllib2.py new file mode 100644 index 0000000..eda7ccc --- /dev/null +++ b/src/greentest/3.5pypy/test_urllib2.py @@ -0,0 +1,1850 @@ +import unittest +from test import support +from test import test_urllib + +import os +import io +import socket +import array +import sys + +import urllib.request +# The proxy bypass method imported below has logic specific to the OSX +# proxy config data structure but is testable on all platforms. +from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, + HTTPPasswordMgrWithPriorAuth, _parse_proxy, + _proxy_bypass_macosx_sysconf, + AbstractDigestAuthHandler) +from urllib.parse import urlparse +import urllib.error +import http.client + +# XXX +# Request +# CacheFTPHandler (hard to write) +# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler + + +class TrivialTests(unittest.TestCase): + + def test___all__(self): + # Verify which names are exposed + for module in 'request', 'response', 'parse', 'error', 'robotparser': + context = {} + exec('from urllib.%s import *' % module, context) + del context['__builtins__'] + if module == 'request' and os.name == 'nt': + u, p = context.pop('url2pathname'), context.pop('pathname2url') + self.assertEqual(u.__module__, 'nturl2path') + self.assertEqual(p.__module__, 'nturl2path') + for k, v in context.items(): + self.assertEqual(v.__module__, 'urllib.%s' % module, + "%r is exposed in 'urllib.%s' but defined in %r" % + (k, module, v.__module__)) + + def test_trivial(self): + # A couple trivial tests + + self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url') + + # XXX Name hacking to get this to work on Windows. + fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/') + + if os.name == 'nt': + file_url = "file:///%s" % fname + else: + file_url = "file://%s" % fname + + f = urllib.request.urlopen(file_url) + + f.read() + f.close() + + def test_parse_http_list(self): + tests = [ + ('a,b,c', ['a', 'b', 'c']), + ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']), + ('a, b, "c", "d", "e,f", g, h', + ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']), + ('a="b\\"c", d="e\\,f", g="h\\\\i"', + ['a="b"c"', 'd="e,f"', 'g="h\\i"'])] + for string, list in tests: + self.assertEqual(urllib.request.parse_http_list(string), list) + + def test_URLError_reasonstr(self): + err = urllib.error.URLError('reason') + self.assertIn(err.reason, str(err)) + + +class RequestHdrsTests(unittest.TestCase): + + def test_request_headers_dict(self): + """ + The Request.headers dictionary is not a documented interface. It + should stay that way, because the complete set of headers are only + accessible through the .get_header(), .has_header(), .header_items() + interface. However, .headers pre-dates those methods, and so real code + will be using the dictionary. + + The introduction in 2.4 of those methods was a mistake for the same + reason: code that previously saw all (urllib2 user)-provided headers in + .headers now sees only a subset. + + """ + url = "http://example.com" + self.assertEqual(Request(url, + headers={"Spam-eggs": "blah"} + ).headers["Spam-eggs"], "blah") + self.assertEqual(Request(url, + headers={"spam-EggS": "blah"} + ).headers["Spam-eggs"], "blah") + + def test_request_headers_methods(self): + """ + Note the case normalization of header names here, to + .capitalize()-case. This should be preserved for + backwards-compatibility. (In the HTTP case, normalization to + .title()-case is done by urllib2 before sending headers to + http.client). + + Note that e.g. r.has_header("spam-EggS") is currently False, and + r.get_header("spam-EggS") returns None, but that could be changed in + future. + + Method r.remove_header should remove items both from r.headers and + r.unredirected_hdrs dictionaries + """ + url = "http://example.com" + req = Request(url, headers={"Spam-eggs": "blah"}) + self.assertTrue(req.has_header("Spam-eggs")) + self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')]) + + req.add_header("Foo-Bar", "baz") + self.assertEqual(sorted(req.header_items()), + [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')]) + self.assertFalse(req.has_header("Not-there")) + self.assertIsNone(req.get_header("Not-there")) + self.assertEqual(req.get_header("Not-there", "default"), "default") + + req.remove_header("Spam-eggs") + self.assertFalse(req.has_header("Spam-eggs")) + + req.add_unredirected_header("Unredirected-spam", "Eggs") + self.assertTrue(req.has_header("Unredirected-spam")) + + req.remove_header("Unredirected-spam") + self.assertFalse(req.has_header("Unredirected-spam")) + + def test_password_manager(self): + mgr = urllib.request.HTTPPasswordMgr() + add = mgr.add_password + find_user_pass = mgr.find_user_password + add("Some Realm", "http://example.com/", "joe", "password") + add("Some Realm", "http://example.com/ni", "ni", "ni") + add("c", "http://example.com/foo", "foo", "ni") + add("c", "http://example.com/bar", "bar", "nini") + add("b", "http://example.com/", "first", "blah") + add("b", "http://example.com/", "second", "spam") + add("a", "http://example.com", "1", "a") + add("Some Realm", "http://c.example.com:3128", "3", "c") + add("Some Realm", "d.example.com", "4", "d") + add("Some Realm", "e.example.com:3128", "5", "e") + + self.assertEqual(find_user_pass("Some Realm", "example.com"), + ('joe', 'password')) + + #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), + # ('ni', 'ni')) + + self.assertEqual(find_user_pass("Some Realm", "http://example.com"), + ('joe', 'password')) + self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), + ('joe', 'password')) + self.assertEqual( + find_user_pass("Some Realm", "http://example.com/spam"), + ('joe', 'password')) + self.assertEqual( + find_user_pass("Some Realm", "http://example.com/spam/spam"), + ('joe', 'password')) + self.assertEqual(find_user_pass("c", "http://example.com/foo"), + ('foo', 'ni')) + self.assertEqual(find_user_pass("c", "http://example.com/bar"), + ('bar', 'nini')) + self.assertEqual(find_user_pass("b", "http://example.com/"), + ('second', 'spam')) + + # No special relationship between a.example.com and example.com: + + self.assertEqual(find_user_pass("a", "http://example.com/"), + ('1', 'a')) + self.assertEqual(find_user_pass("a", "http://a.example.com/"), + (None, None)) + + # Ports: + + self.assertEqual(find_user_pass("Some Realm", "c.example.com"), + (None, None)) + self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"), + ('3', 'c')) + self.assertEqual( + find_user_pass("Some Realm", "http://c.example.com:3128"), + ('3', 'c')) + self.assertEqual(find_user_pass("Some Realm", "d.example.com"), + ('4', 'd')) + self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"), + ('5', 'e')) + + def test_password_manager_default_port(self): + """ + The point to note here is that we can't guess the default port if + there's no scheme. This applies to both add_password and + find_user_password. + """ + mgr = urllib.request.HTTPPasswordMgr() + add = mgr.add_password + find_user_pass = mgr.find_user_password + add("f", "http://g.example.com:80", "10", "j") + add("g", "http://h.example.com", "11", "k") + add("h", "i.example.com:80", "12", "l") + add("i", "j.example.com", "13", "m") + self.assertEqual(find_user_pass("f", "g.example.com:100"), + (None, None)) + self.assertEqual(find_user_pass("f", "g.example.com:80"), + ('10', 'j')) + self.assertEqual(find_user_pass("f", "g.example.com"), + (None, None)) + self.assertEqual(find_user_pass("f", "http://g.example.com:100"), + (None, None)) + self.assertEqual(find_user_pass("f", "http://g.example.com:80"), + ('10', 'j')) + self.assertEqual(find_user_pass("f", "http://g.example.com"), + ('10', 'j')) + self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k')) + self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k')) + self.assertEqual(find_user_pass("g", "http://h.example.com:80"), + ('11', 'k')) + self.assertEqual(find_user_pass("h", "i.example.com"), (None, None)) + self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l')) + self.assertEqual(find_user_pass("h", "http://i.example.com:80"), + ('12', 'l')) + self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm')) + self.assertEqual(find_user_pass("i", "j.example.com:80"), + (None, None)) + self.assertEqual(find_user_pass("i", "http://j.example.com"), + ('13', 'm')) + self.assertEqual(find_user_pass("i", "http://j.example.com:80"), + (None, None)) + + +class MockOpener: + addheaders = [] + + def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.req, self.data, self.timeout = req, data, timeout + + def error(self, proto, *args): + self.proto, self.args = proto, args + + +class MockFile: + def read(self, count=None): + pass + + def readline(self, count=None): + pass + + def close(self): + pass + + +class MockHeaders(dict): + def getheaders(self, name): + return list(self.values()) + + +class MockResponse(io.StringIO): + def __init__(self, code, msg, headers, data, url=None): + io.StringIO.__init__(self, data) + self.code, self.msg, self.headers, self.url = code, msg, headers, url + + def info(self): + return self.headers + + def geturl(self): + return self.url + + +class MockCookieJar: + def add_cookie_header(self, request): + self.ach_req = request + + def extract_cookies(self, response, request): + self.ec_req, self.ec_r = request, response + + +class FakeMethod: + def __init__(self, meth_name, action, handle): + self.meth_name = meth_name + self.handle = handle + self.action = action + + def __call__(self, *args): + return self.handle(self.meth_name, self.action, *args) + + +class MockHTTPResponse(io.IOBase): + def __init__(self, fp, msg, status, reason): + self.fp = fp + self.msg = msg + self.status = status + self.reason = reason + self.code = 200 + + def read(self): + return '' + + def info(self): + return {} + + def geturl(self): + return self.url + + +class MockHTTPClass: + def __init__(self): + self.level = 0 + self.req_headers = [] + self.data = None + self.raise_on_endheaders = False + self.sock = None + self._tunnel_headers = {} + + def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.host = host + self.timeout = timeout + return self + + def set_debuglevel(self, level): + self.level = level + + def set_tunnel(self, host, port=None, headers=None): + self._tunnel_host = host + self._tunnel_port = port + if headers: + self._tunnel_headers = headers + else: + self._tunnel_headers.clear() + + def request(self, method, url, body=None, headers=None): + self.method = method + self.selector = url + if headers is not None: + self.req_headers += headers.items() + self.req_headers.sort() + if body: + self.data = body + if self.raise_on_endheaders: + raise OSError() + + def getresponse(self): + return MockHTTPResponse(MockFile(), {}, 200, "OK") + + def close(self): + pass + + +class MockHandler: + # useful for testing handler machinery + # see add_ordered_mock_handlers() docstring + handler_order = 500 + + def __init__(self, methods): + self._define_methods(methods) + + def _define_methods(self, methods): + for spec in methods: + if len(spec) == 2: + name, action = spec + else: + name, action = spec, None + meth = FakeMethod(name, action, self.handle) + setattr(self.__class__, name, meth) + + def handle(self, fn_name, action, *args, **kwds): + self.parent.calls.append((self, fn_name, args, kwds)) + if action is None: + return None + elif action == "return self": + return self + elif action == "return response": + res = MockResponse(200, "OK", {}, "") + return res + elif action == "return request": + return Request("http://blah/") + elif action.startswith("error"): + code = action[action.rfind(" ")+1:] + try: + code = int(code) + except ValueError: + pass + res = MockResponse(200, "OK", {}, "") + return self.parent.error("http", args[0], res, code, "", {}) + elif action == "raise": + raise urllib.error.URLError("blah") + assert False + + def close(self): + pass + + def add_parent(self, parent): + self.parent = parent + self.parent.calls = [] + + def __lt__(self, other): + if not hasattr(other, "handler_order"): + # No handler_order, leave in original order. Yuck. + return True + return self.handler_order < other.handler_order + + +def add_ordered_mock_handlers(opener, meth_spec): + """Create MockHandlers and add them to an OpenerDirector. + + meth_spec: list of lists of tuples and strings defining methods to define + on handlers. eg: + + [["http_error", "ftp_open"], ["http_open"]] + + defines methods .http_error() and .ftp_open() on one handler, and + .http_open() on another. These methods just record their arguments and + return None. Using a tuple instead of a string causes the method to + perform some action (see MockHandler.handle()), eg: + + [["http_error"], [("http_open", "return request")]] + + defines .http_error() on one handler (which simply returns None), and + .http_open() on another handler, which returns a Request object. + + """ + handlers = [] + count = 0 + for meths in meth_spec: + class MockHandlerSubclass(MockHandler): + pass + + h = MockHandlerSubclass(meths) + h.handler_order += count + h.add_parent(opener) + count = count + 1 + handlers.append(h) + opener.add_handler(h) + return handlers + + +def build_test_opener(*handler_instances): + opener = OpenerDirector() + for h in handler_instances: + opener.add_handler(h) + return opener + + +class MockHTTPHandler(urllib.request.BaseHandler): + # useful for testing redirections and auth + # sends supplied headers and code as first response + # sends 200 OK as second response + def __init__(self, code, headers): + self.code = code + self.headers = headers + self.reset() + + def reset(self): + self._count = 0 + self.requests = [] + + def http_open(self, req): + import email, copy + self.requests.append(copy.deepcopy(req)) + if self._count == 0: + self._count = self._count + 1 + name = http.client.responses[self.code] + msg = email.message_from_string(self.headers) + return self.parent.error( + "http", req, MockFile(), self.code, name, msg) + else: + self.req = req + msg = email.message_from_string("\r\n\r\n") + return MockResponse(200, "OK", msg, "", req.get_full_url()) + + +class MockHTTPSHandler(urllib.request.AbstractHTTPHandler): + # Useful for testing the Proxy-Authorization request by verifying the + # properties of httpcon + + def __init__(self, debuglevel=0): + urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel) + self.httpconn = MockHTTPClass() + + def https_open(self, req): + return self.do_open(self.httpconn, req) + + +class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): + # useful for testing auth + # sends supplied code response + # checks if auth header is specified in request + def __init__(self, code): + self.code = code + self.has_auth_header = False + + def reset(self): + self.has_auth_header = False + + def http_open(self, req): + if req.has_header('Authorization'): + self.has_auth_header = True + name = http.client.responses[self.code] + return MockResponse(self.code, name, MockFile(), "", req.get_full_url()) + + + +class MockPasswordManager: + def add_password(self, realm, uri, user, password): + self.realm = realm + self.url = uri + self.user = user + self.password = password + + def find_user_password(self, realm, authuri): + self.target_realm = realm + self.target_url = authuri + return self.user, self.password + + +class OpenerDirectorTests(unittest.TestCase): + + def test_add_non_handler(self): + class NonHandler(object): + pass + self.assertRaises(TypeError, + OpenerDirector().add_handler, NonHandler()) + + def test_badly_named_methods(self): + # test work-around for three methods that accidentally follow the + # naming conventions for handler methods + # (*_open() / *_request() / *_response()) + + # These used to call the accidentally-named methods, causing a + # TypeError in real code; here, returning self from these mock + # methods would either cause no exception, or AttributeError. + + from urllib.error import URLError + + o = OpenerDirector() + meth_spec = [ + [("do_open", "return self"), ("proxy_open", "return self")], + [("redirect_request", "return self")], + ] + add_ordered_mock_handlers(o, meth_spec) + o.add_handler(urllib.request.UnknownHandler()) + for scheme in "do", "proxy", "redirect": + self.assertRaises(URLError, o.open, scheme+"://example.com/") + + def test_handled(self): + # handler returning non-None means no more handlers will be called + o = OpenerDirector() + meth_spec = [ + ["http_open", "ftp_open", "http_error_302"], + ["ftp_open"], + [("http_open", "return self")], + [("http_open", "return self")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + r = o.open(req) + # Second .http_open() gets called, third doesn't, since second returned + # non-None. Handlers without .http_open() never get any methods called + # on them. + # In fact, second mock handler defining .http_open() returns self + # (instead of response), which becomes the OpenerDirector's return + # value. + self.assertEqual(r, handlers[2]) + calls = [(handlers[0], "http_open"), (handlers[2], "http_open")] + for expected, got in zip(calls, o.calls): + handler, name, args, kwds = got + self.assertEqual((handler, name), expected) + self.assertEqual(args, (req,)) + + def test_handler_order(self): + o = OpenerDirector() + handlers = [] + for meths, handler_order in [([("http_open", "return self")], 500), + (["http_open"], 0)]: + class MockHandlerSubclass(MockHandler): + pass + + h = MockHandlerSubclass(meths) + h.handler_order = handler_order + handlers.append(h) + o.add_handler(h) + + o.open("http://example.com/") + # handlers called in reverse order, thanks to their sort order + self.assertEqual(o.calls[0][0], handlers[1]) + self.assertEqual(o.calls[1][0], handlers[0]) + + def test_raise(self): + # raising URLError stops processing of request + o = OpenerDirector() + meth_spec = [ + [("http_open", "raise")], + [("http_open", "return self")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + self.assertRaises(urllib.error.URLError, o.open, req) + self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) + + def test_http_error(self): + # XXX http_error_default + # http errors are a special case + o = OpenerDirector() + meth_spec = [ + [("http_open", "error 302")], + [("http_error_400", "raise"), "http_open"], + [("http_error_302", "return response"), "http_error_303", + "http_error"], + [("http_error_302")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + class Unknown: + def __eq__(self, other): + return True + + req = Request("http://example.com/") + o.open(req) + assert len(o.calls) == 2 + calls = [(handlers[0], "http_open", (req,)), + (handlers[2], "http_error_302", + (req, Unknown(), 302, "", {}))] + for expected, got in zip(calls, o.calls): + handler, method_name, args = expected + self.assertEqual((handler, method_name), got[:2]) + self.assertEqual(args, got[2]) + + def test_processors(self): + # *_request / *_response methods get called appropriately + o = OpenerDirector() + meth_spec = [ + [("http_request", "return request"), + ("http_response", "return response")], + [("http_request", "return request"), + ("http_response", "return response")], + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://example.com/") + o.open(req) + # processor methods are called on *all* handlers that define them, + # not just the first handler that handles the request + calls = [ + (handlers[0], "http_request"), (handlers[1], "http_request"), + (handlers[0], "http_response"), (handlers[1], "http_response")] + + for i, (handler, name, args, kwds) in enumerate(o.calls): + if i < 2: + # *_request + self.assertEqual((handler, name), calls[i]) + self.assertEqual(len(args), 1) + self.assertIsInstance(args[0], Request) + else: + # *_response + self.assertEqual((handler, name), calls[i]) + self.assertEqual(len(args), 2) + self.assertIsInstance(args[0], Request) + # response from opener.open is None, because there's no + # handler that defines http_open to handle it + if args[1] is not None: + self.assertIsInstance(args[1], MockResponse) + + +def sanepathname2url(path): + try: + path.encode("utf-8") + except UnicodeEncodeError: + raise unittest.SkipTest("path is not encodable to utf8") + urlpath = urllib.request.pathname2url(path) + if os.name == "nt" and urlpath.startswith("///"): + urlpath = urlpath[2:] + # XXX don't ask me about the mac... + return urlpath + + +class HandlerTests(unittest.TestCase): + + def test_ftp(self): + class MockFTPWrapper: + def __init__(self, data): + self.data = data + + def retrfile(self, filename, filetype): + self.filename, self.filetype = filename, filetype + return io.StringIO(self.data), len(self.data) + + def close(self): + pass + + class NullFTPHandler(urllib.request.FTPHandler): + def __init__(self, data): + self.data = data + + def connect_ftp(self, user, passwd, host, port, dirs, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT): + self.user, self.passwd = user, passwd + self.host, self.port = host, port + self.dirs = dirs + self.ftpwrapper = MockFTPWrapper(self.data) + return self.ftpwrapper + + import ftplib + data = "rheum rhaponicum" + h = NullFTPHandler(data) + h.parent = MockOpener() + + for url, host, port, user, passwd, type_, dirs, filename, mimetype in [ + ("ftp://localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://%25parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "%parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://%2542parrot@localhost/foo/bar/baz.html", + "localhost", ftplib.FTP_PORT, "%42parrot", "", "I", + ["foo", "bar"], "baz.html", "text/html"), + ("ftp://localhost:80/foo/bar/", + "localhost", 80, "", "", "D", + ["foo", "bar"], "", None), + ("ftp://localhost/baz.gif;type=a", + "localhost", ftplib.FTP_PORT, "", "", "A", + [], "baz.gif", None), # XXX really this should guess image/gif + ]: + req = Request(url) + req.timeout = None + r = h.ftp_open(req) + # ftp authentication not yet implemented by FTPHandler + self.assertEqual(h.user, user) + self.assertEqual(h.passwd, passwd) + self.assertEqual(h.host, socket.gethostbyname(host)) + self.assertEqual(h.port, port) + self.assertEqual(h.dirs, dirs) + self.assertEqual(h.ftpwrapper.filename, filename) + self.assertEqual(h.ftpwrapper.filetype, type_) + headers = r.info() + self.assertEqual(headers.get("Content-type"), mimetype) + self.assertEqual(int(headers["Content-length"]), len(data)) + + def test_file(self): + import email.utils + h = urllib.request.FileHandler() + o = h.parent = MockOpener() + + TESTFN = support.TESTFN + urlpath = sanepathname2url(os.path.abspath(TESTFN)) + towrite = b"hello, world\n" + urls = [ + "file://localhost%s" % urlpath, + "file://%s" % urlpath, + "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), + ] + try: + localaddr = socket.gethostbyname(socket.gethostname()) + except socket.gaierror: + localaddr = '' + if localaddr: + urls.append("file://%s%s" % (localaddr, urlpath)) + + for url in urls: + f = open(TESTFN, "wb") + try: + try: + f.write(towrite) + finally: + f.close() + + r = h.file_open(Request(url)) + try: + data = r.read() + headers = r.info() + respurl = r.geturl() + finally: + r.close() + stats = os.stat(TESTFN) + modified = email.utils.formatdate(stats.st_mtime, usegmt=True) + finally: + os.remove(TESTFN) + self.assertEqual(data, towrite) + self.assertEqual(headers["Content-type"], "text/plain") + self.assertEqual(headers["Content-length"], "13") + self.assertEqual(headers["Last-modified"], modified) + self.assertEqual(respurl, url) + + for url in [ + "file://localhost:80%s" % urlpath, + "file:///file_does_not_exist.txt", + "file://not-a-local-host.com//dir/file.txt", + "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), + os.getcwd(), TESTFN), + "file://somerandomhost.ontheinternet.com%s/%s" % + (os.getcwd(), TESTFN), + ]: + try: + f = open(TESTFN, "wb") + try: + f.write(towrite) + finally: + f.close() + + self.assertRaises(urllib.error.URLError, + h.file_open, Request(url)) + finally: + os.remove(TESTFN) + + h = urllib.request.FileHandler() + o = h.parent = MockOpener() + # XXXX why does // mean ftp (and /// mean not ftp!), and where + # is file: scheme specified? I think this is really a bug, and + # what was intended was to distinguish between URLs like: + # file:/blah.txt (a file) + # file://localhost/blah.txt (a file) + # file:///blah.txt (a file) + # file://ftp.example.com/blah.txt (an ftp URL) + for url, ftp in [ + ("file://ftp.example.com//foo.txt", False), + ("file://ftp.example.com///foo.txt", False), +# XXXX bug: fails with OSError, should be URLError + ("file://ftp.example.com/foo.txt", False), + ("file://somehost//foo/something.txt", False), + ("file://localhost//foo/something.txt", False), + ]: + req = Request(url) + try: + h.file_open(req) + # XXXX remove OSError when bug fixed + except (urllib.error.URLError, OSError): + self.assertFalse(ftp) + else: + self.assertIs(o.req, req) + self.assertEqual(req.type, "ftp") + self.assertEqual(req.type == "ftp", ftp) + + def test_http(self): + + h = urllib.request.AbstractHTTPHandler() + o = h.parent = MockOpener() + + url = "http://example.com/" + for method, data in [("GET", None), ("POST", b"blah")]: + req = Request(url, data, {"Foo": "bar"}) + req.timeout = None + req.add_unredirected_header("Spam", "eggs") + http = MockHTTPClass() + r = h.do_open(http, req) + + # result attributes + r.read; r.readline # wrapped MockFile methods + r.info; r.geturl # addinfourl methods + r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply() + hdrs = r.info() + hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply() + self.assertEqual(r.geturl(), url) + + self.assertEqual(http.host, "example.com") + self.assertEqual(http.level, 0) + self.assertEqual(http.method, method) + self.assertEqual(http.selector, "/") + self.assertEqual(http.req_headers, + [("Connection", "close"), + ("Foo", "bar"), ("Spam", "eggs")]) + self.assertEqual(http.data, data) + + # check OSError converted to URLError + http.raise_on_endheaders = True + self.assertRaises(urllib.error.URLError, h.do_open, http, req) + + # Check for TypeError on POST data which is str. + req = Request("http://example.com/","badpost") + self.assertRaises(TypeError, h.do_request_, req) + + # check adding of standard headers + o.addheaders = [("Spam", "eggs")] + for data in b"", None: # POST, GET + req = Request("http://example.com/", data) + r = MockResponse(200, "OK", {}, "") + newreq = h.do_request_(req) + if data is None: # GET + self.assertNotIn("Content-length", req.unredirected_hdrs) + self.assertNotIn("Content-type", req.unredirected_hdrs) + else: # POST + self.assertEqual(req.unredirected_hdrs["Content-length"], "0") + self.assertEqual(req.unredirected_hdrs["Content-type"], + "application/x-www-form-urlencoded") + # XXX the details of Host could be better tested + self.assertEqual(req.unredirected_hdrs["Host"], "example.com") + self.assertEqual(req.unredirected_hdrs["Spam"], "eggs") + + # don't clobber existing headers + req.add_unredirected_header("Content-length", "foo") + req.add_unredirected_header("Content-type", "bar") + req.add_unredirected_header("Host", "baz") + req.add_unredirected_header("Spam", "foo") + newreq = h.do_request_(req) + self.assertEqual(req.unredirected_hdrs["Content-length"], "foo") + self.assertEqual(req.unredirected_hdrs["Content-type"], "bar") + self.assertEqual(req.unredirected_hdrs["Host"], "baz") + self.assertEqual(req.unredirected_hdrs["Spam"], "foo") + + # Check iterable body support + def iterable_body(): + yield b"one" + yield b"two" + yield b"three" + + for headers in {}, {"Content-Length": 11}: + req = Request("http://example.com/", iterable_body(), headers) + if not headers: + # Having an iterable body without a Content-Length should + # raise an exception + self.assertRaises(ValueError, h.do_request_, req) + else: + newreq = h.do_request_(req) + + # A file object. + # Test only Content-Length attribute of request. + + file_obj = io.BytesIO() + file_obj.write(b"Something\nSomething\nSomething\n") + + for headers in {}, {"Content-Length": 30}: + req = Request("http://example.com/", file_obj, headers) + if not headers: + # Having an iterable body without a Content-Length should + # raise an exception + self.assertRaises(ValueError, h.do_request_, req) + else: + newreq = h.do_request_(req) + self.assertEqual(int(newreq.get_header('Content-length')), 30) + + file_obj.close() + + # array.array Iterable - Content Length is calculated + + iterable_array = array.array("I",[1,2,3,4]) + + for headers in {}, {"Content-Length": 16}: + req = Request("http://example.com/", iterable_array, headers) + newreq = h.do_request_(req) + self.assertEqual(int(newreq.get_header('Content-length')),16) + + def test_http_handler_debuglevel(self): + o = OpenerDirector() + h = MockHTTPSHandler(debuglevel=1) + o.add_handler(h) + o.open("https://www.example.com") + self.assertEqual(h._debuglevel, 1) + + def test_http_doubleslash(self): + # Checks the presence of any unnecessary double slash in url does not + # break anything. Previously, a double slash directly after the host + # could cause incorrect parsing. + h = urllib.request.AbstractHTTPHandler() + h.parent = MockOpener() + + data = b"" + ds_urls = [ + "http://example.com/foo/bar/baz.html", + "http://example.com//foo/bar/baz.html", + "http://example.com/foo//bar/baz.html", + "http://example.com/foo/bar//baz.html" + ] + + for ds_url in ds_urls: + ds_req = Request(ds_url, data) + + # Check whether host is determined correctly if there is no proxy + np_ds_req = h.do_request_(ds_req) + self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com") + + # Check whether host is determined correctly if there is a proxy + ds_req.set_proxy("someproxy:3128", None) + p_ds_req = h.do_request_(ds_req) + self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com") + + def test_full_url_setter(self): + # Checks to ensure that components are set correctly after setting the + # full_url of a Request object + + urls = [ + 'http://example.com?foo=bar#baz', + 'http://example.com?foo=bar&spam=eggs#bash', + 'http://example.com', + ] + + # testing a reusable request instance, but the url parameter is + # required, so just use a dummy one to instantiate + r = Request('http://example.com') + for url in urls: + r.full_url = url + parsed = urlparse(url) + + self.assertEqual(r.get_full_url(), url) + # full_url setter uses splittag to split into components. + # splittag sets the fragment as None while urlparse sets it to '' + self.assertEqual(r.fragment or '', parsed.fragment) + self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) + + def test_full_url_deleter(self): + r = Request('http://www.example.com') + del r.full_url + self.assertIsNone(r.full_url) + self.assertIsNone(r.fragment) + self.assertEqual(r.selector, '') + + def test_fixpath_in_weirdurls(self): + # Issue4493: urllib2 to supply '/' when to urls where path does not + # start with'/' + + h = urllib.request.AbstractHTTPHandler() + h.parent = MockOpener() + + weird_url = 'http://www.python.org?getspam' + req = Request(weird_url) + newreq = h.do_request_(req) + self.assertEqual(newreq.host, 'www.python.org') + self.assertEqual(newreq.selector, '/?getspam') + + url_without_path = 'http://www.python.org' + req = Request(url_without_path) + newreq = h.do_request_(req) + self.assertEqual(newreq.host, 'www.python.org') + self.assertEqual(newreq.selector, '') + + def test_errors(self): + h = urllib.request.HTTPErrorProcessor() + o = h.parent = MockOpener() + + url = "http://example.com/" + req = Request(url) + # all 2xx are passed through + r = MockResponse(200, "OK", {}, "", url) + newr = h.http_response(req, r) + self.assertIs(r, newr) + self.assertFalse(hasattr(o, "proto")) # o.error not called + r = MockResponse(202, "Accepted", {}, "", url) + newr = h.http_response(req, r) + self.assertIs(r, newr) + self.assertFalse(hasattr(o, "proto")) # o.error not called + r = MockResponse(206, "Partial content", {}, "", url) + newr = h.http_response(req, r) + self.assertIs(r, newr) + self.assertFalse(hasattr(o, "proto")) # o.error not called + # anything else calls o.error (and MockOpener returns None, here) + r = MockResponse(502, "Bad gateway", {}, "", url) + self.assertIsNone(h.http_response(req, r)) + self.assertEqual(o.proto, "http") # o.error called + self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) + + def test_cookies(self): + cj = MockCookieJar() + h = urllib.request.HTTPCookieProcessor(cj) + h.parent = MockOpener() + + req = Request("http://example.com/") + r = MockResponse(200, "OK", {}, "") + newreq = h.http_request(req) + self.assertIs(cj.ach_req, req) + self.assertIs(cj.ach_req, newreq) + self.assertEqual(req.origin_req_host, "example.com") + self.assertFalse(req.unverifiable) + newr = h.http_response(req, r) + self.assertIs(cj.ec_req, req) + self.assertIs(cj.ec_r, r) + self.assertIs(r, newr) + + def test_redirect(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + o = h.parent = MockOpener() + + # ordinary redirect behaviour + for code in 301, 302, 303, 307: + for data in None, "blah\nblah\n": + method = getattr(h, "http_error_%s" % code) + req = Request(from_url, data) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + req.add_header("Nonsense", "viking=withhold") + if data is not None: + req.add_header("Content-Length", str(len(data))) + req.add_unredirected_header("Spam", "spam") + try: + method(req, MockFile(), code, "Blah", + MockHeaders({"location": to_url})) + except urllib.error.HTTPError: + # 307 in response to POST requires user OK + self.assertEqual(code, 307) + self.assertIsNotNone(data) + self.assertEqual(o.req.get_full_url(), to_url) + try: + self.assertEqual(o.req.get_method(), "GET") + except AttributeError: + self.assertFalse(o.req.data) + + # now it's a GET, there should not be headers regarding content + # (possibly dragged from before being a POST) + headers = [x.lower() for x in o.req.headers] + self.assertNotIn("content-length", headers) + self.assertNotIn("content-type", headers) + + self.assertEqual(o.req.headers["Nonsense"], + "viking=withhold") + self.assertNotIn("Spam", o.req.headers) + self.assertNotIn("Spam", o.req.unredirected_hdrs) + + # loop detection + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + def redirect(h, req, url=to_url): + h.http_error_302(req, MockFile(), 302, "Blah", + MockHeaders({"location": url})) + # Note that the *original* request shares the same record of + # redirections with the sub-requests caused by the redirections. + + # detect infinite loop redirect of a URL to itself + req = Request(from_url, origin_req_host="example.com") + count = 0 + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + try: + while 1: + redirect(h, req, "http://example.com/") + count = count + 1 + except urllib.error.HTTPError: + # don't stop until max_repeats, because cookies may introduce state + self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) + + # detect endless non-repeating chain of redirects + req = Request(from_url, origin_req_host="example.com") + count = 0 + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + try: + while 1: + redirect(h, req, "http://example.com/%d" % count) + count = count + 1 + except urllib.error.HTTPError: + self.assertEqual(count, + urllib.request.HTTPRedirectHandler.max_redirections) + + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http','https','ftp'] + invalid_schemes = ['file','imap','ldap'] + schemeless_url = "example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib.error.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + + def test_relative_redirect(self): + from_url = "http://example.com/a.html" + relative_url = "/b.html" + h = urllib.request.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + valid_url = urllib.parse.urljoin(from_url,relative_url) + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + + def test_cookie_redirect(self): + # cookies shouldn't leak into redirected requests + from http.cookiejar import CookieJar + from test.test_http_cookiejar import interact_netscape + + cj = CookieJar() + interact_netscape(cj, "http://www.example.com/", "spam=eggs") + hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n") + hdeh = urllib.request.HTTPDefaultErrorHandler() + hrh = urllib.request.HTTPRedirectHandler() + cp = urllib.request.HTTPCookieProcessor(cj) + o = build_test_opener(hh, hdeh, hrh, cp) + o.open("http://www.example.com/") + self.assertFalse(hh.req.has_header("Cookie")) + + def test_redirect_fragment(self): + redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' + hh = MockHTTPHandler(302, 'Location: ' + redirected_url) + hdeh = urllib.request.HTTPDefaultErrorHandler() + hrh = urllib.request.HTTPRedirectHandler() + o = build_test_opener(hh, hdeh, hrh) + fp = o.open('http://www.example.com') + self.assertEqual(fp.geturl(), redirected_url.strip()) + + def test_redirect_no_path(self): + # Issue 14132: Relative redirect strips original path + real_class = http.client.HTTPConnection + response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n" + http.client.HTTPConnection = test_urllib.fakehttp(response1) + self.addCleanup(setattr, http.client, "HTTPConnection", real_class) + urls = iter(("/path", "/path?query")) + def request(conn, method, url, *pos, **kw): + self.assertEqual(url, next(urls)) + real_class.request(conn, method, url, *pos, **kw) + # Change response for subsequent connection + conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!" + http.client.HTTPConnection.request = request + fp = urllib.request.urlopen("http://python.org/path") + self.assertEqual(fp.geturl(), "http://python.org/path?query") + + def test_redirect_encoding(self): + # Some characters in the redirect target may need special handling, + # but most ASCII characters should be treated as already encoded + class Handler(urllib.request.HTTPHandler): + def http_open(self, req): + result = self.do_open(self.connection, req) + self.last_buf = self.connection.buf + # Set up a normal response for the next request + self.connection = test_urllib.fakehttp( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 3\r\n' + b'\r\n' + b'123' + ) + return result + handler = Handler() + opener = urllib.request.build_opener(handler) + tests = ( + (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'), + (b'/spaced%20path/', b'/spaced%20path/'), + (b'/spaced path/', b'/spaced%20path/'), + (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'), + ) + for [location, result] in tests: + with self.subTest(repr(location)): + handler.connection = test_urllib.fakehttp( + b'HTTP/1.1 302 Redirect\r\n' + b'Location: ' + location + b'\r\n' + b'\r\n' + ) + response = opener.open('http://example.com/') + expected = b'GET ' + result + b' ' + request = handler.last_buf + self.assertTrue(request.startswith(expected), repr(request)) + + def test_proxy(self): + o = OpenerDirector() + ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) + o.add_handler(ph) + meth_spec = [ + [("http_open", "return response")] + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("http://acme.example.com/") + self.assertEqual(req.host, "acme.example.com") + o.open(req) + self.assertEqual(req.host, "proxy.example.com:3128") + + self.assertEqual([(handlers[0], "http_open")], + [tup[0:2] for tup in o.calls]) + + def test_proxy_no_proxy(self): + os.environ['no_proxy'] = 'python.org' + o = OpenerDirector() + ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) + o.add_handler(ph) + req = Request("http://www.perl.org/") + self.assertEqual(req.host, "www.perl.org") + o.open(req) + self.assertEqual(req.host, "proxy.example.com") + req = Request("http://www.python.org") + self.assertEqual(req.host, "www.python.org") + o.open(req) + self.assertEqual(req.host, "www.python.org") + del os.environ['no_proxy'] + + def test_proxy_no_proxy_all(self): + os.environ['no_proxy'] = '*' + o = OpenerDirector() + ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) + o.add_handler(ph) + req = Request("http://www.python.org") + self.assertEqual(req.host, "www.python.org") + o.open(req) + self.assertEqual(req.host, "www.python.org") + del os.environ['no_proxy'] + + def test_proxy_https(self): + o = OpenerDirector() + ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) + o.add_handler(ph) + meth_spec = [ + [("https_open", "return response")] + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + + req = Request("https://www.example.com/") + self.assertEqual(req.host, "www.example.com") + o.open(req) + self.assertEqual(req.host, "proxy.example.com:3128") + self.assertEqual([(handlers[0], "https_open")], + [tup[0:2] for tup in o.calls]) + + def test_proxy_https_proxy_authorization(self): + o = OpenerDirector() + ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128')) + o.add_handler(ph) + https_handler = MockHTTPSHandler() + o.add_handler(https_handler) + req = Request("https://www.example.com/") + req.add_header("Proxy-Authorization", "FooBar") + req.add_header("User-Agent", "Grail") + self.assertEqual(req.host, "www.example.com") + self.assertIsNone(req._tunnel_host) + o.open(req) + # Verify Proxy-Authorization gets tunneled to request. + # httpsconn req_headers do not have the Proxy-Authorization header but + # the req will have. + self.assertNotIn(("Proxy-Authorization", "FooBar"), + https_handler.httpconn.req_headers) + self.assertIn(("User-Agent", "Grail"), + https_handler.httpconn.req_headers) + self.assertIsNotNone(req._tunnel_host) + self.assertEqual(req.host, "proxy.example.com:3128") + self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + + # TODO: This should be only for OSX + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") + def test_osx_proxy_bypass(self): + bypass = { + 'exclude_simple': False, + 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10', + '10.0/16'] + } + # Check hosts that should trigger the proxy bypass + for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1', + '10.0.0.1'): + self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), + 'expected bypass of %s to be True' % host) + # Check hosts that should not trigger the proxy bypass + for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1', + 'notinbypass'): + self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), + 'expected bypass of %s to be False' % host) + + # Check the exclude_simple flag + bypass = {'exclude_simple': True, 'exceptions': []} + self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass)) + + def test_basic_auth(self, quote_char='"'): + opener = OpenerDirector() + password_manager = MockPasswordManager() + auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) + realm = "ACME Widget Store" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' % + (quote_char, realm, quote_char)) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + self._test_basic_auth(opener, auth_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected", + ) + + def test_basic_auth_with_single_quoted_realm(self): + self.test_basic_auth(quote_char="'") + + def test_basic_auth_with_unquoted_realm(self): + opener = OpenerDirector() + password_manager = MockPasswordManager() + auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) + realm = "ACME Widget Store" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + with self.assertWarns(UserWarning): + self._test_basic_auth(opener, auth_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected", + ) + + def test_proxy_basic_auth(self): + opener = OpenerDirector() + ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) + opener.add_handler(ph) + password_manager = MockPasswordManager() + auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager) + realm = "ACME Networks" + http_handler = MockHTTPHandler( + 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + self._test_basic_auth(opener, auth_handler, "Proxy-authorization", + realm, http_handler, password_manager, + "http://acme.example.com:3128/protected", + "proxy.example.com:3128", + ) + + def test_basic_and_digest_auth_handlers(self): + # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* + # response (http://python.org/sf/1479302), where it should instead + # return None to allow another handler (especially + # HTTPBasicAuthHandler) to handle the response. + + # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must + # try digest first (since it's the strongest auth scheme), so we record + # order of calls here to check digest comes first: + class RecordingOpenerDirector(OpenerDirector): + def __init__(self): + OpenerDirector.__init__(self) + self.recorded = [] + + def record(self, info): + self.recorded.append(info) + + class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler): + def http_error_401(self, *args, **kwds): + self.parent.record("digest") + urllib.request.HTTPDigestAuthHandler.http_error_401(self, + *args, **kwds) + + class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler): + def http_error_401(self, *args, **kwds): + self.parent.record("basic") + urllib.request.HTTPBasicAuthHandler.http_error_401(self, + *args, **kwds) + + opener = RecordingOpenerDirector() + password_manager = MockPasswordManager() + digest_handler = TestDigestAuthHandler(password_manager) + basic_handler = TestBasicAuthHandler(password_manager) + realm = "ACME Networks" + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) + opener.add_handler(basic_handler) + opener.add_handler(digest_handler) + opener.add_handler(http_handler) + + # check basic auth isn't blocked by digest handler failing + self._test_basic_auth(opener, basic_handler, "Authorization", + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected", + ) + # check digest was tried before basic (twice, because + # _test_basic_auth called .open() twice) + self.assertEqual(opener.recorded, ["digest", "basic"]*2) + + def test_unsupported_auth_digest_handler(self): + opener = OpenerDirector() + # While using DigestAuthHandler + digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Kerberos\r\n\r\n') + opener.add_handler(digest_auth_handler) + opener.add_handler(http_handler) + self.assertRaises(ValueError, opener.open, "http://www.example.com") + + def test_unsupported_auth_basic_handler(self): + # While using BasicAuthHandler + opener = OpenerDirector() + basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: NTLM\r\n\r\n') + opener.add_handler(basic_auth_handler) + opener.add_handler(http_handler) + self.assertRaises(ValueError, opener.open, "http://www.example.com") + + def _test_basic_auth(self, opener, auth_handler, auth_header, + realm, http_handler, password_manager, + request_url, protected_url): + import base64 + user, password = "wile", "coyote" + + # .add_password() fed through to password manager + auth_handler.add_password(realm, request_url, user, password) + self.assertEqual(realm, password_manager.realm) + self.assertEqual(request_url, password_manager.url) + self.assertEqual(user, password_manager.user) + self.assertEqual(password, password_manager.password) + + opener.open(request_url) + + # should have asked the password manager for the username/password + self.assertEqual(password_manager.target_realm, realm) + self.assertEqual(password_manager.target_url, protected_url) + + # expect one request without authorization, then one with + self.assertEqual(len(http_handler.requests), 2) + self.assertFalse(http_handler.requests[0].has_header(auth_header)) + userpass = bytes('%s:%s' % (user, password), "ascii") + auth_hdr_value = ('Basic ' + + base64.encodebytes(userpass).strip().decode()) + self.assertEqual(http_handler.requests[1].get_header(auth_header), + auth_hdr_value) + self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header], + auth_hdr_value) + # if the password manager can't find a password, the handler won't + # handle the HTTP auth error + password_manager.user = password_manager.password = None + http_handler.reset() + opener.open(request_url) + self.assertEqual(len(http_handler.requests), 1) + self.assertFalse(http_handler.requests[0].has_header(auth_header)) + + def test_basic_prior_auth_auto_send(self): + # Assume already authenticated if is_authenticated=True + # for APIs like Github that don't return 401 + + user, password = "wile", "coyote" + request_url = "http://acme.example.com/protected" + + http_handler = MockHTTPHandlerCheckAuth(200) + + pwd_manager = HTTPPasswordMgrWithPriorAuth() + auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) + auth_prior_handler.add_password( + None, request_url, user, password, is_authenticated=True) + + is_auth = pwd_manager.is_authenticated(request_url) + self.assertTrue(is_auth) + + opener = OpenerDirector() + opener.add_handler(auth_prior_handler) + opener.add_handler(http_handler) + + opener.open(request_url) + + # expect request to be sent with auth header + self.assertTrue(http_handler.has_auth_header) + + def test_basic_prior_auth_send_after_first_success(self): + # Auto send auth header after authentication is successful once + + user, password = 'wile', 'coyote' + request_url = 'http://acme.example.com/protected' + realm = 'ACME' + + pwd_manager = HTTPPasswordMgrWithPriorAuth() + auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) + auth_prior_handler.add_password(realm, request_url, user, password) + + is_auth = pwd_manager.is_authenticated(request_url) + self.assertFalse(is_auth) + + opener = OpenerDirector() + opener.add_handler(auth_prior_handler) + + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None) + opener.add_handler(http_handler) + + opener.open(request_url) + + is_auth = pwd_manager.is_authenticated(request_url) + self.assertTrue(is_auth) + + http_handler = MockHTTPHandlerCheckAuth(200) + self.assertFalse(http_handler.has_auth_header) + + opener = OpenerDirector() + opener.add_handler(auth_prior_handler) + opener.add_handler(http_handler) + + # After getting 200 from MockHTTPHandler + # Next request sends header in the first request + opener.open(request_url) + + # expect request to be sent with auth header + self.assertTrue(http_handler.has_auth_header) + + def test_http_closed(self): + """Test the connection is cleaned up when the response is closed""" + for (transfer, data) in ( + ("Connection: close", b"data"), + ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), + ("Content-Length: 4", b"data"), + ): + header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) + conn = test_urllib.fakehttp(header.encode() + data) + handler = urllib.request.AbstractHTTPHandler() + req = Request("http://dummy/") + req.timeout = None + with handler.do_open(conn, req) as resp: + resp.read() + self.assertTrue(conn.fakesock.closed, + "Connection not closed with {!r}".format(transfer)) + + def test_invalid_closed(self): + """Test the connection is cleaned up after an invalid response""" + conn = test_urllib.fakehttp(b"") + handler = urllib.request.AbstractHTTPHandler() + req = Request("http://dummy/") + req.timeout = None + with self.assertRaises(http.client.BadStatusLine): + handler.do_open(conn, req) + self.assertTrue(conn.fakesock.closed, "Connection not closed") + + + +class MiscTests(unittest.TestCase): + + def opener_has_handler(self, opener, handler_class): + self.assertTrue(any(h.__class__ == handler_class + for h in opener.handlers)) + + def test_build_opener(self): + class MyHTTPHandler(urllib.request.HTTPHandler): + pass + + class FooHandler(urllib.request.BaseHandler): + def foo_open(self): + pass + + class BarHandler(urllib.request.BaseHandler): + def bar_open(self): + pass + + build_opener = urllib.request.build_opener + + o = build_opener(FooHandler, BarHandler) + self.opener_has_handler(o, FooHandler) + self.opener_has_handler(o, BarHandler) + + # can take a mix of classes and instances + o = build_opener(FooHandler, BarHandler()) + self.opener_has_handler(o, FooHandler) + self.opener_has_handler(o, BarHandler) + + # subclasses of default handlers override default handlers + o = build_opener(MyHTTPHandler) + self.opener_has_handler(o, MyHTTPHandler) + + # a particular case of overriding: default handlers can be passed + # in explicitly + o = build_opener() + self.opener_has_handler(o, urllib.request.HTTPHandler) + o = build_opener(urllib.request.HTTPHandler) + self.opener_has_handler(o, urllib.request.HTTPHandler) + o = build_opener(urllib.request.HTTPHandler()) + self.opener_has_handler(o, urllib.request.HTTPHandler) + + # Issue2670: multiple handlers sharing the same base class + class MyOtherHTTPHandler(urllib.request.HTTPHandler): + pass + + o = build_opener(MyHTTPHandler, MyOtherHTTPHandler) + self.opener_has_handler(o, MyHTTPHandler) + self.opener_has_handler(o, MyOtherHTTPHandler) + + @unittest.skipUnless(support.is_resource_enabled('network'), + 'test requires network access') + def test_issue16464(self): + with support.transient_internet("http://www.example.com/"): + opener = urllib.request.build_opener() + request = urllib.request.Request("http://www.example.com/") + self.assertEqual(None, request.data) + + opener.open(request, "1".encode("us-ascii")) + self.assertEqual(b"1", request.data) + self.assertEqual("1", request.get_header("Content-length")) + + opener.open(request, "1234567890".encode("us-ascii")) + self.assertEqual(b"1234567890", request.data) + self.assertEqual("10", request.get_header("Content-length")) + + def test_HTTPError_interface(self): + """ + Issue 13211 reveals that HTTPError didn't implement the URLError + interface even though HTTPError is a subclass of URLError. + """ + msg = 'something bad happened' + url = code = fp = None + hdrs = 'Content-Length: 42' + err = urllib.error.HTTPError(url, code, msg, hdrs, fp) + self.assertTrue(hasattr(err, 'reason')) + self.assertEqual(err.reason, 'something bad happened') + self.assertTrue(hasattr(err, 'headers')) + self.assertEqual(err.headers, 'Content-Length: 42') + expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) + self.assertEqual(str(err), expected_errmsg) + expected_errmsg = '' % (err.code, err.msg) + self.assertEqual(repr(err), expected_errmsg) + + def test_parse_proxy(self): + parse_proxy_test_cases = [ + ('proxy.example.com', + (None, None, None, 'proxy.example.com')), + ('proxy.example.com:3128', + (None, None, None, 'proxy.example.com:3128')), + ('proxy.example.com', (None, None, None, 'proxy.example.com')), + ('proxy.example.com:3128', + (None, None, None, 'proxy.example.com:3128')), + # The authority component may optionally include userinfo + # (assumed to be # username:password): + ('joe:password@proxy.example.com', + (None, 'joe', 'password', 'proxy.example.com')), + ('joe:password@proxy.example.com:3128', + (None, 'joe', 'password', 'proxy.example.com:3128')), + #Examples with URLS + ('http://proxy.example.com/', + ('http', None, None, 'proxy.example.com')), + ('http://proxy.example.com:3128/', + ('http', None, None, 'proxy.example.com:3128')), + ('http://joe:password@proxy.example.com/', + ('http', 'joe', 'password', 'proxy.example.com')), + ('http://joe:password@proxy.example.com:3128', + ('http', 'joe', 'password', 'proxy.example.com:3128')), + # Everything after the authority is ignored + ('ftp://joe:password@proxy.example.com/rubbish:3128', + ('ftp', 'joe', 'password', 'proxy.example.com')), + # Test for no trailing '/' case + ('http://joe:password@proxy.example.com', + ('http', 'joe', 'password', 'proxy.example.com')) + ] + + for tc, expected in parse_proxy_test_cases: + self.assertEqual(_parse_proxy(tc), expected) + + self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), + + def test_unsupported_algorithm(self): + handler = AbstractDigestAuthHandler() + with self.assertRaises(ValueError) as exc: + handler.get_algorithm_impls('invalid') + self.assertEqual( + str(exc.exception), + "Unsupported digest authentication algorithm 'invalid'" + ) + + +class RequestTests(unittest.TestCase): + class PutRequest(Request): + method = 'PUT' + + def setUp(self): + self.get = Request("http://www.python.org/~jeremy/") + self.post = Request("http://www.python.org/~jeremy/", + "data", + headers={"X-Test": "test"}) + self.head = Request("http://www.python.org/~jeremy/", method='HEAD') + self.put = self.PutRequest("http://www.python.org/~jeremy/") + self.force_post = self.PutRequest("http://www.python.org/~jeremy/", + method="POST") + + def test_method(self): + self.assertEqual("POST", self.post.get_method()) + self.assertEqual("GET", self.get.get_method()) + self.assertEqual("HEAD", self.head.get_method()) + self.assertEqual("PUT", self.put.get_method()) + self.assertEqual("POST", self.force_post.get_method()) + + def test_data(self): + self.assertFalse(self.get.data) + self.assertEqual("GET", self.get.get_method()) + self.get.data = "spam" + self.assertTrue(self.get.data) + self.assertEqual("POST", self.get.get_method()) + + # issue 16464 + # if we change data we need to remove content-length header + # (cause it's most probably calculated for previous value) + def test_setting_data_should_remove_content_length(self): + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + self.get.add_unredirected_header("Content-length", 42) + self.assertEqual(42, self.get.unredirected_hdrs["Content-length"]) + self.get.data = "spam" + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + + # issue 17485 same for deleting data. + def test_deleting_data_should_remove_content_length(self): + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + self.get.data = 'foo' + self.get.add_unredirected_header("Content-length", 3) + self.assertEqual(3, self.get.unredirected_hdrs["Content-length"]) + del self.get.data + self.assertNotIn("Content-length", self.get.unredirected_hdrs) + + def test_get_full_url(self): + self.assertEqual("http://www.python.org/~jeremy/", + self.get.get_full_url()) + + def test_selector(self): + self.assertEqual("/~jeremy/", self.get.selector) + req = Request("http://www.python.org/") + self.assertEqual("/", req.selector) + + def test_get_type(self): + self.assertEqual("http", self.get.type) + + def test_get_host(self): + self.assertEqual("www.python.org", self.get.host) + + def test_get_host_unquote(self): + req = Request("http://www.%70ython.org/") + self.assertEqual("www.python.org", req.host) + + def test_proxy(self): + self.assertFalse(self.get.has_proxy()) + self.get.set_proxy("www.perl.org", "http") + self.assertTrue(self.get.has_proxy()) + self.assertEqual("www.python.org", self.get.origin_req_host) + self.assertEqual("www.perl.org", self.get.host) + + def test_wrapped_url(self): + req = Request("") + self.assertEqual("www.python.org", req.host) + + def test_url_fragment(self): + req = Request("http://www.python.org/?qs=query#fragment=true") + self.assertEqual("/?qs=query", req.selector) + req = Request("http://www.python.org/#fun=true") + self.assertEqual("/", req.selector) + + # Issue 11703: geturl() omits fragment in the original URL. + url = 'http://docs.python.org/library/urllib2.html#OK' + req = Request(url) + self.assertEqual(req.get_full_url(), url) + + def test_url_fullurl_get_full_url(self): + urls = ['http://docs.python.org', + 'http://docs.python.org/library/urllib2.html#OK', + 'http://www.python.org/?qs=query#fragment=true'] + for url in urls: + req = Request(url) + self.assertEqual(req.get_full_url(), req.full_url) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_urllib2_localnet.py b/src/greentest/3.5pypy/test_urllib2_localnet.py new file mode 100644 index 0000000..68c523a --- /dev/null +++ b/src/greentest/3.5pypy/test_urllib2_localnet.py @@ -0,0 +1,668 @@ +import base64 +import os +import email +import urllib.parse +import urllib.request +import http.server +import unittest +import hashlib + +from test import support + +threading = support.import_module('threading') + +try: + import ssl +except ImportError: + ssl = None + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') + + +# Loopback http server infrastructure + +class LoopbackHttpServer(http.server.HTTPServer): + """HTTP server w/ a few modifications that make it useful for + loopback testing purposes. + """ + + def __init__(self, server_address, RequestHandlerClass): + http.server.HTTPServer.__init__(self, + server_address, + RequestHandlerClass) + + # Set the timeout of our listening socket really low so + # that we can stop the server easily. + self.socket.settimeout(0.1) + + def get_request(self): + """HTTPServer method, overridden.""" + + request, client_address = self.socket.accept() + + # It's a loopback connection, so setting the timeout + # really low shouldn't affect anything, but should make + # deadlocks less likely to occur. + request.settimeout(10.0) + + return (request, client_address) + +class LoopbackHttpServerThread(threading.Thread): + """Stoppable thread that runs a loopback http server.""" + + def __init__(self, request_handler): + threading.Thread.__init__(self) + self._stop_server = False + self.ready = threading.Event() + request_handler.protocol_version = "HTTP/1.0" + self.httpd = LoopbackHttpServer(("127.0.0.1", 0), + request_handler) + self.port = self.httpd.server_port + + def stop(self): + """Stops the webserver if it's currently running.""" + + self._stop_server = True + + self.join() + self.httpd.server_close() + + def run(self): + self.ready.set() + while not self._stop_server: + self.httpd.handle_request() + +# Authentication infrastructure + +class DigestAuthHandler: + """Handler for performing digest authentication.""" + + def __init__(self): + self._request_num = 0 + self._nonces = [] + self._users = {} + self._realm_name = "Test Realm" + self._qop = "auth" + + def set_qop(self, qop): + self._qop = qop + + def set_users(self, users): + assert isinstance(users, dict) + self._users = users + + def set_realm(self, realm): + self._realm_name = realm + + def _generate_nonce(self): + self._request_num += 1 + nonce = hashlib.md5(str(self._request_num).encode("ascii")).hexdigest() + self._nonces.append(nonce) + return nonce + + def _create_auth_dict(self, auth_str): + first_space_index = auth_str.find(" ") + auth_str = auth_str[first_space_index+1:] + + parts = auth_str.split(",") + + auth_dict = {} + for part in parts: + name, value = part.split("=") + name = name.strip() + if value[0] == '"' and value[-1] == '"': + value = value[1:-1] + else: + value = value.strip() + auth_dict[name] = value + return auth_dict + + def _validate_auth(self, auth_dict, password, method, uri): + final_dict = {} + final_dict.update(auth_dict) + final_dict["password"] = password + final_dict["method"] = method + final_dict["uri"] = uri + HA1_str = "%(username)s:%(realm)s:%(password)s" % final_dict + HA1 = hashlib.md5(HA1_str.encode("ascii")).hexdigest() + HA2_str = "%(method)s:%(uri)s" % final_dict + HA2 = hashlib.md5(HA2_str.encode("ascii")).hexdigest() + final_dict["HA1"] = HA1 + final_dict["HA2"] = HA2 + response_str = "%(HA1)s:%(nonce)s:%(nc)s:" \ + "%(cnonce)s:%(qop)s:%(HA2)s" % final_dict + response = hashlib.md5(response_str.encode("ascii")).hexdigest() + + return response == auth_dict["response"] + + def _return_auth_challenge(self, request_handler): + request_handler.send_response(407, "Proxy Authentication Required") + request_handler.send_header("Content-Type", "text/html") + request_handler.send_header( + 'Proxy-Authenticate', 'Digest realm="%s", ' + 'qop="%s",' + 'nonce="%s", ' % \ + (self._realm_name, self._qop, self._generate_nonce())) + # XXX: Not sure if we're supposed to add this next header or + # not. + #request_handler.send_header('Connection', 'close') + request_handler.end_headers() + request_handler.wfile.write(b"Proxy Authentication Required.") + return False + + def handle_request(self, request_handler): + """Performs digest authentication on the given HTTP request + handler. Returns True if authentication was successful, False + otherwise. + + If no users have been set, then digest auth is effectively + disabled and this method will always return True. + """ + + if len(self._users) == 0: + return True + + if "Proxy-Authorization" not in request_handler.headers: + return self._return_auth_challenge(request_handler) + else: + auth_dict = self._create_auth_dict( + request_handler.headers["Proxy-Authorization"] + ) + if auth_dict["username"] in self._users: + password = self._users[ auth_dict["username"] ] + else: + return self._return_auth_challenge(request_handler) + if not auth_dict.get("nonce") in self._nonces: + return self._return_auth_challenge(request_handler) + else: + self._nonces.remove(auth_dict["nonce"]) + + auth_validated = False + + # MSIE uses short_path in its validation, but Python's + # urllib.request uses the full path, so we're going to see if + # either of them works here. + + for path in [request_handler.path, request_handler.short_path]: + if self._validate_auth(auth_dict, + password, + request_handler.command, + path): + auth_validated = True + + if not auth_validated: + return self._return_auth_challenge(request_handler) + return True + + +class BasicAuthHandler(http.server.BaseHTTPRequestHandler): + """Handler for performing basic authentication.""" + # Server side values + USER = 'testUser' + PASSWD = 'testPass' + REALM = 'Test' + USER_PASSWD = "%s:%s" % (USER, PASSWD) + ENCODED_AUTH = base64.b64encode(USER_PASSWD.encode('ascii')).decode('ascii') + + def __init__(self, *args, **kwargs): + http.server.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def log_message(self, format, *args): + # Suppress console log message + pass + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_AUTHHEAD(self): + self.send_response(401) + self.send_header("WWW-Authenticate", "Basic realm=\"%s\"" % self.REALM) + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_GET(self): + if not self.headers.get("Authorization", ""): + self.do_AUTHHEAD() + self.wfile.write(b"No Auth header received") + elif self.headers.get( + "Authorization", "") == "Basic " + self.ENCODED_AUTH: + self.send_response(200) + self.end_headers() + self.wfile.write(b"It works") + else: + # Request Unauthorized + self.do_AUTHHEAD() + + + +# Proxy test infrastructure + +class FakeProxyHandler(http.server.BaseHTTPRequestHandler): + """This is a 'fake proxy' that makes it look like the entire + internet has gone down due to a sudden zombie invasion. It main + utility is in providing us with authentication support for + testing. + """ + + def __init__(self, digest_auth_handler, *args, **kwargs): + # This has to be set before calling our parent's __init__(), which will + # try to call do_GET(). + self.digest_auth_handler = digest_auth_handler + http.server.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def log_message(self, format, *args): + # Uncomment the next line for debugging. + # sys.stderr.write(format % args) + pass + + def do_GET(self): + (scm, netloc, path, params, query, fragment) = urllib.parse.urlparse( + self.path, "http") + self.short_path = path + if self.digest_auth_handler.handle_request(self): + self.send_response(200, "OK") + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(bytes("You've reached %s!
" % self.path, + "ascii")) + self.wfile.write(b"Our apologies, but our server is down due to " + b"a sudden zombie invasion.") + +# Test cases + +@unittest.skipUnless(threading, "Threading required for this test.") +class BasicAuthTests(unittest.TestCase): + USER = "testUser" + PASSWD = "testPass" + INCORRECT_PASSWD = "Incorrect" + REALM = "Test" + + def setUp(self): + super(BasicAuthTests, self).setUp() + # With Basic Authentication + def http_server_with_basic_auth_handler(*args, **kwargs): + return BasicAuthHandler(*args, **kwargs) + self.server = LoopbackHttpServerThread(http_server_with_basic_auth_handler) + self.addCleanup(self.server.stop) + self.server_url = 'http://127.0.0.1:%s' % self.server.port + self.server.start() + self.server.ready.wait() + + def tearDown(self): + super(BasicAuthTests, self).tearDown() + + def test_basic_auth_success(self): + ah = urllib.request.HTTPBasicAuthHandler() + ah.add_password(self.REALM, self.server_url, self.USER, self.PASSWD) + urllib.request.install_opener(urllib.request.build_opener(ah)) + try: + self.assertTrue(urllib.request.urlopen(self.server_url)) + except urllib.error.HTTPError: + self.fail("Basic auth failed for the url: %s", self.server_url) + + def test_basic_auth_httperror(self): + ah = urllib.request.HTTPBasicAuthHandler() + ah.add_password(self.REALM, self.server_url, self.USER, self.INCORRECT_PASSWD) + urllib.request.install_opener(urllib.request.build_opener(ah)) + self.assertRaises(urllib.error.HTTPError, urllib.request.urlopen, self.server_url) + + +@unittest.skipUnless(threading, "Threading required for this test.") +class ProxyAuthTests(unittest.TestCase): + URL = "http://localhost" + + USER = "tester" + PASSWD = "test123" + REALM = "TestRealm" + + def setUp(self): + super(ProxyAuthTests, self).setUp() + # Ignore proxy bypass settings in the environment. + def restore_environ(old_environ): + os.environ.clear() + os.environ.update(old_environ) + self.addCleanup(restore_environ, os.environ.copy()) + os.environ['NO_PROXY'] = '' + os.environ['no_proxy'] = '' + + self.digest_auth_handler = DigestAuthHandler() + self.digest_auth_handler.set_users({self.USER: self.PASSWD}) + self.digest_auth_handler.set_realm(self.REALM) + # With Digest Authentication. + def create_fake_proxy_handler(*args, **kwargs): + return FakeProxyHandler(self.digest_auth_handler, *args, **kwargs) + + self.server = LoopbackHttpServerThread(create_fake_proxy_handler) + self.server.start() + self.server.ready.wait() + proxy_url = "http://127.0.0.1:%d" % self.server.port + handler = urllib.request.ProxyHandler({"http" : proxy_url}) + self.proxy_digest_handler = urllib.request.ProxyDigestAuthHandler() + self.opener = urllib.request.build_opener( + handler, self.proxy_digest_handler) + + def tearDown(self): + self.server.stop() + super(ProxyAuthTests, self).tearDown() + + def test_proxy_with_bad_password_raises_httperror(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD+"bad") + self.digest_auth_handler.set_qop("auth") + self.assertRaises(urllib.error.HTTPError, + self.opener.open, + self.URL) + + def test_proxy_with_no_password_raises_httperror(self): + self.digest_auth_handler.set_qop("auth") + self.assertRaises(urllib.error.HTTPError, + self.opener.open, + self.URL) + + def test_proxy_qop_auth_works(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD) + self.digest_auth_handler.set_qop("auth") + result = self.opener.open(self.URL) + while result.read(): + pass + result.close() + + def test_proxy_qop_auth_int_works_or_throws_urlerror(self): + self.proxy_digest_handler.add_password(self.REALM, self.URL, + self.USER, self.PASSWD) + self.digest_auth_handler.set_qop("auth-int") + try: + result = self.opener.open(self.URL) + except urllib.error.URLError: + # It's okay if we don't support auth-int, but we certainly + # shouldn't receive any kind of exception here other than + # a URLError. + result = None + if result: + while result.read(): + pass + result.close() + + +def GetRequestHandler(responses): + + class FakeHTTPRequestHandler(http.server.BaseHTTPRequestHandler): + + server_version = "TestHTTP/" + requests = [] + headers_received = [] + port = 80 + + def do_GET(self): + body = self.send_head() + while body: + done = self.wfile.write(body) + body = body[done:] + + def do_POST(self): + content_length = self.headers["Content-Length"] + post_data = self.rfile.read(int(content_length)) + self.do_GET() + self.requests.append(post_data) + + def send_head(self): + FakeHTTPRequestHandler.headers_received = self.headers + self.requests.append(self.path) + response_code, headers, body = responses.pop(0) + + self.send_response(response_code) + + for (header, value) in headers: + self.send_header(header, value % {'port':self.port}) + if body: + self.send_header("Content-type", "text/plain") + self.end_headers() + return body + self.end_headers() + + def log_message(self, *args): + pass + + + return FakeHTTPRequestHandler + + +@unittest.skipUnless(threading, "Threading required for this test.") +class TestUrlopen(unittest.TestCase): + """Tests urllib.request.urlopen using the network. + + These tests are not exhaustive. Assuming that testing using files does a + good job overall of some of the basic interface features. There are no + tests exercising the optional 'data' and 'proxies' arguments. No tests + for transparent redirection have been written. + """ + + def setUp(self): + super(TestUrlopen, self).setUp() + + # Ignore proxies for localhost tests. + def restore_environ(old_environ): + os.environ.clear() + os.environ.update(old_environ) + self.addCleanup(restore_environ, os.environ.copy()) + os.environ['NO_PROXY'] = '*' + os.environ['no_proxy'] = '*' + + def urlopen(self, url, data=None, **kwargs): + l = [] + f = urllib.request.urlopen(url, data, **kwargs) + try: + # Exercise various methods + l.extend(f.readlines(200)) + l.append(f.readline()) + l.append(f.read(1024)) + l.append(f.read()) + finally: + f.close() + return b"".join(l) + + def start_server(self, responses=None): + if responses is None: + responses = [(200, [], b"we don't care")] + handler = GetRequestHandler(responses) + + self.server = LoopbackHttpServerThread(handler) + self.addCleanup(self.server.stop) + self.server.start() + self.server.ready.wait() + port = self.server.port + handler.port = port + return handler + + def start_https_server(self, responses=None, **kwargs): + if not hasattr(urllib.request, 'HTTPSHandler'): + self.skipTest('ssl support required') + from test.ssl_servers import make_https_server + if responses is None: + responses = [(200, [], b"we care a bit")] + handler = GetRequestHandler(responses) + server = make_https_server(self, handler_class=handler, **kwargs) + handler.port = server.port + return handler + + def test_redirection(self): + expected_response = b"We got here..." + responses = [ + (302, [("Location", "http://localhost:%(port)s/somewhere_else")], + ""), + (200, [], expected_response) + ] + + handler = self.start_server(responses) + data = self.urlopen("http://localhost:%s/" % handler.port) + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ["/", "/somewhere_else"]) + + def test_chunked(self): + expected_response = b"hello world" + chunked_start = ( + b'a\r\n' + b'hello worl\r\n' + b'1\r\n' + b'd\r\n' + b'0\r\n' + ) + response = [(200, [("Transfer-Encoding", "chunked")], chunked_start)] + handler = self.start_server(response) + data = self.urlopen("http://localhost:%s/" % handler.port) + self.assertEqual(data, expected_response) + + def test_404(self): + expected_response = b"Bad bad bad..." + handler = self.start_server([(404, [], expected_response)]) + + try: + self.urlopen("http://localhost:%s/weeble" % handler.port) + except urllib.error.URLError as f: + data = f.read() + f.close() + else: + self.fail("404 should raise URLError") + + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ["/weeble"]) + + def test_200(self): + expected_response = b"pycon 2008..." + handler = self.start_server([(200, [], expected_response)]) + data = self.urlopen("http://localhost:%s/bizarre" % handler.port) + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ["/bizarre"]) + + def test_200_with_parameters(self): + expected_response = b"pycon 2008..." + handler = self.start_server([(200, [], expected_response)]) + data = self.urlopen("http://localhost:%s/bizarre" % handler.port, + b"get=with_feeling") + self.assertEqual(data, expected_response) + self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"]) + + def test_https(self): + handler = self.start_https_server() + context = ssl.create_default_context(cafile=CERT_localhost) + data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context) + self.assertEqual(data, b"we care a bit") + + def test_https_with_cafile(self): + handler = self.start_https_server(certfile=CERT_localhost) + # Good cert + data = self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_localhost) + self.assertEqual(data, b"we care a bit") + # Bad cert + with self.assertRaises(urllib.error.URLError) as cm: + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_fakehostname) + # Good cert, but mismatching hostname + handler = self.start_https_server(certfile=CERT_fakehostname) + with self.assertRaises(ssl.CertificateError) as cm: + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cafile=CERT_fakehostname) + + def test_https_with_cadefault(self): + handler = self.start_https_server(certfile=CERT_localhost) + # Self-signed cert should fail verification with system certificate store + with self.assertRaises(urllib.error.URLError) as cm: + self.urlopen("https://localhost:%s/bizarre" % handler.port, + cadefault=True) + + def test_https_sni(self): + if ssl is None: + self.skipTest("ssl module required") + if not ssl.HAS_SNI: + self.skipTest("SNI support required in OpenSSL") + sni_name = None + def cb_sni(ssl_sock, server_name, initial_context): + nonlocal sni_name + sni_name = server_name + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.set_servername_callback(cb_sni) + handler = self.start_https_server(context=context, certfile=CERT_localhost) + context = ssl.create_default_context(cafile=CERT_localhost) + self.urlopen("https://localhost:%s" % handler.port, context=context) + self.assertEqual(sni_name, "localhost") + + def test_sending_headers(self): + handler = self.start_server() + req = urllib.request.Request("http://localhost:%s/" % handler.port, + headers={"Range": "bytes=20-39"}) + with urllib.request.urlopen(req): + pass + self.assertEqual(handler.headers_received["Range"], "bytes=20-39") + + def test_basic(self): + handler = self.start_server() + open_url = urllib.request.urlopen("http://localhost:%s" % handler.port) + for attr in ("read", "close", "info", "geturl"): + self.assertTrue(hasattr(open_url, attr), "object returned from " + "urlopen lacks the %s attribute" % attr) + try: + self.assertTrue(open_url.read(), "calling 'read' failed") + finally: + open_url.close() + + def test_info(self): + handler = self.start_server() + open_url = urllib.request.urlopen( + "http://localhost:%s" % handler.port) + with open_url: + info_obj = open_url.info() + self.assertIsInstance(info_obj, email.message.Message, + "object returned by 'info' is not an " + "instance of email.message.Message") + self.assertEqual(info_obj.get_content_subtype(), "plain") + + def test_geturl(self): + # Make sure same URL as opened is returned by geturl. + handler = self.start_server() + open_url = urllib.request.urlopen("http://localhost:%s" % handler.port) + with open_url: + url = open_url.geturl() + self.assertEqual(url, "http://localhost:%s" % handler.port) + + def test_iteration(self): + expected_response = b"pycon 2008..." + handler = self.start_server([(200, [], expected_response)]) + data = urllib.request.urlopen("http://localhost:%s" % handler.port) + for line in data: + self.assertEqual(line, expected_response) + + def test_line_iteration(self): + lines = [b"We\n", b"got\n", b"here\n", b"verylong " * 8192 + b"\n"] + expected_response = b"".join(lines) + handler = self.start_server([(200, [], expected_response)]) + data = urllib.request.urlopen("http://localhost:%s" % handler.port) + for index, line in enumerate(data): + self.assertEqual(line, lines[index], + "Fetched line number %s doesn't match expected:\n" + " Expected length was %s, got %s" % + (index, len(lines[index]), len(line))) + self.assertEqual(index + 1, len(lines)) + + +threads_key = None + +def setUpModule(): + # Store the threading_setup in a key and ensure that it is cleaned up + # in the tearDown + global threads_key + threads_key = support.threading_setup() + +def tearDownModule(): + if threads_key: + support.threading_cleanup(threads_key) + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_urllib2net.py b/src/greentest/3.5pypy/test_urllib2net.py new file mode 100644 index 0000000..cad83fd --- /dev/null +++ b/src/greentest/3.5pypy/test_urllib2net.py @@ -0,0 +1,329 @@ +import unittest +from test import support +from test.test_urllib2 import sanepathname2url + +import os +import socket +import urllib.error +import urllib.request +import sys + +support.requires("network") + +TIMEOUT = 60 # seconds + + +def _retry_thrice(func, exc, *args, **kwargs): + for i in range(3): + try: + return func(*args, **kwargs) + except exc as e: + last_exc = e + continue + raise last_exc + +def _wrap_with_retry_thrice(func, exc): + def wrapped(*args, **kwargs): + return _retry_thrice(func, exc, *args, **kwargs) + return wrapped + +# Connecting to remote hosts is flaky. Make it more robust by retrying +# the connection several times. +_urlopen_with_retry = _wrap_with_retry_thrice(urllib.request.urlopen, + urllib.error.URLError) + + +class AuthTests(unittest.TestCase): + """Tests urllib2 authentication features.""" + +## Disabled at the moment since there is no page under python.org which +## could be used to HTTP authentication. +# +# def test_basic_auth(self): +# import http.client +# +# test_url = "http://www.python.org/test/test_urllib2/basic_auth" +# test_hostport = "www.python.org" +# test_realm = 'Test Realm' +# test_user = 'test.test_urllib2net' +# test_password = 'blah' +# +# # failure +# try: +# _urlopen_with_retry(test_url) +# except urllib2.HTTPError, exc: +# self.assertEqual(exc.code, 401) +# else: +# self.fail("urlopen() should have failed with 401") +# +# # success +# auth_handler = urllib2.HTTPBasicAuthHandler() +# auth_handler.add_password(test_realm, test_hostport, +# test_user, test_password) +# opener = urllib2.build_opener(auth_handler) +# f = opener.open('http://localhost/') +# response = _urlopen_with_retry("http://www.python.org/") +# +# # The 'userinfo' URL component is deprecated by RFC 3986 for security +# # reasons, let's not implement it! (it's already implemented for proxy +# # specification strings (that is, URLs or authorities specifying a +# # proxy), so we must keep that) +# self.assertRaises(http.client.InvalidURL, +# urllib2.urlopen, "http://evil:thing@example.com") + + +class CloseSocketTest(unittest.TestCase): + + def test_close(self): + # calling .close() on urllib2's response objects should close the + # underlying socket + url = "http://www.example.com/" + with support.transient_internet(url): + response = _urlopen_with_retry(url) + sock = response.fp + self.assertFalse(sock.closed) + response.close() + self.assertTrue(sock.closed) + +class OtherNetworkTests(unittest.TestCase): + def setUp(self): + if 0: # for debugging + import logging + logger = logging.getLogger("test_urllib2net") + logger.addHandler(logging.StreamHandler()) + + # XXX The rest of these tests aren't very good -- they don't check much. + # They do sometimes catch some major disasters, though. + + def test_ftp(self): + urls = [ + 'ftp://ftp.debian.org/debian/README', + ('ftp://ftp.debian.org/debian/non-existent-file', + None, urllib.error.URLError), + ] + self._test_urls(urls, self._extra_handlers()) + + def test_file(self): + TESTFN = support.TESTFN + f = open(TESTFN, 'w') + try: + f.write('hi there\n') + f.close() + urls = [ + 'file:' + sanepathname2url(os.path.abspath(TESTFN)), + ('file:///nonsensename/etc/passwd', None, + urllib.error.URLError), + ] + self._test_urls(urls, self._extra_handlers(), retry=True) + finally: + os.remove(TESTFN) + + self.assertRaises(ValueError, urllib.request.urlopen,'./relative_path/to/file') + + # XXX Following test depends on machine configurations that are internal + # to CNRI. Need to set up a public server with the right authentication + # configuration for test purposes. + +## def test_cnri(self): +## if socket.gethostname() == 'bitdiddle': +## localhost = 'bitdiddle.cnri.reston.va.us' +## elif socket.gethostname() == 'bitdiddle.concentric.net': +## localhost = 'localhost' +## else: +## localhost = None +## if localhost is not None: +## urls = [ +## 'file://%s/etc/passwd' % localhost, +## 'http://%s/simple/' % localhost, +## 'http://%s/digest/' % localhost, +## 'http://%s/not/found.h' % localhost, +## ] + +## bauth = HTTPBasicAuthHandler() +## bauth.add_password('basic_test_realm', localhost, 'jhylton', +## 'password') +## dauth = HTTPDigestAuthHandler() +## dauth.add_password('digest_test_realm', localhost, 'jhylton', +## 'password') + +## self._test_urls(urls, self._extra_handlers()+[bauth, dauth]) + + def test_urlwithfrag(self): + urlwith_frag = "http://www.pythontest.net/index.html#frag" + with support.transient_internet(urlwith_frag): + req = urllib.request.Request(urlwith_frag) + res = urllib.request.urlopen(req) + self.assertEqual(res.geturl(), + "http://www.pythontest.net/index.html#frag") + + def test_redirect_url_withfrag(self): + redirect_url_with_frag = "http://www.pythontest.net/redir/with_frag/" + with support.transient_internet(redirect_url_with_frag): + req = urllib.request.Request(redirect_url_with_frag) + res = urllib.request.urlopen(req) + self.assertEqual(res.geturl(), + "http://www.pythontest.net/elsewhere/#frag") + + def test_custom_headers(self): + url = "http://www.example.com" + with support.transient_internet(url): + opener = urllib.request.build_opener() + request = urllib.request.Request(url) + self.assertFalse(request.header_items()) + opener.open(request) + self.assertTrue(request.header_items()) + self.assertTrue(request.has_header('User-agent')) + request.add_header('User-Agent','Test-Agent') + opener.open(request) + self.assertEqual(request.get_header('User-agent'),'Test-Agent') + + def test_sites_no_connection_close(self): + # Some sites do not send Connection: close header. + # Verify that those work properly. (#issue12576) + + URL = 'http://www.imdb.com' # mangles Connection:close + + with support.transient_internet(URL): + try: + with urllib.request.urlopen(URL) as res: + pass + except ValueError as e: + self.fail("urlopen failed for site not sending \ + Connection:close") + else: + self.assertTrue(res) + + req = urllib.request.urlopen(URL) + res = req.read() + self.assertTrue(res) + + def _test_urls(self, urls, handlers, retry=True): + import time + import logging + debug = logging.getLogger("test_urllib2").debug + + urlopen = urllib.request.build_opener(*handlers).open + if retry: + urlopen = _wrap_with_retry_thrice(urlopen, urllib.error.URLError) + + for url in urls: + with self.subTest(url=url): + if isinstance(url, tuple): + url, req, expected_err = url + else: + req = expected_err = None + + with support.transient_internet(url): + try: + f = urlopen(url, req, TIMEOUT) + # urllib.error.URLError is a subclass of OSError + except OSError as err: + if expected_err: + msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % + (expected_err, url, req, type(err), err)) + self.assertIsInstance(err, expected_err, msg) + else: + raise + else: + try: + with support.time_out, \ + support.socket_peer_reset, \ + support.ioerror_peer_reset: + buf = f.read() + debug("read %d bytes" % len(buf)) + except socket.timeout: + print("" % url, file=sys.stderr) + f.close() + time.sleep(0.1) + + def _extra_handlers(self): + handlers = [] + + cfh = urllib.request.CacheFTPHandler() + self.addCleanup(cfh.clear_cache) + cfh.setTimeout(1) + handlers.append(cfh) + + return handlers + + +class TimeoutTest(unittest.TestCase): + def test_http_basic(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with support.transient_internet(url, timeout=None): + u = _urlopen_with_retry(url) + self.addCleanup(u.close) + self.assertIsNone(u.fp.raw._sock.gettimeout()) + + def test_http_default_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with support.transient_internet(url): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(url) + self.addCleanup(u.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(u.fp.raw._sock.gettimeout(), 60) + + def test_http_no_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + url = "http://www.example.com" + with support.transient_internet(url): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(url, timeout=None) + self.addCleanup(u.close) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(u.fp.raw._sock.gettimeout()) + + def test_http_timeout(self): + url = "http://www.example.com" + with support.transient_internet(url): + u = _urlopen_with_retry(url, timeout=120) + self.addCleanup(u.close) + self.assertEqual(u.fp.raw._sock.gettimeout(), 120) + + FTP_HOST = 'ftp://ftp.debian.org/debian/' + + def test_ftp_basic(self): + self.assertIsNone(socket.getdefaulttimeout()) + with support.transient_internet(self.FTP_HOST, timeout=None): + u = _urlopen_with_retry(self.FTP_HOST) + self.addCleanup(u.close) + self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) + + def test_ftp_default_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + with support.transient_internet(self.FTP_HOST): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(self.FTP_HOST) + self.addCleanup(u.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) + + def test_ftp_no_timeout(self): + self.assertIsNone(socket.getdefaulttimeout()) + with support.transient_internet(self.FTP_HOST): + socket.setdefaulttimeout(60) + try: + u = _urlopen_with_retry(self.FTP_HOST, timeout=None) + self.addCleanup(u.close) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) + + def test_ftp_timeout(self): + with support.transient_internet(self.FTP_HOST): + u = _urlopen_with_retry(self.FTP_HOST, timeout=60) + self.addCleanup(u.close) + self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/test_wsgiref.py b/src/greentest/3.5pypy/test_wsgiref.py new file mode 100644 index 0000000..61a750c --- /dev/null +++ b/src/greentest/3.5pypy/test_wsgiref.py @@ -0,0 +1,785 @@ +from unittest import mock +from test import support +from test.test_httpservers import NoLogRequestHandler +from unittest import TestCase +from wsgiref.util import setup_testing_defaults +from wsgiref.headers import Headers +from wsgiref.handlers import BaseHandler, BaseCGIHandler, SimpleHandler +from wsgiref import util +from wsgiref.validate import validator +from wsgiref.simple_server import WSGIServer, WSGIRequestHandler +from wsgiref.simple_server import make_server +from http.client import HTTPConnection +from io import StringIO, BytesIO, BufferedReader +from socketserver import BaseServer +from platform import python_implementation + +import os +import re +import signal +import sys +import unittest + + +class MockServer(WSGIServer): + """Non-socket HTTP server""" + + def __init__(self, server_address, RequestHandlerClass): + BaseServer.__init__(self, server_address, RequestHandlerClass) + self.server_bind() + + def server_bind(self): + host, port = self.server_address + self.server_name = host + self.server_port = port + self.setup_environ() + + +class MockHandler(WSGIRequestHandler): + """Non-socket HTTP handler""" + def setup(self): + self.connection = self.request + self.rfile, self.wfile = self.connection + + def finish(self): + pass + + +def hello_app(environ,start_response): + start_response("200 OK", [ + ('Content-Type','text/plain'), + ('Date','Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return [b"Hello, world!"] + + +def header_app(environ, start_response): + start_response("200 OK", [ + ('Content-Type', 'text/plain'), + ('Date', 'Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return [';'.join([ + environ['HTTP_X_TEST_HEADER'], environ['QUERY_STRING'], + environ['PATH_INFO'] + ]).encode('iso-8859-1')] + + +def run_amock(app=hello_app, data=b"GET / HTTP/1.0\n\n"): + server = make_server("", 80, app, MockServer, MockHandler) + inp = BufferedReader(BytesIO(data)) + out = BytesIO() + olderr = sys.stderr + err = sys.stderr = StringIO() + + try: + server.finish_request((inp, out), ("127.0.0.1",8888)) + finally: + sys.stderr = olderr + + return out.getvalue(), err.getvalue() + +def compare_generic_iter(make_it,match): + """Utility to compare a generic 2.1/2.2+ iterator with an iterable + + If running under Python 2.2+, this tests the iterator using iter()/next(), + as well as __getitem__. 'make_it' must be a function returning a fresh + iterator to be tested (since this may test the iterator twice).""" + + it = make_it() + n = 0 + for item in match: + if not it[n]==item: raise AssertionError + n+=1 + try: + it[n] + except IndexError: + pass + else: + raise AssertionError("Too many items from __getitem__",it) + + try: + iter, StopIteration + except NameError: + pass + else: + # Only test iter mode under 2.2+ + it = make_it() + if not iter(it) is it: raise AssertionError + for item in match: + if not next(it) == item: raise AssertionError + try: + next(it) + except StopIteration: + pass + else: + raise AssertionError("Too many items from .__next__()", it) + + +class IntegrationTests(TestCase): + + def check_hello(self, out, has_length=True): + pyver = (python_implementation() + "/" + + sys.version.split()[0]) + self.assertEqual(out, + ("HTTP/1.0 200 OK\r\n" + "Server: WSGIServer/0.2 " + pyver +"\r\n" + "Content-Type: text/plain\r\n" + "Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + + (has_length and "Content-Length: 13\r\n" or "") + + "\r\n" + "Hello, world!").encode("iso-8859-1") + ) + + def test_plain_hello(self): + out, err = run_amock() + self.check_hello(out) + + def test_environ(self): + request = ( + b"GET /p%61th/?query=test HTTP/1.0\n" + b"X-Test-Header: Python test \n" + b"X-Test-Header: Python test 2\n" + b"Content-Length: 0\n\n" + ) + out, err = run_amock(header_app, request) + self.assertEqual( + out.splitlines()[-1], + b"Python test,Python test 2;query=test;/path/" + ) + + def test_request_length(self): + out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") + self.assertEqual(out.splitlines()[0], + b"HTTP/1.0 414 Request-URI Too Long") + + def test_validated_hello(self): + out, err = run_amock(validator(hello_app)) + # the middleware doesn't support len(), so content-length isn't there + self.check_hello(out, has_length=False) + + def test_simple_validation_error(self): + def bad_app(environ,start_response): + start_response("200 OK", ('Content-Type','text/plain')) + return ["Hello, world!"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], + "AssertionError: Headers (('Content-Type', 'text/plain')) must" + " be of type list: " + ) + + def test_status_validation_errors(self): + def create_bad_app(status): + def bad_app(environ, start_response): + start_response(status, [("Content-Type", "text/plain; charset=utf-8")]) + return [b"Hello, world!"] + return bad_app + + tests = [ + ('200', 'AssertionError: Status must be at least 4 characters'), + ('20X OK', 'AssertionError: Status message must begin w/3-digit code'), + ('200OK', 'AssertionError: Status message must have a space after code'), + ] + + for status, exc_message in tests: + with self.subTest(status=status): + out, err = run_amock(create_bad_app(status)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual(err.splitlines()[-2], exc_message) + + def test_wsgi_input(self): + def bad_app(e,s): + e["wsgi.input"].read() + s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) + return [b"data"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], "AssertionError" + ) + + def test_bytes_validation(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain; charset=utf-8"), + ("Date", "Wed, 24 Dec 2008 13:29:32 GMT"), + ]) + return [b"data"] + out, err = run_amock(validator(app)) + self.assertTrue(err.endswith('"GET / HTTP/1.0" 200 4\n')) + ver = sys.version.split()[0].encode('ascii') + py = python_implementation().encode('ascii') + pyver = py + b"/" + ver + self.assertEqual( + b"HTTP/1.0 200 OK\r\n" + b"Server: WSGIServer/0.2 "+ pyver + b"\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"Date: Wed, 24 Dec 2008 13:29:32 GMT\r\n" + b"\r\n" + b"data", + out) + + def test_cp1252_url(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain"), + ("Date", "Wed, 24 Dec 2008 13:29:32 GMT"), + ]) + # PEP3333 says environ variables are decoded as latin1. + # Encode as latin1 to get original bytes + return [e["PATH_INFO"].encode("latin1")] + + out, err = run_amock( + validator(app), data=b"GET /\x80%80 HTTP/1.0") + self.assertEqual( + [ + b"HTTP/1.0 200 OK", + mock.ANY, + b"Content-Type: text/plain", + b"Date: Wed, 24 Dec 2008 13:29:32 GMT", + b"", + b"/\x80\x80", + ], + out.splitlines()) + + def test_interrupted_write(self): + # BaseHandler._write() and _flush() have to write all data, even if + # it takes multiple send() calls. Test this by interrupting a send() + # call with a Unix signal. + threading = support.import_module("threading") + pthread_kill = support.get_attribute(signal, "pthread_kill") + + def app(environ, start_response): + start_response("200 OK", []) + return [bytes(support.SOCK_MAX_SIZE)] + + class WsgiHandler(NoLogRequestHandler, WSGIRequestHandler): + pass + + server = make_server(support.HOST, 0, app, handler_class=WsgiHandler) + self.addCleanup(server.server_close) + interrupted = threading.Event() + + def signal_handler(signum, frame): + interrupted.set() + + original = signal.signal(signal.SIGUSR1, signal_handler) + self.addCleanup(signal.signal, signal.SIGUSR1, original) + received = None + main_thread = threading.get_ident() + + def run_client(): + http = HTTPConnection(*server.server_address) + http.request("GET", "/") + with http.getresponse() as response: + response.read(100) + # The main thread should now be blocking in a send() system + # call. But in theory, it could get interrupted by other + # signals, and then retried. So keep sending the signal in a + # loop, in case an earlier signal happens to be delivered at + # an inconvenient moment. + while True: + pthread_kill(main_thread, signal.SIGUSR1) + if interrupted.wait(timeout=float(1)): + break + nonlocal received + received = len(response.read()) + http.close() + + background = threading.Thread(target=run_client) + background.start() + server.handle_request() + background.join() + self.assertEqual(received, support.SOCK_MAX_SIZE - 100) + + +class UtilityTests(TestCase): + + def checkShift(self,sn_in,pi_in,part,sn_out,pi_out): + env = {'SCRIPT_NAME':sn_in,'PATH_INFO':pi_in} + util.setup_testing_defaults(env) + self.assertEqual(util.shift_path_info(env),part) + self.assertEqual(env['PATH_INFO'],pi_out) + self.assertEqual(env['SCRIPT_NAME'],sn_out) + return env + + def checkDefault(self, key, value, alt=None): + # Check defaulting when empty + env = {} + util.setup_testing_defaults(env) + if isinstance(value, StringIO): + self.assertIsInstance(env[key], StringIO) + elif isinstance(value,BytesIO): + self.assertIsInstance(env[key],BytesIO) + else: + self.assertEqual(env[key], value) + + # Check existing value + env = {key:alt} + util.setup_testing_defaults(env) + self.assertIs(env[key], alt) + + def checkCrossDefault(self,key,value,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(kw[key],value) + + def checkAppURI(self,uri,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.application_uri(kw),uri) + + def checkReqURI(self,uri,query=1,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.request_uri(kw,query),uri) + + def checkFW(self,text,size,match): + + def make_it(text=text,size=size): + return util.FileWrapper(StringIO(text),size) + + compare_generic_iter(make_it,match) + + it = make_it() + self.assertFalse(it.filelike.closed) + + for item in it: + pass + + self.assertFalse(it.filelike.closed) + + it.close() + self.assertTrue(it.filelike.closed) + + def testSimpleShifts(self): + self.checkShift('','/', '', '/', '') + self.checkShift('','/x', 'x', '/x', '') + self.checkShift('/','', None, '/', '') + self.checkShift('/a','/x/y', 'x', '/a/x', '/y') + self.checkShift('/a','/x/', 'x', '/a/x', '/') + + def testNormalizedShifts(self): + self.checkShift('/a/b', '/../y', '..', '/a', '/y') + self.checkShift('', '/../y', '..', '', '/y') + self.checkShift('/a/b', '//y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '//y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '/./y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '/./y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '///./..//y/.//', '..', '/a', '/y/') + self.checkShift('/a/b', '///', '', '/a/b/', '') + self.checkShift('/a/b', '/.//', '', '/a/b/', '') + self.checkShift('/a/b', '/x//', 'x', '/a/b/x', '/') + self.checkShift('/a/b', '/.', None, '/a/b', '') + + def testDefaults(self): + for key, value in [ + ('SERVER_NAME','127.0.0.1'), + ('SERVER_PORT', '80'), + ('SERVER_PROTOCOL','HTTP/1.0'), + ('HTTP_HOST','127.0.0.1'), + ('REQUEST_METHOD','GET'), + ('SCRIPT_NAME',''), + ('PATH_INFO','/'), + ('wsgi.version', (1,0)), + ('wsgi.run_once', 0), + ('wsgi.multithread', 0), + ('wsgi.multiprocess', 0), + ('wsgi.input', BytesIO()), + ('wsgi.errors', StringIO()), + ('wsgi.url_scheme','http'), + ]: + self.checkDefault(key,value) + + def testCrossDefaults(self): + self.checkCrossDefault('HTTP_HOST',"foo.bar",SERVER_NAME="foo.bar") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="on") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="1") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="yes") + self.checkCrossDefault('wsgi.url_scheme',"http",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"80",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"443",HTTPS="on") + + def testGuessScheme(self): + self.assertEqual(util.guess_scheme({}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"foo"}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"on"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"yes"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"1"}), "https") + + def testAppURIs(self): + self.checkAppURI("http://127.0.0.1/") + self.checkAppURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkAppURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkAppURI("http://spam.example.com:2071/", + HTTP_HOST="spam.example.com:2071", SERVER_PORT="2071") + self.checkAppURI("http://spam.example.com/", + SERVER_NAME="spam.example.com") + self.checkAppURI("http://127.0.0.1/", + HTTP_HOST="127.0.0.1", SERVER_NAME="spam.example.com") + self.checkAppURI("https://127.0.0.1/", HTTPS="on") + self.checkAppURI("http://127.0.0.1:8000/", SERVER_PORT="8000", + HTTP_HOST=None) + + def testReqURIs(self): + self.checkReqURI("http://127.0.0.1/") + self.checkReqURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkReqURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam", + SCRIPT_NAME="/spammity", PATH_INFO="/spam") + self.checkReqURI("http://127.0.0.1/spammity/sp%E4m", + SCRIPT_NAME="/spammity", PATH_INFO="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam;ham", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;ham") + self.checkReqURI("http://127.0.0.1/spammity/spam;cookie=1234,5678", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;cookie=1234,5678") + self.checkReqURI("http://127.0.0.1/spammity/spam?say=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam?s%E4y=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="s%E4y=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam", 0, + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + + def testFileWrapper(self): + self.checkFW("xyz"*50, 120, ["xyz"*40,"xyz"*10]) + + def testHopByHop(self): + for hop in ( + "Connection Keep-Alive Proxy-Authenticate Proxy-Authorization " + "TE Trailers Transfer-Encoding Upgrade" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertTrue(util.is_hop_by_hop(alt)) + + # Not comprehensive, just a few random header names + for hop in ( + "Accept Cache-Control Date Pragma Trailer Via Warning" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertFalse(util.is_hop_by_hop(alt)) + +class HeaderTests(TestCase): + + def testMappingInterface(self): + test = [('x','y')] + self.assertEqual(len(Headers()), 0) + self.assertEqual(len(Headers([])),0) + self.assertEqual(len(Headers(test[:])),1) + self.assertEqual(Headers(test[:]).keys(), ['x']) + self.assertEqual(Headers(test[:]).values(), ['y']) + self.assertEqual(Headers(test[:]).items(), test) + self.assertIsNot(Headers(test).items(), test) # must be copy! + + h = Headers() + del h['foo'] # should not raise an error + + h['Foo'] = 'bar' + for m in h.__contains__, h.get, h.get_all, h.__getitem__: + self.assertTrue(m('foo')) + self.assertTrue(m('Foo')) + self.assertTrue(m('FOO')) + self.assertFalse(m('bar')) + + self.assertEqual(h['foo'],'bar') + h['foo'] = 'baz' + self.assertEqual(h['FOO'],'baz') + self.assertEqual(h.get_all('foo'),['baz']) + + self.assertEqual(h.get("foo","whee"), "baz") + self.assertEqual(h.get("zoo","whee"), "whee") + self.assertEqual(h.setdefault("foo","whee"), "baz") + self.assertEqual(h.setdefault("zoo","whee"), "whee") + self.assertEqual(h["foo"],"baz") + self.assertEqual(h["zoo"],"whee") + + def testRequireList(self): + self.assertRaises(TypeError, Headers, "foo") + + def testExtras(self): + h = Headers() + self.assertEqual(str(h),'\r\n') + + h.add_header('foo','bar',baz="spam") + self.assertEqual(h['foo'], 'bar; baz="spam"') + self.assertEqual(str(h),'foo: bar; baz="spam"\r\n\r\n') + + h.add_header('Foo','bar',cheese=None) + self.assertEqual(h.get_all('foo'), + ['bar; baz="spam"', 'bar; cheese']) + + self.assertEqual(str(h), + 'foo: bar; baz="spam"\r\n' + 'Foo: bar; cheese\r\n' + '\r\n' + ) + +class ErrorHandler(BaseCGIHandler): + """Simple handler subclass for testing BaseHandler""" + + # BaseHandler records the OS environment at import time, but envvars + # might have been changed later by other tests, which trips up + # HandlerTests.testEnviron(). + os_environ = dict(os.environ.items()) + + def __init__(self,**kw): + setup_testing_defaults(kw) + BaseCGIHandler.__init__( + self, BytesIO(), BytesIO(), StringIO(), kw, + multithread=True, multiprocess=True + ) + +class TestHandler(ErrorHandler): + """Simple handler subclass for testing BaseHandler, w/error passthru""" + + def handle_error(self): + raise # for testing, we want to see what's happening + + +class HandlerTests(TestCase): + + def checkEnvironAttrs(self, handler): + env = handler.environ + for attr in [ + 'version','multithread','multiprocess','run_once','file_wrapper' + ]: + if attr=='file_wrapper' and handler.wsgi_file_wrapper is None: + continue + self.assertEqual(getattr(handler,'wsgi_'+attr),env['wsgi.'+attr]) + + def checkOSEnviron(self,handler): + empty = {}; setup_testing_defaults(empty) + env = handler.environ + from os import environ + for k,v in environ.items(): + if k not in empty: + self.assertEqual(env[k],v) + for k,v in empty.items(): + self.assertIn(k, env) + + def testEnviron(self): + h = TestHandler(X="Y") + h.setup_environ() + self.checkEnvironAttrs(h) + self.checkOSEnviron(h) + self.assertEqual(h.environ["X"],"Y") + + def testCGIEnviron(self): + h = BaseCGIHandler(None,None,None,{}) + h.setup_environ() + for key in 'wsgi.url_scheme', 'wsgi.input', 'wsgi.errors': + self.assertIn(key, h.environ) + + def testScheme(self): + h=TestHandler(HTTPS="on"); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'https') + h=TestHandler(); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'http') + + def testAbstractMethods(self): + h = BaseHandler() + for name in [ + '_flush','get_stdin','get_stderr','add_cgi_vars' + ]: + self.assertRaises(NotImplementedError, getattr(h,name)) + self.assertRaises(NotImplementedError, h._write, "test") + + def testContentLength(self): + # Demo one reason iteration is better than write()... ;) + + def trivial_app1(e,s): + s('200 OK',[]) + return [e['wsgi.url_scheme'].encode('iso-8859-1')] + + def trivial_app2(e,s): + s('200 OK',[])(e['wsgi.url_scheme'].encode('iso-8859-1')) + return [] + + def trivial_app3(e,s): + s('200 OK',[]) + return ['\u0442\u0435\u0441\u0442'.encode("utf-8")] + + def trivial_app4(e,s): + # Simulate a response to a HEAD request + s('200 OK',[('Content-Length', '12345')]) + return [] + + h = TestHandler() + h.run(trivial_app1) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "Content-Length: 4\r\n" + "\r\n" + "http").encode("iso-8859-1")) + + h = TestHandler() + h.run(trivial_app2) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "\r\n" + "http").encode("iso-8859-1")) + + h = TestHandler() + h.run(trivial_app3) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 8\r\n' + b'\r\n' + b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82') + + h = TestHandler() + h.run(trivial_app4) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 12345\r\n' + b'\r\n') + + def testBasicErrorOutput(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + def error_app(e,s): + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(non_error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n").encode("iso-8859-1")) + self.assertEqual(h.stderr.getvalue(),"") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: %s\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: %d\r\n" + "\r\n" % (h.error_status,len(h.error_body))).encode('iso-8859-1') + + h.error_body) + + self.assertIn("AssertionError", h.stderr.getvalue()) + + def testErrorAfterOutput(self): + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "\r\n".encode("iso-8859-1")+MSG)) + self.assertIn("AssertionError", h.stderr.getvalue()) + + def testHeaderFormats(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + stdpat = ( + r"HTTP/%s 200 OK\r\n" + r"Date: \w{3}, [ 0123]\d \w{3} \d{4} \d\d:\d\d:\d\d GMT\r\n" + r"%s" r"Content-Length: 0\r\n" r"\r\n" + ) + shortpat = ( + "Status: 200 OK\r\n" "Content-Length: 0\r\n" "\r\n" + ).encode("iso-8859-1") + + for ssw in "FooBar/1.0", None: + sw = ssw and "Server: %s\r\n" % ssw or "" + + for version in "1.0", "1.1": + for proto in "HTTP/0.9", "HTTP/1.0", "HTTP/1.1": + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = False + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + self.assertEqual(shortpat,h.stdout.getvalue()) + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = True + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + if proto=="HTTP/0.9": + self.assertEqual(h.stdout.getvalue(),b"") + else: + self.assertTrue( + re.match((stdpat%(version,sw)).encode("iso-8859-1"), + h.stdout.getvalue()), + ((stdpat%(version,sw)).encode("iso-8859-1"), + h.stdout.getvalue()) + ) + + def testBytesData(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain; charset=utf-8"), + ]) + return [b"data"] + + h = TestHandler() + h.run(app) + self.assertEqual(b"Status: 200 OK\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"Content-Length: 4\r\n" + b"\r\n" + b"data", + h.stdout.getvalue()) + + def testCloseOnError(self): + side_effects = {'close_called': False} + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + class CrashyIterable(object): + def __iter__(self): + while True: + yield b'blah' + raise AssertionError("This should be caught by handler") + def close(self): + side_effects['close_called'] = True + return CrashyIterable() + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(side_effects['close_called'], True) + + def testPartialWrite(self): + written = bytearray() + + class PartialWriter: + def write(self, b): + partial = b[:7] + written.extend(partial) + return len(partial) + + def flush(self): + pass + + environ = {"SERVER_PROTOCOL": "HTTP/1.0"} + h = SimpleHandler(BytesIO(), PartialWriter(), sys.stderr, environ) + msg = "should not do partial writes" + with self.assertWarnsRegex(DeprecationWarning, msg): + h.run(hello_app) + self.assertEqual(b"HTTP/1.0 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + b"Content-Length: 13\r\n" + b"\r\n" + b"Hello, world!", + written) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.5pypy/version b/src/greentest/3.5pypy/version new file mode 100644 index 0000000..444877d --- /dev/null +++ b/src/greentest/3.5pypy/version @@ -0,0 +1 @@ +3.5.3 diff --git a/src/greentest/3.5pypy/wrongcert.pem b/src/greentest/3.5pypy/wrongcert.pem new file mode 100644 index 0000000..5f92f9b --- /dev/null +++ b/src/greentest/3.5pypy/wrongcert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH +FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T +f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB +AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq +1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW +7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg +SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe +19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg +ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666 +lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs +ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv +frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk +2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6 ++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK +24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G +A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9 +OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ +fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E +usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+ +43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw +eSHj5jpC8iZKjCHBn+mAi4cQ514= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/allsans.pem b/src/greentest/3.6/allsans.pem new file mode 100644 index 0000000..3ee4f59 --- /dev/null +++ b/src/greentest/3.6/allsans.pem @@ -0,0 +1,37 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOoy7/QOtTjQ0niE +6uDcTwtkC0R2Tvy1AjVnXohCntZfdzbTGDoYTgXSOLsP8A697jUiJ8VCePGH50xG +Z4DKnAF3a9O3a9nr2pLXb0iY3XOMv+YEBii7CfI+3oxFYgCl0sMgHzDD2ZTVYAsm +DWgLUVsE2gHEccRwrM2tPf2EgR+FAgMBAAECgYEA3qyfyYVSeTrTYxO93x6ZaVMu +A2IZp9zSxMQL9bKiI2GRj+cV2ebSCGbg2btFnD6qBor7FWsmYz+8g6FNN/9sY4az +61rMqMtQvLBe+7L8w70FeTze4qQ4Y1oQri0qD6tBWhDVlpnbI5Py9bkZKD67yVUk +elcEA/5x4PrYXkuqsAECQQD80NjT0mDvaY0JOOaQFSEpMv6QiUA8GGX8Xli7IoKb +tAolPG8rQBa+qSpcWfDMTrWw/aWHuMEEQoP/bVDH9W4FAkEA7SYQbBAKnojZ5A3G +kOHdV7aeivRQxQk/JN8Fb8oKB9Csvpv/BsuGxPKXHdhFa6CBTTsNRtHQw/szPo4l +xMIjgQJAPoMxqibR+0EBM6+TKzteSL6oPXsCnBl4Vk/J5vPgkbmR7KUl4+7j8N8J +b2554TrxKEN/w7CGYZRE6UrRd7ATNQJAWD7Yz41sli+wfPdPU2xo1BHljyl4wMk/ +EPZYbI/PCbdyAH/F935WyQTIjNeEhZc1Zkq6FwdOWw8ns3hrv3rKgQJAHXv1BqUa +czGPIFxX2TNoqtcl6/En4vrxVB1wzsfzkkDAg98kBl7qsF+S3qujSzKikjeaVbI2 +/CyWR2P3yLtOmA== +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDcjCCAtugAwIBAgIJAN5dc9TOWjB7MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTYwODA1 +MTAyMTExWhcNMjYwODAzMTAyMTExWjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO +Q2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0 +aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQDqMu/0DrU40NJ4hOrg3E8LZAtEdk78tQI1Z16IQp7WX3c20xg6GE4F0ji7D/AO +ve41IifFQnjxh+dMRmeAypwBd2vTt2vZ69qS129ImN1zjL/mBAYouwnyPt6MRWIA +pdLDIB8ww9mU1WALJg1oC1FbBNoBxHHEcKzNrT39hIEfhQIDAQABo4IBODCCATQw +ggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBp +ZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMC +AQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUu +b3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMw +IQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGly +bmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAA +AAAAAAAAAAAAAAAAAYgEKgMEBTANBgkqhkiG9w0BAQsFAAOBgQAy16h+F+nOmeiT +VWR0fc8F/j6FcadbLseAUaogcC15OGxCl4UYpLV88HBkABOoGCpP155qwWTwOrdG +iYPGJSusf1OnJEbvzFejZf6u078bPd9/ZL4VWLjv+FPGkjd+N+/OaqMvgj8Lu99f +3Y/C4S7YbHxxwff6C6l2Xli+q6gnuQ== +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/badcert.pem b/src/greentest/3.6/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/greentest/3.6/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/badkey.pem b/src/greentest/3.6/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/greentest/3.6/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/capath/0e4015b9.0 b/src/greentest/3.6/capath/0e4015b9.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.6/capath/0e4015b9.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/capath/4e1295a3.0 b/src/greentest/3.6/capath/4e1295a3.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.6/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/capath/5ed36f99.0 b/src/greentest/3.6/capath/5ed36f99.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.6/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/capath/6e88d7b8.0 b/src/greentest/3.6/capath/6e88d7b8.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.6/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/capath/99d0fa06.0 b/src/greentest/3.6/capath/99d0fa06.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.6/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/capath/b1930218.0 b/src/greentest/3.6/capath/b1930218.0 new file mode 100644 index 0000000..373349c --- /dev/null +++ b/src/greentest/3.6/capath/b1930218.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/capath/ce7b8643.0 b/src/greentest/3.6/capath/ce7b8643.0 new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.6/capath/ce7b8643.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/capath/ceff1710.0 b/src/greentest/3.6/capath/ceff1710.0 new file mode 100644 index 0000000..373349c --- /dev/null +++ b/src/greentest/3.6/capath/ceff1710.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/dh1024.pem b/src/greentest/3.6/dh1024.pem new file mode 100644 index 0000000..a391176 --- /dev/null +++ b/src/greentest/3.6/dh1024.pem @@ -0,0 +1,7 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAIbzw1s9CT8SV5yv6L7esdAdZYZjPi3qWFs61CYTFFQnf2s/d09NYaJt +rrvJhIzWavqnue71qXCf83/J3nz3FEwUU/L0mGyheVbsSHiI64wUo3u50wK5Igo0 +RNs/LD0irs7m0icZ//hijafTU+JOBiuA8zMI+oZfU7BGuc9XrUprAgEC +-----END DH PARAMETERS----- + +Generated with: openssl dhparam -out dh1024.pem 1024 diff --git a/src/greentest/3.6/keycert.passwd.pem b/src/greentest/3.6/keycert.passwd.pem new file mode 100644 index 0000000..e905748 --- /dev/null +++ b/src/greentest/3.6/keycert.passwd.pem @@ -0,0 +1,33 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/keycert.pem b/src/greentest/3.6/keycert.pem new file mode 100644 index 0000000..64318aa --- /dev/null +++ b/src/greentest/3.6/keycert.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/keycert2.pem b/src/greentest/3.6/keycert2.pem new file mode 100644 index 0000000..e8a9e08 --- /dev/null +++ b/src/greentest/3.6/keycert2.pem @@ -0,0 +1,31 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAJnsJZVrppL+W5I9 +zGQrrawWwE5QJpBK9nWw17mXrZ03R1cD9BamLGivVISbPlRlAVnZBEyh1ATpsB7d +CUQ+WHEvALquvx4+Yw5l+fXeiYRjrLRBYZuVy8yNtXzU3iWcGObcYRkUdiXdOyP7 +sLF2YZHRvQZpzgDBKkrraeQ81w21AgMBAAECgYBEm7n07FMHWlE+0kT0sXNsLYfy +YE+QKZnJw9WkaDN+zFEEPELkhZVt5BjsMraJr6v2fIEqF0gGGJPkbenffVq2B5dC +lWUOxvJHufMK4sM3Cp6s/gOp3LP+QkzVnvJSfAyZU6l+4PGX5pLdUsXYjPxgzjzL +S36tF7/2Uv1WePyLUQJBAMsPhYzUXOPRgmbhcJiqi9A9c3GO8kvSDYTCKt3VMnqz +HBn6MQ4VQasCD1F+7jWTI0FU/3vdw8non/Fj8hhYqZcCQQDCDRdvmZqDiZnpMqDq +L6ZSrLTVtMvZXZbgwForaAD9uHj51TME7+eYT7EG2YCgJTXJ4YvRJEnPNyskwdKt +vTSTAkEAtaaN/vyemEJ82BIGStwONNw0ILsSr5cZ9tBHzqiA/tipY+e36HRFiXhP +QcU9zXlxyWkDH8iz9DSAmE2jbfoqwwJANlMJ65E543cjIlitGcKLMnvtCCLcKpb7 +xSG0XJB6Lo11OKPJ66jp0gcFTSCY1Lx2CXVd+gfJrfwI1Pp562+bhwJBAJ9IfDPU +R8OpO9v1SGd8x33Owm7uXOpB9d63/T70AD1QOXjKUC4eXYbt0WWfWuny/RNPRuyh +w7DXSfUF+kPKolU= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICXTCCAcagAwIBAgIJAIO3upAG445fMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTAeFw0x +MDEwMDkxNTAxMDBaFw0yMDEwMDYxNTAxMDBaMGIxCzAJBgNVBAYTAlhZMRcwFQYD +VQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZv +dW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEAmewllWumkv5bkj3MZCutrBbATlAmkEr2dbDXuZetnTdHVwP0 +FqYsaK9UhJs+VGUBWdkETKHUBOmwHt0JRD5YcS8Auq6/Hj5jDmX59d6JhGOstEFh +m5XLzI21fNTeJZwY5txhGRR2Jd07I/uwsXZhkdG9BmnOAMEqSutp5DzXDbUCAwEA +AaMbMBkwFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBBQUAA4GB +AH+iMClLLGSaKWgwXsmdVo4FhTZZHo8Uprrtg3N9FxEeE50btpDVQysgRt5ias3K +m+bME9zbKwvbVWD5zZdjus4pDgzwF/iHyccL8JyYhxOvS/9zmvAtFXj/APIIbZFp +IT75d9f88ScIGEtknZQejnrdhB64tYki/EqluiuKBqKD +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/keycert3.pem b/src/greentest/3.6/keycert3.pem new file mode 100644 index 0000000..5bfa62c --- /dev/null +++ b/src/greentest/3.6/keycert3.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP +jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM +9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ +aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe +yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j +y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ +AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW +5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL +9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 +1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT +DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh +1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m +JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 +RnJdHOMXWem7/w== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: + 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: + c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: + 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: + f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: + 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: + f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: + af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: + 21:82:a5:3c:88:e5:be:1b:b1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: + e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: + f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: + e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: + d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: + 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: + ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: + 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: + 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: + 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: + 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: + f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: + 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: + a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: + fc:a9:94:71 +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C +tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola +N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 +TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR +iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG +xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo +5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv +mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF +YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh +2EJ36/yplHE= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/keycert4.pem b/src/greentest/3.6/keycert4.pem new file mode 100644 index 0000000..53355c8 --- /dev/null +++ b/src/greentest/3.6/keycert4.pem @@ -0,0 +1,73 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv +L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2 +NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1 +L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L +pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de +R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9 +myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT +drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS +Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx +i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK +Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu +JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN ++/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/ +e83Gq6ffLVfKNQ== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: 12723342612721443282 (0xb09264b1f2da21d2) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Nov 13 19:47:07 2022 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14: + 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9: + cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a: + b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76: + 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7: + 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a: + d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79: + 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd: + 81:7e:bd:1b:ae:0b:5d:c6:39 + Exponent: 65537 (0x10001) + Signature Algorithm: sha1WithRSAEncryption + ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0: + 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f: + 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56: + 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15: + 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1: + 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb: + 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e: + e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18: + d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1: + af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53: + 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68: + 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da: + 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92: + 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa: + 49:12:1e:ce +-----BEGIN CERTIFICATE----- +MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 +WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z +dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU +aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0 +ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ +hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v +xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338 +Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP +XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0 +UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz +aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb +oF+6ufu6+kkSHs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/nokia.pem b/src/greentest/3.6/nokia.pem new file mode 100644 index 0000000..0d044df --- /dev/null +++ b/src/greentest/3.6/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/nullbytecert.pem b/src/greentest/3.6/nullbytecert.pem new file mode 100644 index 0000000..447186c --- /dev/null +++ b/src/greentest/3.6/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/nullcert.pem b/src/greentest/3.6/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/greentest/3.6/pycacert.pem b/src/greentest/3.6/pycacert.pem new file mode 100644 index 0000000..09b1f3e --- /dev/null +++ b/src/greentest/3.6/pycacert.pem @@ -0,0 +1,78 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 4 19:47:07 2013 GMT + Not After : Jan 2 19:47:07 2023 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: + 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: + e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: + e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: + 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: + 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: + a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: + e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: + 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: + 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: + e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: + c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: + cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: + 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: + 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: + 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: + e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: + c5:4d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + X509v3 Authority Key Identifier: + keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: + 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: + a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: + 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: + 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: + 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: + fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: + 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: + 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: + 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: + ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: + 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: + b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: + 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: + 5e:58:c8:9e +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/pycakey.pem b/src/greentest/3.6/pycakey.pem new file mode 100644 index 0000000..fc6effe --- /dev/null +++ b/src/greentest/3.6/pycakey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDn3unjDJ8AtqH9 +K1uW0m/M4L6GuSBe7AN6VavqpOn5SYXSZtXtx3rqVo4tj+dC4mIoqZ/WG47rtbSc +nxSr3+aUi3YdPm0kYe0MvwCKYQzfXMg2cxYAzUe6baSkdIiDIwoZ/AmnPEpL0+cd +LeTqTFQh8ybbiTcY1AK7QDJfpP8tHPfUu+yOz1yCrOZ8CGxIhWEHfyXgXOC8NF/g +uQRHdchHC4281shoXzODYtIgRDWxrYEais28NbBci0fWGOmcGJfMATwpzOge5OTB +uN7nwhEYh1qTNNimJfcUcevkIaLSDy4u1GIANdPW71xgS0ypFOLdFVhGNzMmt+cu +Xe1C5MVNAgMBAAECggEBAJPM7QuUrPn4cLN/Ysd15lwTWn9oHDFFgkYFvCs66gXE +ju/6Kx2BjWE4wTJby09AHM/MqB0DvguT7Mf1Q2j3tPQ1HZowg8OwRDleuwp6KIls +jBbhL0Jdl/5HC67ktWvZ9wNvO/wFG1rQfT6FVajf9LUbWEaSZbOG2SLhHfsHorzu +xjTJaI3bQ/0+79B1exwk5ruwhzFRd/XpY8hls7D/RfPIuHDlBghkW3N59KFWrf5h +6bNEh2THm0+IyGcGqs0FD+QCOXyvsjwSUswqrr2ctLREOeDcd5ReUjSxYgjcJRrm +J7ceIY/+uwDJxw/OlnmBvF6pQMkKwYW2gFztu+g2t4UCgYEA/9yo01Exz4crxXsy +tAlnDJM++nZcm07rtFjTKHUfKY/cCgNTa8udM0svnfwlid/dpgLsI38gx04HHC1i +EZ4acz+ToIWedLxM0nq73//xeRWEazOvCz1mMTZaMldahTWAyzN8qVK2B/625Yy4 +wNYWyweBBwEB8MzaCs73spksXOsCgYEA5/7wvhiofYGFAfMuANeJIwDL2OtBnoOv +mVNfCmi3GC38fzwyi5ZpskWDiS2woJ+LQfs9Qu4EcZbUFLd7gbeOvb5gmFUtYope +LitUUKunIR18MkQ+mQDBpQPQPhk4QJP5reCbWkrfTu7b5o/iS41s6fBTFmuzhLcT +C71vFdCyeKcCgYAiCCqYeOtELDmBOeLDmaCQRqGQ1N96dOPbCBmF/xYXBCCDYG/f +HaUaJnz96YTgstsbcrYP/p/Qgqtlbw/lQf9IpwMuzbcG1ejt8g89OyDWNyt2ytgU +iaUnFJCos3/Byh0Iah/BsdOueo2/OJl2ZMOBW80orlSgv86cs2y037TL4wKBgQDm +OOyW+MlbowhnIvfoBfwlLEkefnej4nKD6WRLZBcue5Qyf355X06Mhsc9foXlH+6G +D9h/bswiHNdhp6N82rdgPGiHQx/CxiUoE/+b/nvgNO5mw6qLE2EXbG1e8pAMJcyE +bHw+YkawggDfELI036fRj5gki8SeUz8nS1nNgElbyQKBgCRDX9Jh+MwSLu4QBWdt +/fi+lv3K6kun/fI7EOV1vCV/j871tICu7pu5BrOLxAHqoVfU9AUX299/2KjCb5pv +kjogiUK6qWCWBlfuqDNWGCoUGt1rhznUva0nNjSMy5rinBhhjpROZC2pw48lOluP +UuvXsaPph7GTqPuy4Kab12YC +-----END PRIVATE KEY----- diff --git a/src/greentest/3.6/revocation.crl b/src/greentest/3.6/revocation.crl new file mode 100644 index 0000000..6d89b08 --- /dev/null +++ b/src/greentest/3.6/revocation.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTEzMTEyMTE3MDg0N1oXDTIzMDkzMDE3MDg0N1qgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQCNJXC2mVKauEeN3LlQ3ZtM5gkH3ExH ++i4bmJjtJn497WwvvoIeUdrmVXgJQR93RtV37hZwN0SXMLlNmUZPH4rHhihayw4m +unCzVj/OhCCY7/TPjKuJ1O/0XhaLBpBVjQN7R/1ujoRKbSia/CD3vcn7Fqxzw7LK +fSRCKRGTj1CZiuxrphtFchwALXSiFDy9mr2ZKhImcyq1PydfgEzU78APpOkMQsIC +UNJ/cf3c9emzf+dUtcMEcejQ3mynBo4eIGg1EW42bz4q4hSjzQlKcBV0muw5qXhc +HOxH2iTFhQ7SrvVuK/dM14rYM4B5mSX3nRC1kNmXpS9j3wJDhuwmjHed +-----END X509 CRL----- diff --git a/src/greentest/3.6/selfsigned_pythontestdotnet.pem b/src/greentest/3.6/selfsigned_pythontestdotnet.pem new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.6/selfsigned_pythontestdotnet.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/sha256.pem b/src/greentest/3.6/sha256.pem new file mode 100644 index 0000000..d3db4b8 --- /dev/null +++ b/src/greentest/3.6/sha256.pem @@ -0,0 +1,128 @@ +# Certificate chain for https://sha256.tbs-internet.com + 0 s:/C=FR/postalCode=14000/ST=Calvados/L=CAEN/street=22 rue de Bretagne/O=TBS INTERNET/OU=0002 440443810/OU=sha-256 production/CN=sha256.tbs-internet.com + i:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC +-----BEGIN CERTIFICATE----- +MIIGXDCCBUSgAwIBAgIRAKpVmHgg9nfCodAVwcP4siwwDQYJKoZIhvcNAQELBQAw +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMB4XDTEyMDEwNDAwMDAwMFoXDTE0MDIxNzIzNTk1OVowgcsxCzAJBgNV +BAYTAkZSMQ4wDAYDVQQREwUxNDAwMDERMA8GA1UECBMIQ2FsdmFkb3MxDTALBgNV +BAcTBENBRU4xGzAZBgNVBAkTEjIyIHJ1ZSBkZSBCcmV0YWduZTEVMBMGA1UEChMM +VEJTIElOVEVSTkVUMRcwFQYDVQQLEw4wMDAyIDQ0MDQ0MzgxMDEbMBkGA1UECxMS +c2hhLTI1NiBwcm9kdWN0aW9uMSAwHgYDVQQDExdzaGEyNTYudGJzLWludGVybmV0 +LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKQIX/zdJcyxty0m +PM1XQSoSSifueS3AVcgqMsaIKS/u+rYzsv4hQ/qA6vLn5m5/ewUcZDj7zdi6rBVf +PaVNXJ6YinLX0tkaW8TEjeVuZG5yksGZlhCt1CJ1Ho9XLiLaP4uJ7MCoNUntpJ+E +LfrOdgsIj91kPmwjDJeztVcQCvKzhjVJA/KxdInc0JvOATn7rpaSmQI5bvIjufgo +qVsTPwVFzuUYULXBk7KxRT7MiEqnd5HvviNh0285QC478zl3v0I0Fb5El4yD3p49 +IthcRnxzMKc0UhU5ogi0SbONyBfm/mzONVfSxpM+MlyvZmJqrbuuLoEDzJD+t8PU +xSuzgbcCAwEAAaOCAj4wggI6MB8GA1UdIwQYMBaAFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMB0GA1UdDgQWBBT/qTGYdaj+f61c2IRFL/B1eEsM8DAOBgNVHQ8BAf8EBAMC +BaAwDAYDVR0TAQH/BAIwADA0BgNVHSUELTArBggrBgEFBQcDAQYIKwYBBQUHAwIG +CisGAQQBgjcKAwMGCWCGSAGG+EIEATBLBgNVHSAERDBCMEAGCisGAQQB5TcCBAEw +MjAwBggrBgEFBQcCARYkaHR0cHM6Ly93d3cudGJzLWludGVybmV0LmNvbS9DQS9D +UFM0MG0GA1UdHwRmMGQwMqAwoC6GLGh0dHA6Ly9jcmwudGJzLWludGVybmV0LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMC6gLKAqhihodHRwOi8vY3JsLnRicy14NTA5LmNv +bS9UQlNYNTA5Q0FTR0MuY3JsMIGmBggrBgEFBQcBAQSBmTCBljA4BggrBgEFBQcw +AoYsaHR0cDovL2NydC50YnMtaW50ZXJuZXQuY29tL1RCU1g1MDlDQVNHQy5jcnQw +NAYIKwYBBQUHMAKGKGh0dHA6Ly9jcnQudGJzLXg1MDkuY29tL1RCU1g1MDlDQVNH +Qy5jcnQwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLnRicy14NTA5LmNvbTA/BgNV +HREEODA2ghdzaGEyNTYudGJzLWludGVybmV0LmNvbYIbd3d3LnNoYTI1Ni50YnMt +aW50ZXJuZXQuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQA0pOuL8QvAa5yksTbGShzX +ABApagunUGoEydv4YJT1MXy9tTp7DrWaozZSlsqBxrYAXP1d9r2fuKbEniYHxaQ0 +UYaf1VSIlDo1yuC8wE7wxbHDIpQ/E5KAyxiaJ8obtDhFstWAPAH+UoGXq0kj2teN +21sFQ5dXgA95nldvVFsFhrRUNB6xXAcaj0VZFhttI0ZfQZmQwEI/P+N9Jr40OGun +aa+Dn0TMeUH4U20YntfLbu2nDcJcYfyurm+8/0Tr4HznLnedXu9pCPYj0TaddrgT +XO0oFiyy7qGaY6+qKh71yD64Y3ycCJ/HR9Wm39mjZYc9ezYwT4noP6r7Lk8YO7/q +-----END CERTIFICATE----- + 1 s:/C=FR/ST=Calvados/L=Caen/O=TBS INTERNET/OU=Terms and Conditions: http://www.tbs-internet.com/CA/repository/OU=TBS INTERNET CA/CN=TBS X509 CA SGC + i:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root +-----BEGIN CERTIFICATE----- +MIIFVjCCBD6gAwIBAgIQXpDZ0ETJMV02WTx3GTnhhTANBgkqhkiG9w0BAQUFADBv +MQswCQYDVQQGEwJTRTEUMBIGA1UEChMLQWRkVHJ1c3QgQUIxJjAkBgNVBAsTHUFk +ZFRydXN0IEV4dGVybmFsIFRUUCBOZXR3b3JrMSIwIAYDVQQDExlBZGRUcnVzdCBF +eHRlcm5hbCBDQSBSb290MB4XDTA1MTIwMTAwMDAwMFoXDTE5MDYyNDE5MDYzMFow +gcQxCzAJBgNVBAYTAkZSMREwDwYDVQQIEwhDYWx2YWRvczENMAsGA1UEBxMEQ2Fl +bjEVMBMGA1UEChMMVEJTIElOVEVSTkVUMUgwRgYDVQQLEz9UZXJtcyBhbmQgQ29u +ZGl0aW9uczogaHR0cDovL3d3dy50YnMtaW50ZXJuZXQuY29tL0NBL3JlcG9zaXRv +cnkxGDAWBgNVBAsTD1RCUyBJTlRFUk5FVCBDQTEYMBYGA1UEAxMPVEJTIFg1MDkg +Q0EgU0dDMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsgOkO3f7wzN6 +rOjg45tR5vjBfzK7qmV9IBxb/QW9EEXxG+E7FNhZqQLtwGBKoSsHTnQqV75wWMk0 +9tinWvftBkSpj5sTi/8cbzJfUvTSVYh3Qxv6AVVjMMH/ruLjE6y+4PoaPs8WoYAQ +ts5R4Z1g8c/WnTepLst2x0/Wv7GmuoQi+gXvHU6YrBiu7XkeYhzc95QdviWSJRDk +owhb5K43qhcvjRmBfO/paGlCliDGZp8mHwrI21mwobWpVjTxZRwYO3bd4+TGcI4G +Ie5wmHwE8F7SK1tgSqbBacKjDa93j7txKkfz/Yd2n7TGqOXiHPsJpG655vrKtnXk +9vs1zoDeJQIDAQABo4IBljCCAZIwHQYDVR0OBBYEFAdEdoWTKLx/bXjSCuv6TEvf +2YIfMA4GA1UdDwEB/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMCAGA1UdJQQZ +MBcGCisGAQQBgjcKAwMGCWCGSAGG+EIEATAYBgNVHSAEETAPMA0GCysGAQQBgOU3 +AgQBMHsGA1UdHwR0MHIwOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0Fk +ZFRydXN0RXh0ZXJuYWxDQVJvb3QuY3JsMDagNKAyhjBodHRwOi8vY3JsLmNvbW9k +by5uZXQvQWRkVHJ1c3RFeHRlcm5hbENBUm9vdC5jcmwwgYAGCCsGAQUFBwEBBHQw +cjA4BggrBgEFBQcwAoYsaHR0cDovL2NydC5jb21vZG9jYS5jb20vQWRkVHJ1c3RV +VE5TR0NDQS5jcnQwNgYIKwYBBQUHMAKGKmh0dHA6Ly9jcnQuY29tb2RvLm5ldC9B +ZGRUcnVzdFVUTlNHQ0NBLmNydDARBglghkgBhvhCAQEEBAMCAgQwDQYJKoZIhvcN +AQEFBQADggEBAK2zEzs+jcIrVK9oDkdDZNvhuBYTdCfpxfFs+OAujW0bIfJAy232 +euVsnJm6u/+OrqKudD2tad2BbejLLXhMZViaCmK7D9nrXHx4te5EP8rL19SUVqLY +1pTnv5dhNgEgvA7n5lIzDSYs7yRLsr7HJsYPr6SeYSuZizyX1SNz7ooJ32/F3X98 +RB0Mlc/E0OyOrkQ9/y5IrnpnaSora8CnUrV5XNOg+kyCz9edCyx4D5wXYcwZPVWz +8aDqquESrezPyjtfi4WRO4s/VD3HLZvOxzMrWAVYCDG9FxaOhF0QGuuG1F7F3GKV +v6prNyCl016kRl2j1UT+a7gLd8fA25A4C9E= +-----END CERTIFICATE----- + 2 s:/C=SE/O=AddTrust AB/OU=AddTrust External TTP Network/CN=AddTrust External CA Root + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEZjCCA06gAwIBAgIQUSYKkxzif5zDpV954HKugjANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw0wNTA2MDcwODA5MTBaFw0xOTA2MjQxOTA2MzBaMG8xCzAJBgNVBAYT +AlNFMRQwEgYDVQQKEwtBZGRUcnVzdCBBQjEmMCQGA1UECxMdQWRkVHJ1c3QgRXh0 +ZXJuYWwgVFRQIE5ldHdvcmsxIjAgBgNVBAMTGUFkZFRydXN0IEV4dGVybmFsIENB +IFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC39xoz5vIABC05 +4E5b7R+8bA/Ntfojts7emxEzl6QpTH2Tn71KvJPtAxrjj8/lbVBa1pcplFqAsEl6 +2y6V/bjKvzc4LR4+kUGtcFbH8E8/6DKedMrIkFTpxl8PeJ2aQDwOrGGqXhSPnoeh +alDc15pOrwWzpnGUnHGzUGAKxxOdOAeGAqjpqGkmGJCrTLBPI6s6T4TY386f4Wlv +u9dC12tE5Met7m1BX3JacQg3s3llpFmglDf3AC8NwpJy2tA4ctsUqEXEXSp9t7TW +xO6szRNEt8kr3UMAJfphuWlqWCMRt6czj1Z1WfXNKddGtworZbbTQm8Vsrh7++/p +XVPVNFonAgMBAAGjgdgwgdUwHwYDVR0jBBgwFoAUUzLRs89/+uDxoF2FTpLSnkUd +tE8wHQYDVR0OBBYEFK29mHo0tCb3+sQmVO8DveAky1QaMA4GA1UdDwEB/wQEAwIB +BjAPBgNVHRMBAf8EBTADAQH/MBEGCWCGSAGG+EIBAQQEAwIBAjAgBgNVHSUEGTAX +BgorBgEEAYI3CgMDBglghkgBhvhCBAEwPQYDVR0fBDYwNDAyoDCgLoYsaHR0cDov +L2NybC51c2VydHJ1c3QuY29tL1VUTi1EQVRBQ29ycFNHQy5jcmwwDQYJKoZIhvcN +AQEFBQADggEBAMbuUxdoFLJRIh6QWA2U/b3xcOWGLcM2MY9USEbnLQg3vGwKYOEO +rVE04BKT6b64q7gmtOmWPSiPrmQH/uAB7MXjkesYoPF1ftsK5p+R26+udd8jkWjd +FwBaS/9kbHDrARrQkNnHptZt9hPk/7XJ0h4qy7ElQyZ42TCbTg0evmnv3+r+LbPM ++bDdtRTKkdSytaX7ARmjR3mfnYyVhzT4HziS2jamEfpr62vp3EV4FTkG101B5CHI +3C+H0be/SGB1pWLLJN47YaApIKa+xWycxOkKaSLvkTr6Jq/RW0GnOuL4OAdCq8Fb ++M5tug8EPzI0rNwEKNdwMBQmBsTkm5jVz3g= +-----END CERTIFICATE----- + 3 s:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC + i:/C=US/ST=UT/L=Salt Lake City/O=The USERTRUST Network/OU=http://www.usertrust.com/CN=UTN - DATACorp SGC +-----BEGIN CERTIFICATE----- +MIIEXjCCA0agAwIBAgIQRL4Mi1AAIbQR0ypoBqmtaTANBgkqhkiG9w0BAQUFADCB +kzELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAlVUMRcwFQYDVQQHEw5TYWx0IExha2Ug +Q2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBOZXR3b3JrMSEwHwYDVQQLExho +dHRwOi8vd3d3LnVzZXJ0cnVzdC5jb20xGzAZBgNVBAMTElVUTiAtIERBVEFDb3Jw +IFNHQzAeFw05OTA2MjQxODU3MjFaFw0xOTA2MjQxOTA2MzBaMIGTMQswCQYDVQQG +EwJVUzELMAkGA1UECBMCVVQxFzAVBgNVBAcTDlNhbHQgTGFrZSBDaXR5MR4wHAYD +VQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxITAfBgNVBAsTGGh0dHA6Ly93d3cu +dXNlcnRydXN0LmNvbTEbMBkGA1UEAxMSVVROIC0gREFUQUNvcnAgU0dDMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3+5YEKIrblXEjr8uRgnn4AgPLit6 +E5Qbvfa2gI5lBZMAHryv4g+OGQ0SR+ysraP6LnD43m77VkIVni5c7yPeIbkFdicZ +D0/Ww5y0vpQZY/KmEQrrU0icvvIpOxboGqBMpsn0GFlowHDyUwDAXlCCpVZvNvlK +4ESGoE1O1kduSUrLZ9emxAW5jh70/P/N5zbgnAVssjMiFdC04MwXwLLA9P4yPykq +lXvY8qdOD1R8oQ2AswkDwf9c3V6aPryuvEeKaq5xyh+xKrhfQgUL7EYw0XILyulW +bfXv33i+Ybqypa4ETLyorGkVl73v67SMvzX41MPRKA5cOp9wGDMgd8SirwIDAQAB +o4GrMIGoMAsGA1UdDwQEAwIBxjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRT +MtGzz3/64PGgXYVOktKeRR20TzA9BgNVHR8ENjA0MDKgMKAuhixodHRwOi8vY3Js +LnVzZXJ0cnVzdC5jb20vVVROLURBVEFDb3JwU0dDLmNybDAqBgNVHSUEIzAhBggr +BgEFBQcDAQYKKwYBBAGCNwoDAwYJYIZIAYb4QgQBMA0GCSqGSIb3DQEBBQUAA4IB +AQAnNZcAiosovcYzMB4p/OL31ZjUQLtgyr+rFywJNn9Q+kHcrpY6CiM+iVnJowft +Gzet/Hy+UUla3joKVAgWRcKZsYfNjGjgaQPpxE6YsjuMFrMOoAyYUJuTqXAJyCyj +j98C5OBxOvG0I3KgqgHf35g+FFCgMSa9KOlaMCZ1+XtgHI3zzVAmbQQnmt/VDUVH +KWss5nbZqSl9Mt3JNjy9rjXxEZ4du5A/EkdOjtd+D2JzHVImOBwYSf0wdJrE5SIv +2MCN7ZF6TACPcn9d2t0bi0Vr591pl6jFVkwPDPafepE39peC4N1xaf92P2BNPM/3 +mfnGV/TJVTl4uix5yaaIK/QI +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/ssl_cert.pem b/src/greentest/3.6/ssl_cert.pem new file mode 100644 index 0000000..47a7d7e --- /dev/null +++ b/src/greentest/3.6/ssl_cert.pem @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/src/greentest/3.6/ssl_key.passwd.pem b/src/greentest/3.6/ssl_key.passwd.pem new file mode 100644 index 0000000..2524672 --- /dev/null +++ b/src/greentest/3.6/ssl_key.passwd.pem @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- diff --git a/src/greentest/3.6/ssl_key.pem b/src/greentest/3.6/ssl_key.pem new file mode 100644 index 0000000..3fd3bbd --- /dev/null +++ b/src/greentest/3.6/ssl_key.pem @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm +LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 +ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP +USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt +CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq +SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK +UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y +BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ +ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 +oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik +eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F +0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS +x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ +SPIXQuT8RMPDVNQ= +-----END PRIVATE KEY----- diff --git a/src/greentest/3.6/test_asyncore.py b/src/greentest/3.6/test_asyncore.py new file mode 100644 index 0000000..07edf22 --- /dev/null +++ b/src/greentest/3.6/test_asyncore.py @@ -0,0 +1,847 @@ +import asyncore +import unittest +import select +import os +import socket +import sys +import time +import errno +import struct + +from test import support +from io import BytesIO + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +try: + import threading +except ImportError: + threading = None + +TIMEOUT = 3 +HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') + +class dummysocket: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def fileno(self): + return 42 + +class dummychannel: + def __init__(self): + self.socket = dummysocket() + + def close(self): + self.socket.close() + +class exitingdummy: + def __init__(self): + pass + + def handle_read_event(self): + raise asyncore.ExitNow() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + +class crashingdummy: + def __init__(self): + self.error_handled = False + + def handle_read_event(self): + raise Exception() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + + def handle_error(self): + self.error_handled = True + +# used when testing senders; just collects what it gets until newline is sent +def capture_server(evt, buf, serv): + try: + serv.listen() + conn, addr = serv.accept() + except socket.timeout: + pass + else: + n = 200 + start = time.time() + while n > 0 and time.time() - start < 3.0: + r, w, e = select.select([conn], [], [], 0.1) + if r: + n -= 1 + data = conn.recv(10) + # keep everything except for the newline terminator + buf.write(data.replace(b'\n', b'')) + if b'\n' in data: + break + time.sleep(0.01) + + conn.close() + finally: + serv.close() + evt.set() + +def bind_af_aware(sock, addr): + """Helper function to bind a socket according to its family.""" + if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: + # Make sure the path doesn't exist. + support.unlink(addr) + support.bind_unix_socket(sock, addr) + else: + sock.bind(addr) + + +class HelperFunctionTests(unittest.TestCase): + def test_readwriteexc(self): + # Check exception handling behavior of read, write and _exception + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore read/write/_exception calls + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.read, tr1) + self.assertRaises(asyncore.ExitNow, asyncore.write, tr1) + self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + asyncore.read(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore.write(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore._exception(tr2) + self.assertEqual(tr2.error_handled, True) + + # asyncore.readwrite uses constants in the select module that + # are not present in Windows systems (see this thread: + # http://mail.python.org/pipermail/python-list/2001-October/109973.html) + # These constants should be present as long as poll is available + + @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') + def test_readwrite(self): + # Check that correct methods are called by readwrite() + + attributes = ('read', 'expt', 'write', 'closed', 'error_handled') + + expected = ( + (select.POLLIN, 'read'), + (select.POLLPRI, 'expt'), + (select.POLLOUT, 'write'), + (select.POLLERR, 'closed'), + (select.POLLHUP, 'closed'), + (select.POLLNVAL, 'closed'), + ) + + class testobj: + def __init__(self): + self.read = False + self.write = False + self.closed = False + self.expt = False + self.error_handled = False + + def handle_read_event(self): + self.read = True + + def handle_write_event(self): + self.write = True + + def handle_close(self): + self.closed = True + + def handle_expt_event(self): + self.expt = True + + def handle_error(self): + self.error_handled = True + + for flag, expectedattr in expected: + tobj = testobj() + self.assertEqual(getattr(tobj, expectedattr), False) + asyncore.readwrite(tobj, flag) + + # Only the attribute modified by the routine we expect to be + # called should be True. + for attr in attributes: + self.assertEqual(getattr(tobj, attr), attr==expectedattr) + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore readwrite call + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + self.assertEqual(tr2.error_handled, False) + asyncore.readwrite(tr2, flag) + self.assertEqual(tr2.error_handled, True) + + def test_closeall(self): + self.closeall_check(False) + + def test_closeall_default(self): + self.closeall_check(True) + + def closeall_check(self, usedefault): + # Check that close_all() closes everything in a given map + + l = [] + testmap = {} + for i in range(10): + c = dummychannel() + l.append(c) + self.assertEqual(c.socket.closed, False) + testmap[i] = c + + if usedefault: + socketmap = asyncore.socket_map + try: + asyncore.socket_map = testmap + asyncore.close_all() + finally: + testmap, asyncore.socket_map = asyncore.socket_map, socketmap + else: + asyncore.close_all(testmap) + + self.assertEqual(len(testmap), 0) + + for c in l: + self.assertEqual(c.socket.closed, True) + + def test_compact_traceback(self): + try: + raise Exception("I don't like spam!") + except: + real_t, real_v, real_tb = sys.exc_info() + r = asyncore.compact_traceback() + else: + self.fail("Expected exception") + + (f, function, line), t, v, info = r + self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py') + self.assertEqual(function, 'test_compact_traceback') + self.assertEqual(t, real_t) + self.assertEqual(v, real_v) + self.assertEqual(info, '[%s|%s|%s]' % (f, function, line)) + + +class DispatcherTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + def test_basic(self): + d = asyncore.dispatcher() + self.assertEqual(d.readable(), True) + self.assertEqual(d.writable(), True) + + def test_repr(self): + d = asyncore.dispatcher() + self.assertEqual(repr(d), '' % id(d)) + + def test_log(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log() (to stderr) + l1 = "Lovely spam! Wonderful spam!" + l2 = "I don't like spam!" + with support.captured_stderr() as stderr: + d.log(l1) + d.log(l2) + + lines = stderr.getvalue().splitlines() + self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2]) + + def test_log_info(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log_info() (to stdout via print) + l1 = "Have you got anything without spam?" + l2 = "Why can't she have egg bacon spam and sausage?" + l3 = "THAT'S got spam in it!" + with support.captured_stdout() as stdout: + d.log_info(l1, 'EGGS') + d.log_info(l2) + d.log_info(l3, 'SPAM') + + lines = stdout.getvalue().splitlines() + expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3] + self.assertEqual(lines, expected) + + def test_unhandled(self): + d = asyncore.dispatcher() + d.ignore_log_types = () + + # capture output of dispatcher.log_info() (to stdout via print) + with support.captured_stdout() as stdout: + d.handle_expt() + d.handle_read() + d.handle_write() + d.handle_connect() + + lines = stdout.getvalue().splitlines() + expected = ['warning: unhandled incoming priority event', + 'warning: unhandled read event', + 'warning: unhandled write event', + 'warning: unhandled connect event'] + self.assertEqual(lines, expected) + + def test_strerror(self): + # refers to bug #8573 + err = asyncore._strerror(errno.EPERM) + if hasattr(os, 'strerror'): + self.assertEqual(err, os.strerror(errno.EPERM)) + err = asyncore._strerror(-1) + self.assertTrue(err != "") + + +class dispatcherwithsend_noread(asyncore.dispatcher_with_send): + def readable(self): + return False + + def handle_connect(self): + pass + + +class DispatcherWithSendTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + @unittest.skipUnless(threading, 'Threading required for this test.') + @support.reap_threads + def test_send(self): + evt = threading.Event() + sock = socket.socket() + sock.settimeout(3) + port = support.bind_port(sock) + + cap = BytesIO() + args = (evt, cap, sock) + t = threading.Thread(target=capture_server, args=args) + t.start() + try: + # wait a little longer for the server to initialize (it sometimes + # refuses connections on slow machines without this wait) + time.sleep(0.2) + + data = b"Suppose there isn't a 16-ton weight?" + d = dispatcherwithsend_noread() + d.create_socket() + d.connect((support.HOST, port)) + + # give time for socket to connect + time.sleep(0.1) + + d.send(data) + d.send(data) + d.send(b'\n') + + n = 1000 + while d.out_buffer and n > 0: + asyncore.poll() + n -= 1 + + evt.wait() + + self.assertEqual(cap.getvalue(), data*2) + finally: + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") + + +@unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), + 'asyncore.file_wrapper required') +class FileWrapperTest(unittest.TestCase): + def setUp(self): + self.d = b"It's not dead, it's sleeping!" + with open(support.TESTFN, 'wb') as file: + file.write(self.d) + + def tearDown(self): + support.unlink(support.TESTFN) + + def test_recv(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + w = asyncore.file_wrapper(fd) + os.close(fd) + + self.assertNotEqual(w.fd, fd) + self.assertNotEqual(w.fileno(), fd) + self.assertEqual(w.recv(13), b"It's not dead") + self.assertEqual(w.read(6), b", it's") + w.close() + self.assertRaises(OSError, w.read, 1) + + def test_send(self): + d1 = b"Come again?" + d2 = b"I want to buy some cheese." + fd = os.open(support.TESTFN, os.O_WRONLY | os.O_APPEND) + w = asyncore.file_wrapper(fd) + os.close(fd) + + w.write(d1) + w.send(d2) + w.close() + with open(support.TESTFN, 'rb') as file: + self.assertEqual(file.read(), self.d + d1 + d2) + + @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'), + 'asyncore.file_dispatcher required') + def test_dispatcher(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + data = [] + class FileDispatcher(asyncore.file_dispatcher): + def handle_read(self): + data.append(self.recv(29)) + s = FileDispatcher(fd) + os.close(fd) + asyncore.loop(timeout=0.01, use_poll=True, count=2) + self.assertEqual(b"".join(data), self.d) + + def test_resource_warning(self): + # Issue #11453 + fd = os.open(support.TESTFN, os.O_RDONLY) + f = asyncore.file_wrapper(fd) + + os.close(fd) + with support.check_warnings(('', ResourceWarning)): + f = None + support.gc_collect() + + def test_close_twice(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + f = asyncore.file_wrapper(fd) + os.close(fd) + + os.close(f.fd) # file_wrapper dupped fd + with self.assertRaises(OSError): + f.close() + + self.assertEqual(f.fd, -1) + # calling close twice should not fail + f.close() + + +class BaseTestHandler(asyncore.dispatcher): + + def __init__(self, sock=None): + asyncore.dispatcher.__init__(self, sock) + self.flag = False + + def handle_accept(self): + raise Exception("handle_accept not supposed to be called") + + def handle_accepted(self): + raise Exception("handle_accepted not supposed to be called") + + def handle_connect(self): + raise Exception("handle_connect not supposed to be called") + + def handle_expt(self): + raise Exception("handle_expt not supposed to be called") + + def handle_close(self): + raise Exception("handle_close not supposed to be called") + + def handle_error(self): + raise + + +class BaseServer(asyncore.dispatcher): + """A server which listens on an address and dispatches the + connection to a handler. + """ + + def __init__(self, family, addr, handler=BaseTestHandler): + asyncore.dispatcher.__init__(self) + self.create_socket(family) + self.set_reuse_addr() + bind_af_aware(self.socket, addr) + self.listen(5) + self.handler = handler + + @property + def address(self): + return self.socket.getsockname() + + def handle_accepted(self, sock, addr): + self.handler(sock) + + def handle_error(self): + raise + + +class BaseClient(BaseTestHandler): + + def __init__(self, family, address): + BaseTestHandler.__init__(self) + self.create_socket(family) + self.connect(address) + + def handle_connect(self): + pass + + +class BaseTestAPI: + + def tearDown(self): + asyncore.close_all(ignore_all=True) + + def loop_waiting_for_flag(self, instance, timeout=5): + timeout = float(timeout) / 100 + count = 100 + while asyncore.socket_map and count > 0: + asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) + if instance.flag: + return + count -= 1 + time.sleep(timeout) + self.fail("flag not set") + + def test_handle_connect(self): + # make sure handle_connect is called on connect() + + class TestClient(BaseClient): + def handle_connect(self): + self.flag = True + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_accept(self): + # make sure handle_accept() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self, family, addr): + BaseTestHandler.__init__(self) + self.create_socket(family) + bind_af_aware(self.socket, addr) + self.listen(5) + self.address = self.socket.getsockname() + + def handle_accept(self): + self.flag = True + + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) + self.loop_waiting_for_flag(server) + + def test_handle_accepted(self): + # make sure handle_accepted() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self, family, addr): + BaseTestHandler.__init__(self) + self.create_socket(family) + bind_af_aware(self.socket, addr) + self.listen(5) + self.address = self.socket.getsockname() + + def handle_accept(self): + asyncore.dispatcher.handle_accept(self) + + def handle_accepted(self, sock, addr): + sock.close() + self.flag = True + + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) + self.loop_waiting_for_flag(server) + + + def test_handle_read(self): + # make sure handle_read is called on data received + + class TestClient(BaseClient): + def handle_read(self): + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.send(b'x' * 1024) + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_write(self): + # make sure handle_write is called + + class TestClient(BaseClient): + def handle_write(self): + self.flag = True + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close(self): + # make sure handle_close is called when the other end closes + # the connection + + class TestClient(BaseClient): + + def handle_read(self): + # in order to make handle_close be called we are supposed + # to make at least one recv() call + self.recv(1024) + + def handle_close(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.close() + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close_after_conn_broken(self): + # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and + # #11265). + + data = b'\0' * 128 + + class TestClient(BaseClient): + + def handle_write(self): + self.send(data) + + def handle_close(self): + self.flag = True + self.close() + + def handle_expt(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + + def handle_read(self): + self.recv(len(data)) + self.close() + + def writable(self): + return False + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + @unittest.skipIf(sys.platform.startswith("sunos"), + "OOB support is broken on Solaris") + def test_handle_expt(self): + # Make sure handle_expt is called on OOB data received. + # Note: this might fail on some platforms as OOB data is + # tenuously supported and rarely used. + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + + if sys.platform == "darwin" and self.use_poll: + self.skipTest("poll may fail on macOS; see issue #28087") + + class TestClient(BaseClient): + def handle_expt(self): + self.socket.recv(1024, socket.MSG_OOB) + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB) + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_error(self): + + class TestClient(BaseClient): + def handle_write(self): + 1.0 / 0 + def handle_error(self): + self.flag = True + try: + raise + except ZeroDivisionError: + pass + else: + raise Exception("exception not raised") + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_connection_attributes(self): + server = BaseServer(self.family, self.addr) + client = BaseClient(self.family, server.address) + + # we start disconnected + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + # this can't be taken for granted across all platforms + #self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # execute some loops so that client connects to server + asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100) + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertTrue(client.connected) + self.assertFalse(client.accepting) + + # disconnect the client + client.close() + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # stop serving + server.close() + self.assertFalse(server.connected) + self.assertFalse(server.accepting) + + def test_create_socket(self): + s = asyncore.dispatcher() + s.create_socket(self.family) + self.assertEqual(s.socket.family, self.family) + SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) + sock_type = socket.SOCK_STREAM | SOCK_NONBLOCK + if hasattr(socket, 'SOCK_CLOEXEC'): + self.assertIn(s.socket.type, + (sock_type | socket.SOCK_CLOEXEC, sock_type)) + else: + self.assertEqual(s.socket.type, sock_type) + + def test_bind(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + s1 = asyncore.dispatcher() + s1.create_socket(self.family) + s1.bind(self.addr) + s1.listen(5) + port = s1.socket.getsockname()[1] + + s2 = asyncore.dispatcher() + s2.create_socket(self.family) + # EADDRINUSE indicates the socket was correctly bound + self.assertRaises(OSError, s2.bind, (self.addr[0], port)) + + def test_set_reuse_addr(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + + with socket.socket(self.family) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except OSError: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket(self.family)) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.socket.close() + s.create_socket(self.family) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + + @unittest.skipUnless(threading, 'Threading required for this test.') + @support.reap_threads + def test_quick_connect(self): + # see: http://bugs.python.org/issue10340 + if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): + self.skipTest("test specific to AF_INET and AF_INET6") + + server = BaseServer(self.family, self.addr) + # run the thread 500 ms: the socket should be connected in 200 ms + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, + count=5)) + t.start() + try: + with socket.socket(self.family, socket.SOCK_STREAM) as s: + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + + try: + s.connect(server.address) + except OSError: + pass + finally: + t.join(timeout=TIMEOUT) + if t.is_alive(): + self.fail("join() timed out") + +class TestAPI_UseIPv4Sockets(BaseTestAPI): + family = socket.AF_INET + addr = (support.HOST, 0) + +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 support required') +class TestAPI_UseIPv6Sockets(BaseTestAPI): + family = socket.AF_INET6 + addr = (support.HOSTv6, 0) + +@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required') +class TestAPI_UseUnixSockets(BaseTestAPI): + if HAS_UNIX_SOCKETS: + family = socket.AF_UNIX + addr = support.TESTFN + + def tearDown(self): + support.unlink(self.addr) + BaseTestAPI.tearDown(self) + +class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets, unittest.TestCase): + use_poll = True + +class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets, unittest.TestCase): + use_poll = True + +class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets, unittest.TestCase): + use_poll = True + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.6/test_ftplib.py b/src/greentest/3.6/test_ftplib.py new file mode 100644 index 0000000..b593313 --- /dev/null +++ b/src/greentest/3.6/test_ftplib.py @@ -0,0 +1,1091 @@ +"""Test script for ftplib module.""" + +# Modified by Giampaolo Rodola' to test FTP class, IPv6 and TLS +# environment + +import ftplib +import asyncore +import asynchat +import socket +import io +import errno +import os +import time +try: + import ssl +except ImportError: + ssl = None + +from unittest import TestCase, skipUnless +from test import support +from test.support import HOST, HOSTv6 +threading = support.import_module('threading') + +TIMEOUT = 3 +# the dummy data returned by server over the data channel when +# RETR, LIST, NLST, MLSD commands are issued +RETR_DATA = 'abcde12345\r\n' * 1000 +LIST_DATA = 'foo\r\nbar\r\n' +NLST_DATA = 'foo\r\nbar\r\n' +MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n" + "type=pdir;perm=e;unique==keVO1+d?3; ..\r\n" + "type=OS.unix=slink:/foobar;perm=;unique==keVO1+4G4; foobar\r\n" + "type=OS.unix=chr-13/29;perm=;unique==keVO1+5G4; device\r\n" + "type=OS.unix=blk-11/108;perm=;unique==keVO1+6G4; block\r\n" + "type=file;perm=awr;unique==keVO1+8G4; writable\r\n" + "type=dir;perm=cpmel;unique==keVO1+7G4; promiscuous\r\n" + "type=dir;perm=;unique==keVO1+1t2; no-exec\r\n" + "type=file;perm=r;unique==keVO1+EG4; two words\r\n" + "type=file;perm=r;unique==keVO1+IH4; leading space\r\n" + "type=file;perm=r;unique==keVO1+1G4; file1\r\n" + "type=dir;perm=cpmel;unique==keVO1+7G4; incoming\r\n" + "type=file;perm=r;unique==keVO1+1G4; file2\r\n" + "type=file;perm=r;unique==keVO1+1G4; file3\r\n" + "type=file;perm=r;unique==keVO1+1G4; file4\r\n") + + +class DummyDTPHandler(asynchat.async_chat): + dtp_conn_closed = False + + def __init__(self, conn, baseclass): + asynchat.async_chat.__init__(self, conn) + self.baseclass = baseclass + self.baseclass.last_received_data = '' + + def handle_read(self): + self.baseclass.last_received_data += self.recv(1024).decode('ascii') + + def handle_close(self): + # XXX: this method can be called many times in a row for a single + # connection, including in clear-text (non-TLS) mode. + # (behaviour witnessed with test_data_connection) + if not self.dtp_conn_closed: + self.baseclass.push('226 transfer complete') + self.close() + self.dtp_conn_closed = True + + def push(self, what): + if self.baseclass.next_data is not None: + what = self.baseclass.next_data + self.baseclass.next_data = None + if not what: + return self.close_when_done() + super(DummyDTPHandler, self).push(what.encode('ascii')) + + def handle_error(self): + raise Exception + + +class DummyFTPHandler(asynchat.async_chat): + + dtp_handler = DummyDTPHandler + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + # tells the socket to handle urgent data inline (ABOR command) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_OOBINLINE, 1) + self.set_terminator(b"\r\n") + self.in_buffer = [] + self.dtp = None + self.last_received_cmd = None + self.last_received_data = '' + self.next_response = '' + self.next_data = None + self.rest = None + self.next_retr_data = RETR_DATA + self.push('220 welcome') + + def collect_incoming_data(self, data): + self.in_buffer.append(data) + + def found_terminator(self): + line = b''.join(self.in_buffer).decode('ascii') + self.in_buffer = [] + if self.next_response: + self.push(self.next_response) + self.next_response = '' + cmd = line.split(' ')[0].lower() + self.last_received_cmd = cmd + space = line.find(' ') + if space != -1: + arg = line[space + 1:] + else: + arg = "" + if hasattr(self, 'cmd_' + cmd): + method = getattr(self, 'cmd_' + cmd) + method(arg) + else: + self.push('550 command "%s" not understood.' %cmd) + + def handle_error(self): + raise Exception + + def push(self, data): + asynchat.async_chat.push(self, data.encode('ascii') + b'\r\n') + + def cmd_port(self, arg): + addr = list(map(int, arg.split(','))) + ip = '%d.%d.%d.%d' %tuple(addr[:4]) + port = (addr[4] * 256) + addr[5] + s = socket.create_connection((ip, port), timeout=TIMEOUT) + self.dtp = self.dtp_handler(s, baseclass=self) + self.push('200 active data connection established') + + def cmd_pasv(self, arg): + with socket.socket() as sock: + sock.bind((self.socket.getsockname()[0], 0)) + sock.listen() + sock.settimeout(TIMEOUT) + ip, port = sock.getsockname()[:2] + ip = ip.replace('.', ','); p1 = port / 256; p2 = port % 256 + self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2)) + conn, addr = sock.accept() + self.dtp = self.dtp_handler(conn, baseclass=self) + + def cmd_eprt(self, arg): + af, ip, port = arg.split(arg[0])[1:-1] + port = int(port) + s = socket.create_connection((ip, port), timeout=TIMEOUT) + self.dtp = self.dtp_handler(s, baseclass=self) + self.push('200 active data connection established') + + def cmd_epsv(self, arg): + with socket.socket(socket.AF_INET6) as sock: + sock.bind((self.socket.getsockname()[0], 0)) + sock.listen() + sock.settimeout(TIMEOUT) + port = sock.getsockname()[1] + self.push('229 entering extended passive mode (|||%d|)' %port) + conn, addr = sock.accept() + self.dtp = self.dtp_handler(conn, baseclass=self) + + def cmd_echo(self, arg): + # sends back the received string (used by the test suite) + self.push(arg) + + def cmd_noop(self, arg): + self.push('200 noop ok') + + def cmd_user(self, arg): + self.push('331 username ok') + + def cmd_pass(self, arg): + self.push('230 password ok') + + def cmd_acct(self, arg): + self.push('230 acct ok') + + def cmd_rnfr(self, arg): + self.push('350 rnfr ok') + + def cmd_rnto(self, arg): + self.push('250 rnto ok') + + def cmd_dele(self, arg): + self.push('250 dele ok') + + def cmd_cwd(self, arg): + self.push('250 cwd ok') + + def cmd_size(self, arg): + self.push('250 1000') + + def cmd_mkd(self, arg): + self.push('257 "%s"' %arg) + + def cmd_rmd(self, arg): + self.push('250 rmd ok') + + def cmd_pwd(self, arg): + self.push('257 "pwd ok"') + + def cmd_type(self, arg): + self.push('200 type ok') + + def cmd_quit(self, arg): + self.push('221 quit ok') + self.close() + + def cmd_abor(self, arg): + self.push('226 abor ok') + + def cmd_stor(self, arg): + self.push('125 stor ok') + + def cmd_rest(self, arg): + self.rest = arg + self.push('350 rest ok') + + def cmd_retr(self, arg): + self.push('125 retr ok') + if self.rest is not None: + offset = int(self.rest) + else: + offset = 0 + self.dtp.push(self.next_retr_data[offset:]) + self.dtp.close_when_done() + self.rest = None + + def cmd_list(self, arg): + self.push('125 list ok') + self.dtp.push(LIST_DATA) + self.dtp.close_when_done() + + def cmd_nlst(self, arg): + self.push('125 nlst ok') + self.dtp.push(NLST_DATA) + self.dtp.close_when_done() + + def cmd_opts(self, arg): + self.push('200 opts ok') + + def cmd_mlsd(self, arg): + self.push('125 mlsd ok') + self.dtp.push(MLSD_DATA) + self.dtp.close_when_done() + + def cmd_setlongretr(self, arg): + # For testing. Next RETR will return long line. + self.next_retr_data = 'x' * int(arg) + self.push('125 setlongretr ok') + + +class DummyFTPServer(asyncore.dispatcher, threading.Thread): + + handler = DummyFTPHandler + + def __init__(self, address, af=socket.AF_INET): + threading.Thread.__init__(self) + asyncore.dispatcher.__init__(self) + self.create_socket(af, socket.SOCK_STREAM) + self.bind(address) + self.listen(5) + self.active = False + self.active_lock = threading.Lock() + self.host, self.port = self.socket.getsockname()[:2] + self.handler_instance = None + + def start(self): + assert not self.active + self.__flag = threading.Event() + threading.Thread.start(self) + self.__flag.wait() + + def run(self): + self.active = True + self.__flag.set() + while self.active and asyncore.socket_map: + self.active_lock.acquire() + asyncore.loop(timeout=0.1, count=1) + self.active_lock.release() + asyncore.close_all(ignore_all=True) + + def stop(self): + assert self.active + self.active = False + self.join() + + def handle_accepted(self, conn, addr): + self.handler_instance = self.handler(conn) + + def handle_connect(self): + self.close() + handle_read = handle_connect + + def writable(self): + return 0 + + def handle_error(self): + raise Exception + + +if ssl is not None: + + CERTFILE = os.path.join(os.path.dirname(__file__), "keycert3.pem") + CAFILE = os.path.join(os.path.dirname(__file__), "pycacert.pem") + + class SSLConnection(asyncore.dispatcher): + """An asyncore.dispatcher subclass supporting TLS/SSL.""" + + _ssl_accepting = False + _ssl_closing = False + + def secure_connection(self): + context = ssl.SSLContext() + context.load_cert_chain(CERTFILE) + socket = context.wrap_socket(self.socket, + suppress_ragged_eofs=False, + server_side=True, + do_handshake_on_connect=False) + self.del_channel() + self.set_socket(socket) + self._ssl_accepting = True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + elif err.args[0] == ssl.SSL_ERROR_EOF: + return self.handle_close() + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def _do_ssl_shutdown(self): + self._ssl_closing = True + try: + self.socket = self.socket.unwrap() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + except OSError as err: + # Any "socket error" corresponds to a SSL_ERROR_SYSCALL return + # from OpenSSL's SSL_shutdown(), corresponding to a + # closed socket condition. See also: + # http://www.mail-archive.com/openssl-users@openssl.org/msg60710.html + pass + self._ssl_closing = False + if getattr(self, '_ccc', False) is False: + super(SSLConnection, self).close() + else: + pass + + def handle_read_event(self): + if self._ssl_accepting: + self._do_ssl_handshake() + elif self._ssl_closing: + self._do_ssl_shutdown() + else: + super(SSLConnection, self).handle_read_event() + + def handle_write_event(self): + if self._ssl_accepting: + self._do_ssl_handshake() + elif self._ssl_closing: + self._do_ssl_shutdown() + else: + super(SSLConnection, self).handle_write_event() + + def send(self, data): + try: + return super(SSLConnection, self).send(data) + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN, + ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return 0 + raise + + def recv(self, buffer_size): + try: + return super(SSLConnection, self).recv(buffer_size) + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return b'' + if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN): + self.handle_close() + return b'' + raise + + def handle_error(self): + raise Exception + + def close(self): + if (isinstance(self.socket, ssl.SSLSocket) and + self.socket._sslobj is not None): + self._do_ssl_shutdown() + else: + super(SSLConnection, self).close() + + + class DummyTLS_DTPHandler(SSLConnection, DummyDTPHandler): + """A DummyDTPHandler subclass supporting TLS/SSL.""" + + def __init__(self, conn, baseclass): + DummyDTPHandler.__init__(self, conn, baseclass) + if self.baseclass.secure_data_channel: + self.secure_connection() + + + class DummyTLS_FTPHandler(SSLConnection, DummyFTPHandler): + """A DummyFTPHandler subclass supporting TLS/SSL.""" + + dtp_handler = DummyTLS_DTPHandler + + def __init__(self, conn): + DummyFTPHandler.__init__(self, conn) + self.secure_data_channel = False + self._ccc = False + + def cmd_auth(self, line): + """Set up secure control channel.""" + self.push('234 AUTH TLS successful') + self.secure_connection() + + def cmd_ccc(self, line): + self.push('220 Reverting back to clear-text') + self._ccc = True + self._do_ssl_shutdown() + + def cmd_pbsz(self, line): + """Negotiate size of buffer for secure data transfer. + For TLS/SSL the only valid value for the parameter is '0'. + Any other value is accepted but ignored. + """ + self.push('200 PBSZ=0 successful.') + + def cmd_prot(self, line): + """Setup un/secure data channel.""" + arg = line.upper() + if arg == 'C': + self.push('200 Protection set to Clear') + self.secure_data_channel = False + elif arg == 'P': + self.push('200 Protection set to Private') + self.secure_data_channel = True + else: + self.push("502 Unrecognized PROT type (use C or P).") + + + class DummyTLS_FTPServer(DummyFTPServer): + handler = DummyTLS_FTPHandler + + +class TestFTPClass(TestCase): + + def setUp(self): + self.server = DummyFTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None + asyncore.close_all(ignore_all=True) + + def check_data(self, received, expected): + self.assertEqual(len(received), len(expected)) + self.assertEqual(received, expected) + + def test_getwelcome(self): + self.assertEqual(self.client.getwelcome(), '220 welcome') + + def test_sanitize(self): + self.assertEqual(self.client.sanitize('foo'), repr('foo')) + self.assertEqual(self.client.sanitize('pass 12345'), repr('pass *****')) + self.assertEqual(self.client.sanitize('PASS 12345'), repr('PASS *****')) + + def test_exceptions(self): + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\r\n0') + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\n0') + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\r0') + self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 400') + self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 499') + self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 500') + self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 599') + self.assertRaises(ftplib.error_proto, self.client.sendcmd, 'echo 999') + + def test_all_errors(self): + exceptions = (ftplib.error_reply, ftplib.error_temp, ftplib.error_perm, + ftplib.error_proto, ftplib.Error, OSError, + EOFError) + for x in exceptions: + try: + raise x('exception not included in all_errors set') + except ftplib.all_errors: + pass + + def test_set_pasv(self): + # passive mode is supposed to be enabled by default + self.assertTrue(self.client.passiveserver) + self.client.set_pasv(True) + self.assertTrue(self.client.passiveserver) + self.client.set_pasv(False) + self.assertFalse(self.client.passiveserver) + + def test_voidcmd(self): + self.client.voidcmd('echo 200') + self.client.voidcmd('echo 299') + self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') + self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') + + def test_login(self): + self.client.login() + + def test_acct(self): + self.client.acct('passwd') + + def test_rename(self): + self.client.rename('a', 'b') + self.server.handler_instance.next_response = '200' + self.assertRaises(ftplib.error_reply, self.client.rename, 'a', 'b') + + def test_delete(self): + self.client.delete('foo') + self.server.handler_instance.next_response = '199' + self.assertRaises(ftplib.error_reply, self.client.delete, 'foo') + + def test_size(self): + self.client.size('foo') + + def test_mkd(self): + dir = self.client.mkd('/foo') + self.assertEqual(dir, '/foo') + + def test_rmd(self): + self.client.rmd('foo') + + def test_cwd(self): + dir = self.client.cwd('/foo') + self.assertEqual(dir, '250 cwd ok') + + def test_pwd(self): + dir = self.client.pwd() + self.assertEqual(dir, 'pwd ok') + + def test_quit(self): + self.assertEqual(self.client.quit(), '221 quit ok') + # Ensure the connection gets closed; sock attribute should be None + self.assertEqual(self.client.sock, None) + + def test_abort(self): + self.client.abort() + + def test_retrbinary(self): + def callback(data): + received.append(data.decode('ascii')) + received = [] + self.client.retrbinary('retr', callback) + self.check_data(''.join(received), RETR_DATA) + + def test_retrbinary_rest(self): + def callback(data): + received.append(data.decode('ascii')) + for rest in (0, 10, 20): + received = [] + self.client.retrbinary('retr', callback, rest=rest) + self.check_data(''.join(received), RETR_DATA[rest:]) + + def test_retrlines(self): + received = [] + self.client.retrlines('retr', received.append) + self.check_data(''.join(received), RETR_DATA.replace('\r\n', '')) + + def test_storbinary(self): + f = io.BytesIO(RETR_DATA.encode('ascii')) + self.client.storbinary('stor', f) + self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + # test new callback arg + flag = [] + f.seek(0) + self.client.storbinary('stor', f, callback=lambda x: flag.append(None)) + self.assertTrue(flag) + + def test_storbinary_rest(self): + f = io.BytesIO(RETR_DATA.replace('\r\n', '\n').encode('ascii')) + for r in (30, '30'): + f.seek(0) + self.client.storbinary('stor', f, rest=r) + self.assertEqual(self.server.handler_instance.rest, str(r)) + + def test_storlines(self): + f = io.BytesIO(RETR_DATA.replace('\r\n', '\n').encode('ascii')) + self.client.storlines('stor', f) + self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + # test new callback arg + flag = [] + f.seek(0) + self.client.storlines('stor foo', f, callback=lambda x: flag.append(None)) + self.assertTrue(flag) + + f = io.StringIO(RETR_DATA.replace('\r\n', '\n')) + # storlines() expects a binary file, not a text file + with support.check_warnings(('', BytesWarning), quiet=True): + self.assertRaises(TypeError, self.client.storlines, 'stor foo', f) + + def test_nlst(self): + self.client.nlst() + self.assertEqual(self.client.nlst(), NLST_DATA.split('\r\n')[:-1]) + + def test_dir(self): + l = [] + self.client.dir(lambda x: l.append(x)) + self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) + + def test_mlsd(self): + list(self.client.mlsd()) + list(self.client.mlsd(path='/')) + list(self.client.mlsd(path='/', facts=['size', 'type'])) + + ls = list(self.client.mlsd()) + for name, facts in ls: + self.assertIsInstance(name, str) + self.assertIsInstance(facts, dict) + self.assertTrue(name) + self.assertIn('type', facts) + self.assertIn('perm', facts) + self.assertIn('unique', facts) + + def set_data(data): + self.server.handler_instance.next_data = data + + def test_entry(line, type=None, perm=None, unique=None, name=None): + type = 'type' if type is None else type + perm = 'perm' if perm is None else perm + unique = 'unique' if unique is None else unique + name = 'name' if name is None else name + set_data(line) + _name, facts = next(self.client.mlsd()) + self.assertEqual(_name, name) + self.assertEqual(facts['type'], type) + self.assertEqual(facts['perm'], perm) + self.assertEqual(facts['unique'], unique) + + # plain + test_entry('type=type;perm=perm;unique=unique; name\r\n') + # "=" in fact value + test_entry('type=ty=pe;perm=perm;unique=unique; name\r\n', type="ty=pe") + test_entry('type==type;perm=perm;unique=unique; name\r\n', type="=type") + test_entry('type=t=y=pe;perm=perm;unique=unique; name\r\n', type="t=y=pe") + test_entry('type=====;perm=perm;unique=unique; name\r\n', type="====") + # spaces in name + test_entry('type=type;perm=perm;unique=unique; na me\r\n', name="na me") + test_entry('type=type;perm=perm;unique=unique; name \r\n', name="name ") + test_entry('type=type;perm=perm;unique=unique; name\r\n', name=" name") + test_entry('type=type;perm=perm;unique=unique; n am e\r\n', name="n am e") + # ";" in name + test_entry('type=type;perm=perm;unique=unique; na;me\r\n', name="na;me") + test_entry('type=type;perm=perm;unique=unique; ;name\r\n', name=";name") + test_entry('type=type;perm=perm;unique=unique; ;name;\r\n', name=";name;") + test_entry('type=type;perm=perm;unique=unique; ;;;;\r\n', name=";;;;") + # case sensitiveness + set_data('Type=type;TyPe=perm;UNIQUE=unique; name\r\n') + _name, facts = next(self.client.mlsd()) + for x in facts: + self.assertTrue(x.islower()) + # no data (directory empty) + set_data('') + self.assertRaises(StopIteration, next, self.client.mlsd()) + set_data('') + for x in self.client.mlsd(): + self.fail("unexpected data %s" % x) + + def test_makeport(self): + with self.client.makeport(): + # IPv4 is in use, just make sure send_eprt has not been used + self.assertEqual(self.server.handler_instance.last_received_cmd, + 'port') + + def test_makepasv(self): + host, port = self.client.makepasv() + conn = socket.create_connection((host, port), timeout=TIMEOUT) + conn.close() + # IPv4 is in use, just make sure send_epsv has not been used + self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv') + + def test_with_statement(self): + self.client.quit() + + def is_client_connected(): + if self.client.sock is None: + return False + try: + self.client.sendcmd('noop') + except (OSError, EOFError): + return False + return True + + # base test + with ftplib.FTP(timeout=TIMEOUT) as self.client: + self.client.connect(self.server.host, self.server.port) + self.client.sendcmd('noop') + self.assertTrue(is_client_connected()) + self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit') + self.assertFalse(is_client_connected()) + + # QUIT sent inside the with block + with ftplib.FTP(timeout=TIMEOUT) as self.client: + self.client.connect(self.server.host, self.server.port) + self.client.sendcmd('noop') + self.client.quit() + self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit') + self.assertFalse(is_client_connected()) + + # force a wrong response code to be sent on QUIT: error_perm + # is expected and the connection is supposed to be closed + try: + with ftplib.FTP(timeout=TIMEOUT) as self.client: + self.client.connect(self.server.host, self.server.port) + self.client.sendcmd('noop') + self.server.handler_instance.next_response = '550 error on quit' + except ftplib.error_perm as err: + self.assertEqual(str(err), '550 error on quit') + else: + self.fail('Exception not raised') + # needed to give the threaded server some time to set the attribute + # which otherwise would still be == 'noop' + time.sleep(0.1) + self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit') + self.assertFalse(is_client_connected()) + + def test_source_address(self): + self.client.quit() + port = support.find_unused_port() + try: + self.client.connect(self.server.host, self.server.port, + source_address=(HOST, port)) + self.assertEqual(self.client.sock.getsockname()[1], port) + self.client.quit() + except OSError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + + def test_source_address_passive_connection(self): + port = support.find_unused_port() + self.client.source_address = (HOST, port) + try: + with self.client.transfercmd('list') as sock: + self.assertEqual(sock.getsockname()[1], port) + except OSError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + + def test_parse257(self): + self.assertEqual(ftplib.parse257('257 "/foo/bar"'), '/foo/bar') + self.assertEqual(ftplib.parse257('257 "/foo/bar" created'), '/foo/bar') + self.assertEqual(ftplib.parse257('257 ""'), '') + self.assertEqual(ftplib.parse257('257 "" created'), '') + self.assertRaises(ftplib.error_reply, ftplib.parse257, '250 "/foo/bar"') + # The 257 response is supposed to include the directory + # name and in case it contains embedded double-quotes + # they must be doubled (see RFC-959, chapter 7, appendix 2). + self.assertEqual(ftplib.parse257('257 "/foo/b""ar"'), '/foo/b"ar') + self.assertEqual(ftplib.parse257('257 "/foo/b""ar" created'), '/foo/b"ar') + + def test_line_too_long(self): + self.assertRaises(ftplib.Error, self.client.sendcmd, + 'x' * self.client.maxline * 2) + + def test_retrlines_too_long(self): + self.client.sendcmd('SETLONGRETR %d' % (self.client.maxline * 2)) + received = [] + self.assertRaises(ftplib.Error, + self.client.retrlines, 'retr', received.append) + + def test_storlines_too_long(self): + f = io.BytesIO(b'x' * self.client.maxline * 2) + self.assertRaises(ftplib.Error, self.client.storlines, 'stor', f) + + +@skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") +class TestIPv6Environment(TestCase): + + def setUp(self): + self.server = DummyFTPServer((HOSTv6, 0), af=socket.AF_INET6) + self.server.start() + self.client = ftplib.FTP(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None + asyncore.close_all(ignore_all=True) + + def test_af(self): + self.assertEqual(self.client.af, socket.AF_INET6) + + def test_makeport(self): + with self.client.makeport(): + self.assertEqual(self.server.handler_instance.last_received_cmd, + 'eprt') + + def test_makepasv(self): + host, port = self.client.makepasv() + conn = socket.create_connection((host, port), timeout=TIMEOUT) + conn.close() + self.assertEqual(self.server.handler_instance.last_received_cmd, 'epsv') + + def test_transfer(self): + def retr(): + def callback(data): + received.append(data.decode('ascii')) + received = [] + self.client.retrbinary('retr', callback) + self.assertEqual(len(''.join(received)), len(RETR_DATA)) + self.assertEqual(''.join(received), RETR_DATA) + self.client.set_pasv(True) + retr() + self.client.set_pasv(False) + retr() + + +@skipUnless(ssl, "SSL not available") +class TestTLS_FTPClassMixin(TestFTPClass): + """Repeat TestFTPClass tests starting the TLS layer for both control + and data connections first. + """ + + def setUp(self): + self.server = DummyTLS_FTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP_TLS(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + # enable TLS + self.client.auth() + self.client.prot_p() + + +@skipUnless(ssl, "SSL not available") +class TestTLS_FTPClass(TestCase): + """Specific TLS_FTP class tests.""" + + def setUp(self): + self.server = DummyTLS_FTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP_TLS(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None + asyncore.close_all(ignore_all=True) + + def test_control_connection(self): + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.auth() + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + + def test_data_connection(self): + # clear text + with self.client.transfercmd('list') as sock: + self.assertNotIsInstance(sock, ssl.SSLSocket) + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + # secured, after PROT P + self.client.prot_p() + with self.client.transfercmd('list') as sock: + self.assertIsInstance(sock, ssl.SSLSocket) + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + # PROT C is issued, the connection must be in cleartext again + self.client.prot_c() + with self.client.transfercmd('list') as sock: + self.assertNotIsInstance(sock, ssl.SSLSocket) + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + def test_login(self): + # login() is supposed to implicitly secure the control connection + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.login() + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + # make sure that AUTH TLS doesn't get issued again + self.client.login() + + def test_auth_issued_twice(self): + self.client.auth() + self.assertRaises(ValueError, self.client.auth) + + def test_auth_ssl(self): + try: + self.client.ssl_version = ssl.PROTOCOL_SSLv23 + self.client.auth() + self.assertRaises(ValueError, self.client.auth) + finally: + self.client.ssl_version = ssl.PROTOCOL_TLSv1 + + def test_context(self): + self.client.quit() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE, + context=ctx) + self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, + context=ctx) + self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, + keyfile=CERTFILE, context=ctx) + + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.auth() + self.assertIs(self.client.sock.context, ctx) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + + self.client.prot_p() + with self.client.transfercmd('list') as sock: + self.assertIs(sock.context, ctx) + self.assertIsInstance(sock, ssl.SSLSocket) + + def test_ccc(self): + self.assertRaises(ValueError, self.client.ccc) + self.client.login(secure=True) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + self.client.ccc() + self.assertRaises(ValueError, self.client.sock.unwrap) + + def test_check_hostname(self): + self.client.quit() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.check_hostname = True + ctx.load_verify_locations(CAFILE) + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) + + # 127.0.0.1 doesn't match SAN + self.client.connect(self.server.host, self.server.port) + with self.assertRaises(ssl.CertificateError): + self.client.auth() + # exception quits connection + + self.client.connect(self.server.host, self.server.port) + self.client.prot_p() + with self.assertRaises(ssl.CertificateError): + with self.client.transfercmd("list") as sock: + pass + self.client.quit() + + self.client.connect("localhost", self.server.port) + self.client.auth() + self.client.quit() + + self.client.connect("localhost", self.server.port) + self.client.prot_p() + with self.client.transfercmd("list") as sock: + pass + + +class TestTimeouts(TestCase): + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(20) + self.port = support.bind_port(self.sock) + self.server_thread = threading.Thread(target=self.server) + self.server_thread.start() + # Wait for the server to be ready. + self.evt.wait() + self.evt.clear() + self.old_port = ftplib.FTP.port + ftplib.FTP.port = self.port + + def tearDown(self): + ftplib.FTP.port = self.old_port + self.server_thread.join() + # Explicitly clear the attribute to prevent dangling thread + self.server_thread = None + + def server(self): + # This method sets the evt 3 times: + # 1) when the connection is ready to be accepted. + # 2) when it is safe for the caller to close the connection + # 3) when we have closed the socket + self.sock.listen() + # (1) Signal the caller that we are ready to accept the connection. + self.evt.set() + try: + conn, addr = self.sock.accept() + except socket.timeout: + pass + else: + conn.sendall(b"1 Hola mundo\n") + conn.shutdown(socket.SHUT_WR) + # (2) Signal the caller that it is safe to close the socket. + self.evt.set() + conn.close() + finally: + self.sock.close() + + def testTimeoutDefault(self): + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + ftp = ftplib.FTP(HOST) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutNone(self): + # no timeout -- do not use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + ftp = ftplib.FTP(HOST, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(ftp.sock.gettimeout()) + self.evt.wait() + ftp.close() + + def testTimeoutValue(self): + # a value + ftp = ftplib.FTP(HOST, timeout=30) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutConnect(self): + ftp = ftplib.FTP() + ftp.connect(HOST, timeout=30) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutDifferentOrder(self): + ftp = ftplib.FTP(timeout=30) + ftp.connect(HOST) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutDirectAccess(self): + ftp = ftplib.FTP() + ftp.timeout = 30 + ftp.connect(HOST) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + +class MiscTestCase(TestCase): + def test__all__(self): + blacklist = {'MSG_OOB', 'FTP_PORT', 'MAXLINE', 'CRLF', 'B_CRLF', + 'Error', 'parse150', 'parse227', 'parse229', 'parse257', + 'print_line', 'ftpcp', 'test'} + support.check__all__(self, ftplib, blacklist=blacklist) + + +def test_main(): + tests = [TestFTPClass, TestTimeouts, + TestIPv6Environment, + TestTLS_FTPClassMixin, TestTLS_FTPClass, + MiscTestCase] + + thread_info = support.threading_setup() + try: + support.run_unittest(*tests) + finally: + support.threading_cleanup(*thread_info) + + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/3.6/test_httplib.py b/src/greentest/3.6/test_httplib.py new file mode 100644 index 0000000..68f6946 --- /dev/null +++ b/src/greentest/3.6/test_httplib.py @@ -0,0 +1,1903 @@ +import errno +from http import client +import io +import itertools +import os +import array +import socket + +import unittest +TestCase = unittest.TestCase + +from test import support + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') +# Self-signed cert file for self-signed.pythontest.net +CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') + +# constants for testing chunked encoding +chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd! \r\n' + '8\r\n' + 'and now \r\n' + '22\r\n' + 'for something completely different\r\n' +) +chunked_expected = b'hello world! and now for something completely different' +chunk_extension = ";foo=bar" +last_chunk = "0\r\n" +last_chunk_extended = "0" + chunk_extension + "\r\n" +trailers = "X-Dummy: foo\r\nX-Dumm2: bar\r\n" +chunked_end = "\r\n" + +HOST = support.HOST + +class FakeSocket: + def __init__(self, text, fileclass=io.BytesIO, host=None, port=None): + if isinstance(text, str): + text = text.encode("ascii") + self.text = text + self.fileclass = fileclass + self.data = b'' + self.sendall_calls = 0 + self.file_closed = False + self.host = host + self.port = port + + def sendall(self, data): + self.sendall_calls += 1 + self.data += data + + def makefile(self, mode, bufsize=None): + if mode != 'r' and mode != 'rb': + raise client.UnimplementedFileMode() + # keep the file around so we can check how much was read from it + self.file = self.fileclass(self.text) + self.file.close = self.file_close #nerf close () + return self.file + + def file_close(self): + self.file_closed = True + + def close(self): + pass + + def setsockopt(self, level, optname, value): + pass + +class EPipeSocket(FakeSocket): + + def __init__(self, text, pipe_trigger): + # When sendall() is called with pipe_trigger, raise EPIPE. + FakeSocket.__init__(self, text) + self.pipe_trigger = pipe_trigger + + def sendall(self, data): + if self.pipe_trigger in data: + raise OSError(errno.EPIPE, "gotcha") + self.data += data + + def close(self): + pass + +class NoEOFBytesIO(io.BytesIO): + """Like BytesIO, but raises AssertionError on EOF. + + This is used below to test that http.client doesn't try to read + more from the underlying file than it should. + """ + def read(self, n=-1): + data = io.BytesIO.read(self, n) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + + def readline(self, length=None): + data = io.BytesIO.readline(self, length) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + +class FakeSocketHTTPConnection(client.HTTPConnection): + """HTTPConnection subclass using FakeSocket; counts connect() calls""" + + def __init__(self, *args): + self.connections = 0 + super().__init__('example.com') + self.fake_socket_args = args + self._create_connection = self.create_connection + + def connect(self): + """Count the number of times connect() is invoked""" + self.connections += 1 + return super().connect() + + def create_connection(self, *pos, **kw): + return FakeSocket(*self.fake_socket_args) + +class HeaderTests(TestCase): + def test_auto_headers(self): + # Some headers are added automatically, but should not be added by + # .request() if they are explicitly set. + + class HeaderCountingBuffer(list): + def __init__(self): + self.count = {} + def append(self, item): + kv = item.split(b':') + if len(kv) > 1: + # item is a 'Key: Value' header string + lcKey = kv[0].decode('ascii').lower() + self.count.setdefault(lcKey, 0) + self.count[lcKey] += 1 + list.append(self, item) + + for explicit_header in True, False: + for header in 'Content-length', 'Host', 'Accept-encoding': + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('blahblahblah') + conn._buffer = HeaderCountingBuffer() + + body = 'spamspamspam' + headers = {} + if explicit_header: + headers[header] = str(len(body)) + conn.request('POST', '/', body, headers) + self.assertEqual(conn._buffer.count[header.lower()], 1) + + def test_content_length_0(self): + + class ContentLengthChecker(list): + def __init__(self): + list.__init__(self) + self.content_length = None + def append(self, item): + kv = item.split(b':', 1) + if len(kv) > 1 and kv[0].lower() == b'content-length': + self.content_length = kv[1].strip() + list.append(self, item) + + # Here, we're testing that methods expecting a body get a + # content-length set to zero if the body is empty (either None or '') + bodies = (None, '') + methods_with_body = ('PUT', 'POST', 'PATCH') + for method, body in itertools.product(methods_with_body, bodies): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', body) + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # For these methods, we make sure that content-length is not set when + # the body is None because it might cause unexpected behaviour on the + # server. + methods_without_body = ( + 'GET', 'CONNECT', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', + ) + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', None) + self.assertEqual( + conn._buffer.content_length, None, + 'Header Content-Length set for empty body on {}'.format(method) + ) + + # If the body is set to '', that's considered to be "present but + # empty" rather than "missing", so content length would be set, even + # for methods that don't expect a body. + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', '') + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # If the body is set, make sure Content-Length is set. + for method in itertools.chain(methods_without_body, methods_with_body): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', ' ') + self.assertEqual( + conn._buffer.content_length, b'1', + 'Header Content-Length incorrect on {}'.format(method) + ) + + def test_putheader(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.putrequest('GET','/') + conn.putheader('Content-length', 42) + self.assertIn(b'Content-length: 42', conn._buffer) + + conn.putheader('Foo', ' bar ') + self.assertIn(b'Foo: bar ', conn._buffer) + conn.putheader('Bar', '\tbaz\t') + self.assertIn(b'Bar: \tbaz\t', conn._buffer) + conn.putheader('Authorization', 'Bearer mytoken') + self.assertIn(b'Authorization: Bearer mytoken', conn._buffer) + conn.putheader('IterHeader', 'IterA', 'IterB') + self.assertIn(b'IterHeader: IterA\r\n\tIterB', conn._buffer) + conn.putheader('LatinHeader', b'\xFF') + self.assertIn(b'LatinHeader: \xFF', conn._buffer) + conn.putheader('Utf8Header', b'\xc3\x80') + self.assertIn(b'Utf8Header: \xc3\x80', conn._buffer) + conn.putheader('C1-Control', b'next\x85line') + self.assertIn(b'C1-Control: next\x85line', conn._buffer) + conn.putheader('Embedded-Fold-Space', 'is\r\n allowed') + self.assertIn(b'Embedded-Fold-Space: is\r\n allowed', conn._buffer) + conn.putheader('Embedded-Fold-Tab', 'is\r\n\tallowed') + self.assertIn(b'Embedded-Fold-Tab: is\r\n\tallowed', conn._buffer) + conn.putheader('Key Space', 'value') + self.assertIn(b'Key Space: value', conn._buffer) + conn.putheader('KeySpace ', 'value') + self.assertIn(b'KeySpace : value', conn._buffer) + conn.putheader(b'Nonbreak\xa0Space', 'value') + self.assertIn(b'Nonbreak\xa0Space: value', conn._buffer) + conn.putheader(b'\xa0NonbreakSpace', 'value') + self.assertIn(b'\xa0NonbreakSpace: value', conn._buffer) + + def test_ipv6host_header(self): + # Default host header on IPv6 transaction should be wrapped by [] if + # it is an IPv6 address + expected = b'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001::]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + expected = b'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001:102A::]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + def test_malformed_headers_coped_with(self): + # Issue 19996 + body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + + self.assertEqual(resp.getheader('First'), 'val') + self.assertEqual(resp.getheader('Second'), 'val') + + def test_parse_all_octets(self): + # Ensure no valid header field octet breaks the parser + body = ( + b'HTTP/1.1 200 OK\r\n' + b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters + b'VCHAR: ' + bytes(range(0x21, 0x7E + 1)) + b'\r\n' + b'obs-text: ' + bytes(range(0x80, 0xFF + 1)) + b'\r\n' + b'obs-fold: text\r\n' + b' folded with space\r\n' + b'\tfolded with tab\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.getheader('Content-Length'), '0') + self.assertEqual(resp.msg['Content-Length'], '0') + self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value') + self.assertEqual(resp.msg["!#$%&'*+-.^_`|~"], 'value') + vchar = ''.join(map(chr, range(0x21, 0x7E + 1))) + self.assertEqual(resp.getheader('VCHAR'), vchar) + self.assertEqual(resp.msg['VCHAR'], vchar) + self.assertIsNotNone(resp.getheader('obs-text')) + self.assertIn('obs-text', resp.msg) + for folded in (resp.getheader('obs-fold'), resp.msg['obs-fold']): + self.assertTrue(folded.startswith('text')) + self.assertIn(' folded with space', folded) + self.assertTrue(folded.endswith('folded with tab')) + + def test_invalid_headers(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/') + + # http://tools.ietf.org/html/rfc7230#section-3.2.4, whitespace is no + # longer allowed in header names + cases = ( + (b'Invalid\r\nName', b'ValidValue'), + (b'Invalid\rName', b'ValidValue'), + (b'Invalid\nName', b'ValidValue'), + (b'\r\nInvalidName', b'ValidValue'), + (b'\rInvalidName', b'ValidValue'), + (b'\nInvalidName', b'ValidValue'), + (b' InvalidName', b'ValidValue'), + (b'\tInvalidName', b'ValidValue'), + (b'Invalid:Name', b'ValidValue'), + (b':InvalidName', b'ValidValue'), + (b'ValidName', b'Invalid\r\nValue'), + (b'ValidName', b'Invalid\rValue'), + (b'ValidName', b'Invalid\nValue'), + (b'ValidName', b'InvalidValue\r\n'), + (b'ValidName', b'InvalidValue\r'), + (b'ValidName', b'InvalidValue\n'), + ) + for name, value in cases: + with self.subTest((name, value)): + with self.assertRaisesRegex(ValueError, 'Invalid header'): + conn.putheader(name, value) + + +class TransferEncodingTest(TestCase): + expected_body = b"It's just a flesh wound" + + def test_endheaders_chunked(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.putrequest('POST', '/') + conn.endheaders(self._make_body(), encode_chunked=True) + + _, _, body = self._parse_request(conn.sock.data) + body = self._parse_chunked(body) + self.assertEqual(body, self.expected_body) + + def test_explicit_headers(self): + # explicit chunked + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + # this shouldn't actually be automatically chunk-encoded because the + # calling code has explicitly stated that it's taking care of it + conn.request( + 'POST', '/', self._make_body(), {'Transfer-Encoding': 'chunked'}) + + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers.keys()]) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertEqual(body, self.expected_body) + + # explicit chunked, string body + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request( + 'POST', '/', self.expected_body.decode('latin-1'), + {'Transfer-Encoding': 'chunked'}) + + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers.keys()]) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertEqual(body, self.expected_body) + + # User-specified TE, but request() does the chunk encoding + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request('POST', '/', + headers={'Transfer-Encoding': 'gzip, chunked'}, + encode_chunked=True, + body=self._make_body()) + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers]) + self.assertEqual(headers['Transfer-Encoding'], 'gzip, chunked') + self.assertEqual(self._parse_chunked(body), self.expected_body) + + def test_request(self): + for empty_lines in (False, True,): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request( + 'POST', '/', self._make_body(empty_lines=empty_lines)) + + _, headers, body = self._parse_request(conn.sock.data) + body = self._parse_chunked(body) + self.assertEqual(body, self.expected_body) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + + # Content-Length and Transfer-Encoding SHOULD not be sent in the + # same request + self.assertNotIn('content-length', [k.lower() for k in headers]) + + def test_empty_body(self): + # Zero-length iterable should be treated like any other iterable + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request('POST', '/', ()) + _, headers, body = self._parse_request(conn.sock.data) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertNotIn('content-length', [k.lower() for k in headers]) + self.assertEqual(body, b"0\r\n\r\n") + + def _make_body(self, empty_lines=False): + lines = self.expected_body.split(b' ') + for idx, line in enumerate(lines): + # for testing handling empty lines + if empty_lines and idx % 2: + yield b'' + if idx < len(lines) - 1: + yield line + b' ' + else: + yield line + + def _parse_request(self, data): + lines = data.split(b'\r\n') + request = lines[0] + headers = {} + n = 1 + while n < len(lines) and len(lines[n]) > 0: + key, val = lines[n].split(b':') + key = key.decode('latin-1').strip() + headers[key] = val.decode('latin-1').strip() + n += 1 + + return request, headers, b'\r\n'.join(lines[n + 1:]) + + def _parse_chunked(self, data): + body = [] + trailers = {} + n = 0 + lines = data.split(b'\r\n') + # parse body + while True: + size, chunk = lines[n:n+2] + size = int(size, 16) + + if size == 0: + n += 1 + break + + self.assertEqual(size, len(chunk)) + body.append(chunk) + + n += 2 + # we /should/ hit the end chunk, but check against the size of + # lines so we're not stuck in an infinite loop should we get + # malformed data + if n > len(lines): + break + + return b''.join(body) + + +class BasicTest(TestCase): + def test_status_lines(self): + # Test HTTP status lines + + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(0), b'') # Issue #20007 + self.assertFalse(resp.isclosed()) + self.assertFalse(resp.closed) + self.assertEqual(resp.read(), b"Text") + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + self.assertRaises(client.BadStatusLine, resp.begin) + + def test_bad_status_repr(self): + exc = client.BadStatusLine('') + self.assertEqual(repr(exc), '''BadStatusLine("\'\'",)''') + + def test_partial_reads(self): + # if we have Content-Length, HTTPResponse knows when to close itself, + # the same behaviour as when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_mixed_reads(self): + # readline() should update the remaining length, so that read() knows + # how much data is left and does not raise IncompleteRead + body = "HTTP/1.1 200 Ok\r\nContent-Length: 13\r\n\r\nText\r\nAnother" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.readline(), b'Text\r\n') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(), b'Another') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos(self): + # if we have Content-Length, HTTPResponse knows when to close itself, + # the same behaviour as when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_reads_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + + def test_partial_reads_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + + def test_partial_readintos_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:80", "www.python.org", 80), + ("www.python.org:", "www.python.org", 80), + ("www.python.org", "www.python.org", 80), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", 80)): + c = client.HTTPConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + def test_response_headers(self): + # test response with multiple message headers with the same field name. + text = ('HTTP/1.1 200 OK\r\n' + 'Set-Cookie: Customer="WILE_E_COYOTE"; ' + 'Version="1"; Path="/acme"\r\n' + 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' + ' Path="/acme"\r\n' + '\r\n' + 'No body\r\n') + hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"' + ', ' + 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"') + s = FakeSocket(text) + r = client.HTTPResponse(s) + r.begin() + cookies = r.getheader("Set-Cookie") + self.assertEqual(cookies, hdr) + + def test_read_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + if resp.read(): + self.fail("Did not expect response from HEAD request") + + def test_readinto_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + if resp.readinto(b) != 0: + self.fail("Did not expect response from HEAD request") + self.assertEqual(bytes(b), b'\x00'*5) + + def test_too_many_headers(self): + headers = '\r\n'.join('Header%d: foo' % i + for i in range(client._MAXHEADERS + 1)) + '\r\n' + text = ('HTTP/1.1 200 OK\r\n' + headers) + s = FakeSocket(text) + r = client.HTTPResponse(s) + self.assertRaisesRegex(client.HTTPException, + r"got more than \d+ headers", r.begin) + + def test_send_file(self): + expected = (b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' + b'Accept-Encoding: identity\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n') + + with open(__file__, 'rb') as body: + conn = client.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertTrue(sock.data.startswith(expected), '%r != %r' % + (sock.data[:len(expected)], expected)) + + def test_send(self): + expected = b'this is a test this is only a test' + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + conn.send(expected) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(array.array('b', expected)) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(io.BytesIO(expected)) + self.assertEqual(expected, sock.data) + + def test_send_updating_file(self): + def data(): + yield 'data' + yield None + yield 'data_two' + + class UpdatingFile(io.TextIOBase): + mode = 'r' + d = data() + def read(self, blocksize=-1): + return next(self.d) + + expected = b'data' + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.send(UpdatingFile()) + self.assertEqual(sock.data, expected) + + + def test_send_iter(self): + expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \ + b'\r\nonetwothree' + + def body(): + yield b"one" + yield b"two" + yield b"three" + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.request('GET', '/foo', body(), {'Content-Length': '11'}) + self.assertEqual(sock.data, expected) + + def test_send_type_error(self): + # See: Issue #12676 + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + with self.assertRaises(TypeError): + conn.request('POST', 'test', conn) + + def test_chunked(self): + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(n) + resp.read(n) + resp.read(), expected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_readinto_chunked(self): + + expected = chunked_expected + nexpected = len(expected) + b = bytearray(128) + + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + n = resp.readinto(b) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(n, nexpected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + m = memoryview(b) + i = resp.readinto(m[0:n]) + i += resp.readinto(m[i:n + i]) + i += resp.readinto(m[i:]) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(i, nexpected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + n = resp.readinto(b) + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_readinto_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertEqual(bytes(b), b'\x00'*5) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_negative_content_length(self): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), b'Hello\r\n') + self.assertTrue(resp.isclosed()) + + def test_incomplete_read(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, b'Hello\r\n') + self.assertEqual(repr(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertEqual(str(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertTrue(resp.isclosed()) + else: + self.fail('IncompleteRead expected') + + def test_epipe(self): + sock = EPipeSocket( + "HTTP/1.0 401 Authorization Required\r\n" + "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"example\"\r\n", + b"Content-Length") + conn = client.HTTPConnection("example.com") + conn.sock = sock + self.assertRaises(OSError, + lambda: conn.request("PUT", "/url", "body")) + resp = conn.getresponse() + self.assertEqual(401, resp.status) + self.assertEqual("Basic realm=\"example\"", + resp.getheader("www-authenticate")) + + # Test lines overflowing the max line size (_MAXLINE in http.client) + + def test_overflowing_status_line(self): + body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n" + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises((client.LineTooLong, client.BadStatusLine), resp.begin) + + def test_overflowing_header_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises(client.LineTooLong, resp.begin) + + def test_overflowing_chunked_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + + '0' * 65536 + 'a\r\n' + 'hello world\r\n' + '0\r\n' + '\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertRaises(client.LineTooLong, resp.read) + + def test_early_eof(self): + # Test httpresponse with no \r\n termination, + body = "HTTP/1.1 200 Ok" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_error_leak(self): + # Test that the socket is not leaked if getresponse() fails + conn = client.HTTPConnection('example.com') + response = None + class Response(client.HTTPResponse): + def __init__(self, *pos, **kw): + nonlocal response + response = self # Avoid garbage collector closing the socket + client.HTTPResponse.__init__(self, *pos, **kw) + conn.response_class = Response + conn.sock = FakeSocket('Invalid status line') + conn.request('GET', '/') + self.assertRaises(client.BadStatusLine, conn.getresponse) + self.assertTrue(response.closed) + self.assertTrue(conn.sock.file_closed) + + def test_chunked_extension(self): + extra = '3;foo=bar\r\n' + 'abc\r\n' + expected = chunked_expected + b'abc' + + sock = FakeSocket(chunked_start + extra + last_chunk_extended + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + def test_chunked_missing_end(self): + """some servers may serve up a short chunked encoding stream""" + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk) #no terminating crlf + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + def test_chunked_trailers(self): + """See that trailers are read and ignored""" + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # we should have reached the end of the file + self.assertEqual(sock.file.read(), b"") #we read to the end + resp.close() + + def test_chunked_sync(self): + """Check that we don't read past the end of the chunked-encoding stream""" + expected = chunked_expected + extradata = "extradata" + sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata.encode("ascii")) #we read to the end + resp.close() + + def test_content_length_sync(self): + """Check that we don't read past the end of the Content-Length stream""" + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_readlines_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.readlines(2000), [expected]) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_read1_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read1(2000), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_readline_bound_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.readline(10), expected) + self.assertEqual(resp.readline(10), b"") + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_read1_bound_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 30\r\n\r\n' + expected*3 + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read1(20), expected*2) + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_response_fileno(self): + # Make sure fd returned by fileno is valid. + threading = support.import_module("threading") + + serv = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + self.addCleanup(serv.close) + serv.bind((HOST, 0)) + serv.listen() + + result = None + def run_server(): + [conn, address] = serv.accept() + with conn, conn.makefile("rb") as reader: + # Read the request header until a blank line + while True: + line = reader.readline() + if not line.rstrip(b"\r\n"): + break + conn.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n") + nonlocal result + result = reader.read() + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, float(1)) + conn = client.HTTPConnection(*serv.getsockname()) + conn.request("CONNECT", "dummy:1234") + response = conn.getresponse() + try: + self.assertEqual(response.status, client.OK) + s = socket.socket(fileno=response.fileno()) + try: + s.sendall(b"proxied data\n") + finally: + s.detach() + finally: + response.close() + conn.close() + thread.join() + self.assertEqual(result, b"proxied data\n") + +class ExtendedReadTest(TestCase): + """ + Test peek(), read1(), readline() + """ + lines = ( + 'HTTP/1.1 200 OK\r\n' + '\r\n' + 'hello world!\n' + 'and now \n' + 'for something completely different\n' + 'foo' + ) + lines_expected = lines[lines.find('hello'):].encode("ascii") + lines_chunked = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd!\n\r\n' + '9\r\n' + 'and now \n\r\n' + '23\r\n' + 'for something completely different\n\r\n' + '3\r\n' + 'foo\r\n' + '0\r\n' # terminating chunk + '\r\n' # end of trailers + ) + + def setUp(self): + sock = FakeSocket(self.lines) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + resp.fp = io.BufferedReader(resp.fp) + self.resp = resp + + + + def test_peek(self): + resp = self.resp + # patch up the buffered peek so that it returns not too much stuff + oldpeek = resp.fp.peek + def mypeek(n=-1): + p = oldpeek(n) + if n >= 0: + return p[:n] + return p[:10] + resp.fp.peek = mypeek + + all = [] + while True: + # try a short peek + p = resp.peek(3) + if p: + self.assertGreater(len(p), 0) + # then unbounded peek + p2 = resp.peek() + self.assertGreaterEqual(len(p2), len(p)) + self.assertTrue(p2.startswith(p)) + next = resp.read(len(p2)) + self.assertEqual(next, p2) + else: + next = resp.read() + self.assertFalse(next) + all.append(next) + if not next: + break + self.assertEqual(b"".join(all), self.lines_expected) + + def test_readline(self): + resp = self.resp + self._verify_readline(self.resp.readline, self.lines_expected) + + def _verify_readline(self, readline, expected): + all = [] + while True: + # short readlines + line = readline(5) + if line and line != b"foo": + if len(line) < 5: + self.assertTrue(line.endswith(b"\n")) + all.append(line) + if not line: + break + self.assertEqual(b"".join(all), expected) + + def test_read1(self): + resp = self.resp + def r(): + res = resp.read1(4) + self.assertLessEqual(len(res), 4) + return res + readliner = Readliner(r) + self._verify_readline(readliner.readline, self.lines_expected) + + def test_read1_unbounded(self): + resp = self.resp + all = [] + while True: + data = resp.read1() + if not data: + break + all.append(data) + self.assertEqual(b"".join(all), self.lines_expected) + + def test_read1_bounded(self): + resp = self.resp + all = [] + while True: + data = resp.read1(10) + if not data: + break + self.assertLessEqual(len(data), 10) + all.append(data) + self.assertEqual(b"".join(all), self.lines_expected) + + def test_read1_0(self): + self.assertEqual(self.resp.read1(0), b"") + + def test_peek_0(self): + p = self.resp.peek(0) + self.assertLessEqual(0, len(p)) + +class ExtendedReadTestChunked(ExtendedReadTest): + """ + Test peek(), read1(), readline() in chunked mode + """ + lines = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd!\n\r\n' + '9\r\n' + 'and now \n\r\n' + '23\r\n' + 'for something completely different\n\r\n' + '3\r\n' + 'foo\r\n' + '0\r\n' # terminating chunk + '\r\n' # end of trailers + ) + + +class Readliner: + """ + a simple readline class that uses an arbitrary read function and buffering + """ + def __init__(self, readfunc): + self.readfunc = readfunc + self.remainder = b"" + + def readline(self, limit): + data = [] + datalen = 0 + read = self.remainder + try: + while True: + idx = read.find(b'\n') + if idx != -1: + break + if datalen + len(read) >= limit: + idx = limit - datalen - 1 + # read more data + data.append(read) + read = self.readfunc() + if not read: + idx = 0 #eof condition + break + idx += 1 + data.append(read[:idx]) + self.remainder = read[idx:] + return b"".join(data) + except: + self.remainder = b"".join(data) + raise + + +class OfflineTest(TestCase): + def test_all(self): + # Documented objects defined in the module should be in __all__ + expected = {"responses"} # White-list documented dict() object + # HTTPMessage, parse_headers(), and the HTTP status code constants are + # intentionally omitted for simplicity + blacklist = {"HTTPMessage", "parse_headers"} + for name in dir(client): + if name.startswith("_") or name in blacklist: + continue + module_object = getattr(client, name) + if getattr(module_object, "__module__", None) == "http.client": + expected.add(name) + self.assertCountEqual(client.__all__, expected) + + def test_responses(self): + self.assertEqual(client.responses[client.NOT_FOUND], "Not Found") + + def test_client_constants(self): + # Make sure we don't break backward compatibility with 3.4 + expected = [ + 'CONTINUE', + 'SWITCHING_PROTOCOLS', + 'PROCESSING', + 'OK', + 'CREATED', + 'ACCEPTED', + 'NON_AUTHORITATIVE_INFORMATION', + 'NO_CONTENT', + 'RESET_CONTENT', + 'PARTIAL_CONTENT', + 'MULTI_STATUS', + 'IM_USED', + 'MULTIPLE_CHOICES', + 'MOVED_PERMANENTLY', + 'FOUND', + 'SEE_OTHER', + 'NOT_MODIFIED', + 'USE_PROXY', + 'TEMPORARY_REDIRECT', + 'BAD_REQUEST', + 'UNAUTHORIZED', + 'PAYMENT_REQUIRED', + 'FORBIDDEN', + 'NOT_FOUND', + 'METHOD_NOT_ALLOWED', + 'NOT_ACCEPTABLE', + 'PROXY_AUTHENTICATION_REQUIRED', + 'REQUEST_TIMEOUT', + 'CONFLICT', + 'GONE', + 'LENGTH_REQUIRED', + 'PRECONDITION_FAILED', + 'REQUEST_ENTITY_TOO_LARGE', + 'REQUEST_URI_TOO_LONG', + 'UNSUPPORTED_MEDIA_TYPE', + 'REQUESTED_RANGE_NOT_SATISFIABLE', + 'EXPECTATION_FAILED', + 'UNPROCESSABLE_ENTITY', + 'LOCKED', + 'FAILED_DEPENDENCY', + 'UPGRADE_REQUIRED', + 'PRECONDITION_REQUIRED', + 'TOO_MANY_REQUESTS', + 'REQUEST_HEADER_FIELDS_TOO_LARGE', + 'INTERNAL_SERVER_ERROR', + 'NOT_IMPLEMENTED', + 'BAD_GATEWAY', + 'SERVICE_UNAVAILABLE', + 'GATEWAY_TIMEOUT', + 'HTTP_VERSION_NOT_SUPPORTED', + 'INSUFFICIENT_STORAGE', + 'NOT_EXTENDED', + 'NETWORK_AUTHENTICATION_REQUIRED', + ] + for const in expected: + with self.subTest(constant=const): + self.assertTrue(hasattr(client, const)) + + +class SourceAddressTest(TestCase): + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.source_port = support.find_unused_port() + self.serv.listen() + self.conn = None + + def tearDown(self): + if self.conn: + self.conn.close() + self.conn = None + self.serv.close() + self.serv = None + + def testHTTPConnectionSourceAddress(self): + self.conn = client.HTTPConnection(HOST, self.port, + source_address=('', self.source_port)) + self.conn.connect() + self.assertEqual(self.conn.sock.getsockname()[1], self.source_port) + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not defined') + def testHTTPSConnectionSourceAddress(self): + self.conn = client.HTTPSConnection(HOST, self.port, + source_address=('', self.source_port)) + # We don't test anything here other than the constructor not barfing as + # this code doesn't deal with setting up an active running SSL server + # for an ssl_wrapped connect() to actually return from. + + +class TimeoutTest(TestCase): + PORT = None + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + TimeoutTest.PORT = support.bind_port(self.serv) + self.serv.listen() + + def tearDown(self): + self.serv.close() + self.serv = None + + def testTimeoutAttribute(self): + # This will prove that the timeout gets through HTTPConnection + # and into the socket. + + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + # no timeout -- do not use global socket default + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, + timeout=None) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), None) + httpConn.close() + + # a value + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30) + httpConn.connect() + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + +class PersistenceTest(TestCase): + + def test_reuse_reconnect(self): + # Should reuse or reconnect depending on header from server + tests = ( + ('1.0', '', False), + ('1.0', 'Connection: keep-alive\r\n', True), + ('1.1', '', True), + ('1.1', 'Connection: close\r\n', False), + ('1.0', 'Connection: keep-ALIVE\r\n', True), + ('1.1', 'Connection: cloSE\r\n', False), + ) + for version, header, reuse in tests: + with self.subTest(version=version, header=header): + msg = ( + 'HTTP/{} 200 OK\r\n' + '{}' + 'Content-Length: 12\r\n' + '\r\n' + 'Dummy body\r\n' + ).format(version, header) + conn = FakeSocketHTTPConnection(msg) + self.assertIsNone(conn.sock) + conn.request('GET', '/open-connection') + with conn.getresponse() as response: + self.assertEqual(conn.sock is None, not reuse) + response.read() + self.assertEqual(conn.sock is None, not reuse) + self.assertEqual(conn.connections, 1) + conn.request('GET', '/subsequent-request') + self.assertEqual(conn.connections, 1 if reuse else 2) + + def test_disconnected(self): + + def make_reset_reader(text): + """Return BufferedReader that raises ECONNRESET at EOF""" + stream = io.BytesIO(text) + def readinto(buffer): + size = io.BytesIO.readinto(stream, buffer) + if size == 0: + raise ConnectionResetError() + return size + stream.readinto = readinto + return io.BufferedReader(stream) + + tests = ( + (io.BytesIO, client.RemoteDisconnected), + (make_reset_reader, ConnectionResetError), + ) + for stream_factory, exception in tests: + with self.subTest(exception=exception): + conn = FakeSocketHTTPConnection(b'', stream_factory) + conn.request('GET', '/eof-response') + self.assertRaises(exception, conn.getresponse) + self.assertIsNone(conn.sock) + # HTTPConnection.connect() should be automatically invoked + conn.request('GET', '/reconnect') + self.assertEqual(conn.connections, 2) + + def test_100_close(self): + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + # Missing final response + ) + conn.request('GET', '/', headers={'Expect': '100-continue'}) + self.assertRaises(client.RemoteDisconnected, conn.getresponse) + self.assertIsNone(conn.sock) + conn.request('GET', '/reconnect') + self.assertEqual(conn.connections, 2) + + +class HTTPSTest(TestCase): + + def setUp(self): + if not hasattr(client, 'HTTPSConnection'): + self.skipTest('ssl support required') + + def make_server(self, certfile): + from test.ssl_servers import make_https_server + return make_https_server(self, certfile=certfile) + + def test_attributes(self): + # simple test to check it's storing the timeout + h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) + self.assertEqual(h.timeout, 30) + + def test_networked(self): + # Default settings: requires a valid cert from a trusted CA + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + h = client.HTTPSConnection('self-signed.pythontest.net', 443) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_networked_noverification(self): + # Switch off cert verification + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl._create_unverified_context() + h = client.HTTPSConnection('self-signed.pythontest.net', 443, + context=context) + h.request('GET', '/') + resp = h.getresponse() + h.close() + self.assertIn('nginx', resp.getheader('server')) + resp.close() + + @support.system_must_validate_cert + def test_networked_trusted_by_default_cert(self): + # Default settings: requires a valid cert from a trusted CA + support.requires('network') + with support.transient_internet('www.python.org'): + h = client.HTTPSConnection('www.python.org', 443) + h.request('GET', '/') + resp = h.getresponse() + content_type = resp.getheader('content-type') + resp.close() + h.close() + self.assertIn('text/html', content_type) + + def test_networked_good_cert(self): + # We feed the server's cert as a validating cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_selfsigned_pythontestdotnet) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h.request('GET', '/') + resp = h.getresponse() + server_string = resp.getheader('server') + resp.close() + h.close() + self.assertIn('nginx', server_string) + + def test_networked_bad_cert(self): + # We feed a "CA" cert that is unrelated to the server's cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_unknown_cert(self): + # The custom cert isn't known to the default trust bundle + import ssl + server = self.make_server(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_good_hostname(self): + # The (valid) cert validates the HTTP hostname + import ssl + server = self.make_server(CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port, context=context) + self.addCleanup(h.close) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.addCleanup(resp.close) + self.assertEqual(resp.status, 404) + + def test_local_bad_hostname(self): + # The (valid) cert doesn't validate the HTTP hostname + import ssl + server = self.make_server(CERT_fakehostname) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(CERT_fakehostname) + h = client.HTTPSConnection('localhost', server.port, context=context) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # Same with explicit check_hostname=True + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # With check_hostname=False, the mismatching is ignored + context.check_hostname = False + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=False) + h.request('GET', '/nonexistent') + resp = h.getresponse() + resp.close() + h.close() + self.assertEqual(resp.status, 404) + # The context's check_hostname setting is used if one isn't passed to + # HTTPSConnection. + context.check_hostname = False + h = client.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + resp.close() + h.close() + # Passing check_hostname to HTTPSConnection should override the + # context's setting. + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not available') + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPSConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:443", "www.python.org", 443), + ("www.python.org:", "www.python.org", 443), + ("www.python.org", "www.python.org", 443), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 443), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", + 443)): + c = client.HTTPSConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + +class RequestBodyTest(TestCase): + """Test cases where a request includes a message body.""" + + def setUp(self): + self.conn = client.HTTPConnection('example.com') + self.conn.sock = self.sock = FakeSocket("") + self.conn.sock = self.sock + + def get_headers_and_fp(self): + f = io.BytesIO(self.sock.data) + f.readline() # read the request line + message = client.parse_headers(f) + return message, f + + def test_list_body(self): + # Note that no content-length is automatically calculated for + # an iterable. The request will fall back to send chunked + # transfer encoding. + cases = ( + ([b'foo', b'bar'], b'3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n'), + ((b'foo', b'bar'), b'3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n'), + ) + for body, expected in cases: + with self.subTest(body): + self.conn = client.HTTPConnection('example.com') + self.conn.sock = self.sock = FakeSocket('') + + self.conn.request('PUT', '/url', body) + msg, f = self.get_headers_and_fp() + self.assertNotIn('Content-Type', msg) + self.assertNotIn('Content-Length', msg) + self.assertEqual(msg.get('Transfer-Encoding'), 'chunked') + self.assertEqual(expected, f.read()) + + def test_manual_content_length(self): + # Set an incorrect content-length so that we can verify that + # it will not be over-ridden by the library. + self.conn.request("PUT", "/url", "body", + {"Content-Length": "42"}) + message, f = self.get_headers_and_fp() + self.assertEqual("42", message.get("content-length")) + self.assertEqual(4, len(f.read())) + + def test_ascii_body(self): + self.conn.request("PUT", "/url", "body") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("4", message.get("content-length")) + self.assertEqual(b'body', f.read()) + + def test_latin1_body(self): + self.conn.request("PUT", "/url", "body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_bytes_body(self): + self.conn.request("PUT", "/url", b"body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_text_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as f: + f.write("body") + with open(support.TESTFN) as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + # No content-length will be determined for files; the body + # will be sent using chunked transfer encoding instead. + self.assertIsNone(message.get("content-length")) + self.assertEqual("chunked", message.get("transfer-encoding")) + self.assertEqual(b'4\r\nbody\r\n0\r\n\r\n', f.read()) + + def test_binary_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "wb") as f: + f.write(b"body\xc1") + with open(support.TESTFN, "rb") as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("chunked", message.get("Transfer-Encoding")) + self.assertNotIn("Content-Length", message) + self.assertEqual(b'5\r\nbody\xc1\r\n0\r\n\r\n', f.read()) + + +class HTTPResponseTest(TestCase): + + def setUp(self): + body = "HTTP/1.1 200 Ok\r\nMy-Header: first-value\r\nMy-Header: \ + second-value\r\n\r\nText" + sock = FakeSocket(body) + self.resp = client.HTTPResponse(sock) + self.resp.begin() + + def test_getting_header(self): + header = self.resp.getheader('My-Header') + self.assertEqual(header, 'first-value, second-value') + + header = self.resp.getheader('My-Header', 'some default') + self.assertEqual(header, 'first-value, second-value') + + def test_getting_nonexistent_header_with_string_default(self): + header = self.resp.getheader('No-Such-Header', 'default-value') + self.assertEqual(header, 'default-value') + + def test_getting_nonexistent_header_with_iterable_default(self): + header = self.resp.getheader('No-Such-Header', ['default', 'values']) + self.assertEqual(header, 'default, values') + + header = self.resp.getheader('No-Such-Header', ('default', 'values')) + self.assertEqual(header, 'default, values') + + def test_getting_nonexistent_header_without_default(self): + header = self.resp.getheader('No-Such-Header') + self.assertEqual(header, None) + + def test_getting_header_defaultint(self): + header = self.resp.getheader('No-Such-Header',default=42) + self.assertEqual(header, 42) + +class TunnelTests(TestCase): + def setUp(self): + response_text = ( + 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n' # Reply to HEAD + 'Content-Length: 42\r\n\r\n' + ) + self.host = 'proxy.com' + self.conn = client.HTTPConnection(self.host) + self.conn._create_connection = self._create_connection(response_text) + + def tearDown(self): + self.conn.close() + + def _create_connection(self, response_text): + def create_connection(address, timeout=None, source_address=None): + return FakeSocket(response_text, host=address[0], port=address[1]) + return create_connection + + def test_set_tunnel_host_port_headers(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers) + + def test_disallow_set_tunnel_after_connect(self): + # Once connected, we shouldn't be able to tunnel anymore + self.conn.connect() + self.assertRaises(RuntimeError, self.conn.set_tunnel, + 'destination.com') + + def test_connect_with_tunnel(self): + self.conn.set_tunnel('destination.com') + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + # issue22095 + self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + # This test should be removed when CONNECT gets the HTTP/1.1 blessing + self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) + + def test_connect_put_request(self): + self.conn.set_tunnel('destination.com') + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + def test_tunnel_debuglog(self): + expected_header = 'X-Dummy: 1' + response_text = 'HTTP/1.0 200 OK\r\n{}\r\n\r\n'.format(expected_header) + + self.conn.set_debuglevel(1) + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + with support.captured_stdout() as output: + self.conn.request('PUT', '/', '') + lines = output.getvalue().splitlines() + self.assertIn('header: {}'.format(expected_header), lines) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/greentest/3.6/test_select.py b/src/greentest/3.6/test_select.py new file mode 100644 index 0000000..a973f3f --- /dev/null +++ b/src/greentest/3.6/test_select.py @@ -0,0 +1,82 @@ +import errno +import os +import select +import sys +import unittest +from test import support + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + try: + select.select([fd], [], [], 0) + except OSError as err: + self.assertEqual(err.errno, errno.EBADF) + else: + self.fail("exception not raised") + + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + def test_select(self): + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print('timeout =', tout) + rfd, wfd, xfd = select.select([p], [], [], tout) + if (rfd, wfd, xfd) == ([], [], []): + continue + if (rfd, wfd, xfd) == ([p], [], []): + line = p.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select():', rfd, wfd, xfd) + p.close() + + # Issue 16230: Crash on select resized list + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + self.assertEqual(select.select([], a, []), ([], a[:5], [])) + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.6/test_selectors.py b/src/greentest/3.6/test_selectors.py new file mode 100644 index 0000000..852b2fe --- /dev/null +++ b/src/greentest/3.6/test_selectors.py @@ -0,0 +1,526 @@ +import errno +import os +import random +import selectors +import signal +import socket +import sys +from test import support +from time import sleep +import unittest +import unittest.mock +import tempfile +from time import monotonic as time +try: + import resource +except ImportError: + resource = None + + +if hasattr(socket, 'socketpair'): + socketpair = socket.socketpair +else: + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): + with socket.socket(family, type, proto) as l: + l.bind((support.HOST, 0)) + l.listen() + c = socket.socket(family, type, proto) + try: + c.connect(l.getsockname()) + caddr = c.getsockname() + while True: + a, addr = l.accept() + # check that we've got the correct client + if addr == caddr: + return c, a + a.close() + except OSError: + c.close() + raise + + +def find_ready_matching(ready, flag): + match = [] + for key, events in ready: + if events & flag: + match.append(key.fileobj) + return match + + +class BaseSelectorTestCase(unittest.TestCase): + + def make_socketpair(self): + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + return rd, wr + + def test_register(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIsInstance(key, selectors.SelectorKey) + self.assertEqual(key.fileobj, rd) + self.assertEqual(key.fd, rd.fileno()) + self.assertEqual(key.events, selectors.EVENT_READ) + self.assertEqual(key.data, "data") + + # register an unknown event + self.assertRaises(ValueError, s.register, 0, 999999) + + # register an invalid FD + self.assertRaises(ValueError, s.register, -10, selectors.EVENT_READ) + + # register twice + self.assertRaises(KeyError, s.register, rd, selectors.EVENT_READ) + + # register the same FD, but with a different object + self.assertRaises(KeyError, s.register, rd.fileno(), + selectors.EVENT_READ) + + def test_unregister(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + s.unregister(rd) + + # unregister an unknown file obj + self.assertRaises(KeyError, s.unregister, 999999) + + # unregister twice + self.assertRaises(KeyError, s.unregister, rd) + + def test_unregister_after_fd_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + r, w = rd.fileno(), wr.fileno() + s.register(r, selectors.EVENT_READ) + s.register(w, selectors.EVENT_WRITE) + rd.close() + wr.close() + s.unregister(r) + s.unregister(w) + + @unittest.skipUnless(os.name == 'posix', "requires posix") + def test_unregister_after_fd_close_and_reuse(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + r, w = rd.fileno(), wr.fileno() + s.register(r, selectors.EVENT_READ) + s.register(w, selectors.EVENT_WRITE) + rd2, wr2 = self.make_socketpair() + rd.close() + wr.close() + os.dup2(rd2.fileno(), r) + os.dup2(wr2.fileno(), w) + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + s.unregister(r) + s.unregister(w) + + def test_unregister_after_socket_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + rd.close() + wr.close() + s.unregister(rd) + s.unregister(wr) + + def test_modify(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ) + + # modify events + key2 = s.modify(rd, selectors.EVENT_WRITE) + self.assertNotEqual(key.events, key2.events) + self.assertEqual(key2, s.get_key(rd)) + + s.unregister(rd) + + # modify data + d1 = object() + d2 = object() + + key = s.register(rd, selectors.EVENT_READ, d1) + key2 = s.modify(rd, selectors.EVENT_READ, d2) + self.assertEqual(key.events, key2.events) + self.assertNotEqual(key.data, key2.data) + self.assertEqual(key2, s.get_key(rd)) + self.assertEqual(key2.data, d2) + + # modify unknown file obj + self.assertRaises(KeyError, s.modify, 999999, selectors.EVENT_READ) + + # modify use a shortcut + d3 = object() + s.register = unittest.mock.Mock() + s.unregister = unittest.mock.Mock() + + s.modify(rd, selectors.EVENT_READ, d3) + self.assertFalse(s.register.called) + self.assertFalse(s.unregister.called) + + def test_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + mapping = s.get_map() + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + + s.close() + self.assertRaises(RuntimeError, s.get_key, rd) + self.assertRaises(RuntimeError, s.get_key, wr) + self.assertRaises(KeyError, mapping.__getitem__, rd) + self.assertRaises(KeyError, mapping.__getitem__, wr) + + def test_get_key(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertEqual(key, s.get_key(rd)) + + # unknown file obj + self.assertRaises(KeyError, s.get_key, 999999) + + def test_get_map(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + keys = s.get_map() + self.assertFalse(keys) + self.assertEqual(len(keys), 0) + self.assertEqual(list(keys), []) + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIn(rd, keys) + self.assertEqual(key, keys[rd]) + self.assertEqual(len(keys), 1) + self.assertEqual(list(keys), [rd.fileno()]) + self.assertEqual(list(keys.values()), [key]) + + # unknown file obj + with self.assertRaises(KeyError): + keys[999999] + + # Read-only mapping + with self.assertRaises(TypeError): + del keys[rd] + + def test_select(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + wr_key = s.register(wr, selectors.EVENT_WRITE) + + result = s.select() + for key, events in result: + self.assertTrue(isinstance(key, selectors.SelectorKey)) + self.assertTrue(events) + self.assertFalse(events & ~(selectors.EVENT_READ | + selectors.EVENT_WRITE)) + + self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result) + + def test_context_manager(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + with s as sel: + sel.register(rd, selectors.EVENT_READ) + sel.register(wr, selectors.EVENT_WRITE) + + self.assertRaises(RuntimeError, s.get_key, rd) + self.assertRaises(RuntimeError, s.get_key, wr) + + def test_fileno(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + if hasattr(s, 'fileno'): + fd = s.fileno() + self.assertTrue(isinstance(fd, int)) + self.assertGreaterEqual(fd, 0) + + def test_selector(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + NUM_SOCKETS = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_SOCKETS): + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = s.select() + ready_writers = find_ready_matching(ready, selectors.EVENT_WRITE) + if not ready_writers: + self.fail("no sockets ready for writing") + wr = random.choice(ready_writers) + wr.send(MSG) + + for i in range(10): + ready = s.select() + ready_readers = find_ready_matching(ready, + selectors.EVENT_READ) + if ready_readers: + break + # there might be a delay between the write to the write end and + # the read end is reported ready + sleep(0.1) + else: + self.fail("no sockets ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = rd.recv(MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + s.unregister(r2w[rd]) + s.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_SOCKETS) + + @unittest.skipIf(sys.platform == 'win32', + 'select.select() cannot be used with empty fd sets') + def test_empty_select(self): + # Issue #23009: Make sure EpollSelector.select() works when no FD is + # registered. + s = self.SELECTOR() + self.addCleanup(s.close) + self.assertEqual(s.select(timeout=0), []) + + def test_timeout(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(wr, selectors.EVENT_WRITE) + t = time() + self.assertEqual(1, len(s.select(0))) + self.assertEqual(1, len(s.select(-1))) + self.assertLess(time() - t, 0.5) + + s.unregister(wr) + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(0)) + self.assertFalse(s.select(-1)) + self.assertLess(time() - t, 0.5) + + t0 = time() + self.assertFalse(s.select(1)) + t1 = time() + dt = t1 - t0 + # Tolerate 2.0 seconds for very slow buildbots + self.assertTrue(0.8 <= dt <= 2.0, dt) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_select_interrupt_exc(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + class InterruptSelect(Exception): + pass + + def handler(*args): + raise InterruptSelect + + orig_alrm_handler = signal.signal(signal.SIGALRM, handler) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(signal.alarm, 0) + + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + # select() is interrupted by a signal which raises an exception + with self.assertRaises(InterruptSelect): + s.select(30) + # select() was interrupted before the timeout of 30 seconds + self.assertLess(time() - t, 5.0) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_select_interrupt_noraise(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + orig_alrm_handler = signal.signal(signal.SIGALRM, lambda *args: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(signal.alarm, 0) + + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + # select() is interrupted by a signal, but the signal handler doesn't + # raise an exception, so select() should by retries with a recomputed + # timeout + self.assertFalse(s.select(1.5)) + self.assertGreaterEqual(time() - t, 1.0) + + +class ScalableSelectorMixIn: + + # see issue #18963 for why it's skipped on older OS X versions + @support.requires_mac_ver(10, 5) + @unittest.skipUnless(resource, "Test needs resource module") + def test_above_fd_setsize(self): + # A scalable implementation should have no problem with more than + # FD_SETSIZE file descriptors. Since we don't know the value, we just + # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling. + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, + (soft, hard)) + NUM_FDS = min(hard, 2**16) + except (OSError, ValueError): + NUM_FDS = soft + + # guard for already allocated FDs (stdin, stdout...) + NUM_FDS -= 32 + + s = self.SELECTOR() + self.addCleanup(s.close) + + for i in range(NUM_FDS // 2): + try: + rd, wr = self.make_socketpair() + except OSError: + # too many FDs, skip - note that we should only catch EMFILE + # here, but apparently *BSD and Solaris can fail upon connect() + # or bind() with EADDRNOTAVAIL, so let's be safe + self.skipTest("FD limit reached") + + try: + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + except OSError as e: + if e.errno == errno.ENOSPC: + # this can be raised by epoll if we go over + # fs.epoll.max_user_watches sysctl + self.skipTest("FD limit reached") + raise + + self.assertEqual(NUM_FDS // 2, len(s.select())) + + +class DefaultSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.DefaultSelector + + +class SelectSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.SelectSelector + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class PollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'PollSelector', None) + + +@unittest.skipUnless(hasattr(selectors, 'EpollSelector'), + "Test needs selectors.EpollSelector") +class EpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'EpollSelector', None) + + def test_register_file(self): + # epoll(7) returns EPERM when given a file to watch + s = self.SELECTOR() + with tempfile.NamedTemporaryFile() as f: + with self.assertRaises(IOError): + s.register(f, selectors.EVENT_READ) + # the SelectorKey has been removed + with self.assertRaises(KeyError): + s.get_key(f) + + +@unittest.skipUnless(hasattr(selectors, 'KqueueSelector'), + "Test needs selectors.KqueueSelector)") +class KqueueSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'KqueueSelector', None) + + def test_register_bad_fd(self): + # a file descriptor that's been closed should raise an OSError + # with EBADF + s = self.SELECTOR() + bad_f = support.make_bad_fd() + with self.assertRaises(OSError) as cm: + s.register(bad_f, selectors.EVENT_READ) + self.assertEqual(cm.exception.errno, errno.EBADF) + # the SelectorKey has been removed + with self.assertRaises(KeyError): + s.get_key(bad_f) + + +@unittest.skipUnless(hasattr(selectors, 'DevpollSelector'), + "Test needs selectors.DevpollSelector") +class DevpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'DevpollSelector', None) + + + +def test_main(): + tests = [DefaultSelectorTestCase, SelectSelectorTestCase, + PollSelectorTestCase, EpollSelectorTestCase, + KqueueSelectorTestCase, DevpollSelectorTestCase] + support.run_unittest(*tests) + support.reap_children() + + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.6/test_smtpd.py b/src/greentest/3.6/test_smtpd.py new file mode 100644 index 0000000..3eebe94 --- /dev/null +++ b/src/greentest/3.6/test_smtpd.py @@ -0,0 +1,1013 @@ +import unittest +import textwrap +from test import support, mock_socket +import socket +import io +import smtpd +import asyncore + + +class DummyServer(smtpd.SMTPServer): + def __init__(self, *args, **kwargs): + smtpd.SMTPServer.__init__(self, *args, **kwargs) + self.messages = [] + if self._decode_data: + self.return_status = 'return status' + else: + self.return_status = b'return status' + + def process_message(self, peer, mailfrom, rcpttos, data, **kw): + self.messages.append((peer, mailfrom, rcpttos, data)) + if data == self.return_status: + return '250 Okish' + if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']: + return '250 SMTPUTF8 message okish' + + +class DummyDispatcherBroken(Exception): + pass + + +class BrokenDummyServer(DummyServer): + def listen(self, num): + raise DummyDispatcherBroken() + + +class SMTPDServerTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def test_process_message_unimplemented(self): + server = smtpd.SMTPServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + + write_line(b'HELO example') + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') + + def test_decode_data_and_enable_SMTPUTF8_raises(self): + self.assertRaises( + ValueError, + smtpd.SMTPServer, + (support.HOST, 0), + ('b', 0), + enable_SMTPUTF8=True, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class DebuggingServerTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def send_data(self, channel, data, enable_SMTPUTF8=False): + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + write_line(b'EHLO example') + if enable_SMTPUTF8: + write_line(b'MAIL From:eggs@example BODY=8BITMIME SMTPUTF8') + else: + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + write_line(data) + write_line(b'.') + + def test_process_message_with_decode_data_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nhello\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + From: test + X-Peer: peer-address + + hello + ------------ END MESSAGE ------------ + """)) + + def test_process_message_with_decode_data_false(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def test_process_message_with_enable_SMTPUTF8_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def test_process_SMTPUTF8_message_with_enable_SMTPUTF8_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n', + enable_SMTPUTF8=True) + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + mail options: ['BODY=8BITMIME', 'SMTPUTF8'] + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class TestFamilyDetection(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + @unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") + def test_socket_uses_IPv6(self): + server = smtpd.SMTPServer((support.HOSTv6, 0), (support.HOST, 0)) + self.assertEqual(server.socket.family, socket.AF_INET6) + + def test_socket_uses_IPv4(self): + server = smtpd.SMTPServer((support.HOST, 0), (support.HOSTv6, 0)) + self.assertEqual(server.socket.family, socket.AF_INET) + + +class TestRcptOptionParsing(unittest.TestCase): + error_response = (b'555 RCPT TO parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_params_rejected(self): + server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: foo=bar') + self.assertEqual(channel.socket.last, self.error_response) + + def test_nothing_accepted(self): + server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: ') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class TestMailOptionParsing(unittest.TestCase): + error_response = (b'555 MAIL FROM parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_with_decode_data_true(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + b'MAIL from: size=20 BODY=UNKNOWN', + b'MAIL from: size=20 body=8bitmime', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line(channel, b'MAIL from: size=20') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_decode_data_false(self): + server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line( + channel, + b'MAIL from: size=20 SMTPUTF8 BODY=UNKNOWN') + self.assertEqual( + channel.socket.last, + b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n') + self.write_line( + channel, b'MAIL from: size=20 body=8bitmime') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_enable_smtputf8_true(self): + server = DummyServer((support.HOST, 0), ('b', 0), enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + self.write_line(channel, b'EHLO example') + self.write_line( + channel, + b'MAIL from: size=20 body=8bitmime smtputf8') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class SMTPDChannelTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_broken_connect(self): + self.assertRaises( + DummyDispatcherBroken, BrokenDummyServer, + (support.HOST, 0), ('b', 0), decode_data=True) + + def test_decode_data_and_enable_SMTPUTF8_raises(self): + self.assertRaises( + ValueError, smtpd.SMTPChannel, + self.server, self.channel.conn, self.channel.addr, + enable_SMTPUTF8=True, decode_data=True) + + def test_server_accept(self): + self.server.handle_accept() + + def test_missing_data(self): + self.write_line(b'') + self.assertEqual(self.channel.socket.last, + b'500 Error: bad syntax\r\n') + + def test_EHLO(self): + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, b'250 HELP\r\n') + + def test_EHLO_bad_syntax(self): + self.write_line(b'EHLO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: EHLO hostname\r\n') + + def test_EHLO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_EHLO_HELO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO(self): + name = smtpd.socket.getfqdn() + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + '250 {}\r\n'.format(name).encode('ascii')) + + def test_HELO_EHLO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELP(self): + self.write_line(b'HELP') + self.assertEqual(self.channel.socket.last, + b'250 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELP_command(self): + self.write_line(b'HELP MAIL') + self.assertEqual(self.channel.socket.last, + b'250 Syntax: MAIL FROM:
\r\n') + + def test_HELP_command_unknown(self): + self.write_line(b'HELP SPAM') + self.assertEqual(self.channel.socket.last, + b'501 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELO_bad_syntax(self): + self.write_line(b'HELO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: HELO hostname\r\n') + + def test_HELO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO_parameter_rejected_when_extensions_not_enabled(self): + self.extended_smtp = False + self.write_line(b'HELO example') + self.write_line(b'MAIL from: SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_allows_space_after_colon(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_extended_MAIL_allows_space_after_colon(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: size=20') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_NOOP(self): + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_HELO_NOOP(self): + self.write_line(b'HELO example') + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_NOOP_bad_syntax(self): + self.write_line(b'NOOP hi') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: NOOP\r\n') + + def test_QUIT(self): + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_HELO_QUIT(self): + self.write_line(b'HELO example') + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_QUIT_arg_ignored(self): + self.write_line(b'QUIT bye bye') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_bad_state(self): + self.channel.smtp_state = 'BAD STATE' + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'451 Internal confusion\r\n') + + def test_command_too_long(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ' + + b'a' * self.channel.command_size_limit + + b'@example') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_limit_extended_with_SIZE(self): + self.write_line(b'EHLO example') + fill_len = self.channel.command_size_limit - len('MAIL from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 26) + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_rejects_SMTPUTF8_by_default(self): + self.write_line(b'EHLO example') + self.write_line( + b'MAIL from: BODY=8BITMIME SMTPUTF8') + self.assertEqual(self.channel.socket.last[0:1], b'5') + + def test_data_longer_than_default_data_size_limit(self): + # Hack the default so we don't have to generate so much data. + self.channel.data_size_limit = 1048 + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'A' * self.channel.data_size_limit + + b'A\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + def test_MAIL_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=512') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_MAIL_invalid_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=invalid') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_RCPT_unknown_parameters(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 MAIL FROM parameters not recognized or not implemented\r\n') + + self.write_line(b'MAIL FROM:') + self.write_line(b'RCPT TO: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 RCPT TO parameters not recognized or not implemented\r\n') + + def test_MAIL_size_parameter_larger_than_default_data_size_limit(self): + self.channel.data_size_limit = 1048 + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=2096') + self.assertEqual(self.channel.socket.last, + b'552 Error: message size exceeds fixed maximum message size\r\n') + + def test_need_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'RCPT to:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: need MAIL command\r\n') + + def test_MAIL_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_missing_address(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_chevrons(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_empty_chevrons(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from:<>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_quoted_localpart(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_nested_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:eggs@example') + self.write_line(b'MAIL from:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: nested MAIL command\r\n') + + def test_VRFY(self): + self.write_line(b'VRFY eggs@example') + self.assertEqual(self.channel.socket.last, + b'252 Cannot VRFY user, but will accept message and attempt ' + \ + b'delivery\r\n') + + def test_VRFY_syntax(self): + self.write_line(b'VRFY') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: VRFY
\r\n') + + def test_EXPN_not_implemented(self): + self.write_line(b'EXPN') + self.assertEqual(self.channel.socket.last, + b'502 EXPN not implemented\r\n') + + def test_no_HELO_MAIL(self): + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_need_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'503 Error: need RCPT command\r\n') + + def test_RCPT_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
\r\n') + + def test_RCPT_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
[SP ]\r\n') + + def test_RCPT_lowercase_to_OK(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to: ') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_no_HELO_RCPT(self): + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example'], + 'data\nmore')]) + + def test_DATA_syntax(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n') + + def test_no_HELO_DATA(self): + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_transparency_section_4_5_2(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'..\r\n.\r\n') + self.assertEqual(self.channel.received_data, '.') + + def test_multiple_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RCPT To:ham@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example','ham@example'], + 'data')]) + + def test_manual_status(self): + # checks that the Channel is able to return a custom status message + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'return status\r\n.') + self.assertEqual(self.channel.socket.last, b'250 Okish\r\n') + + def test_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'MAIL From:foo@example') + self.write_line(b'RCPT To:eggs@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'foo@example', + ['eggs@example'], + 'data')]) + + def test_HELO_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_RSET_syntax(self): + self.write_line(b'RSET hi') + self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n') + + def test_unknown_command(self): + self.write_line(b'UNKNOWN_CMD') + self.assertEqual(self.channel.socket.last, + b'500 Error: command "UNKNOWN_CMD" not ' + \ + b'recognized\r\n') + + def test_attribute_deprecations(self): + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__server + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__server = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__line + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__line = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__state + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__state = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__greeting + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__greeting = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__mailfrom + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__mailfrom = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__rcpttos + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__rcpttos = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__data + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__data = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__fqdn + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__fqdn = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__peer + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__peer = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__conn + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__conn = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__addr + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__addr = 'spam' + +@unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") +class SMTPDChannelIPv6Test(SMTPDChannelTest): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOSTv6, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + +class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set DATA size limit to 32 bytes for easy testing + self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_data_limit_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example'], + 'data\nmore')]) + + def test_data_limit_dialog_too_much_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'This message is longer than 32 bytes\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + +class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, b'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87\n' + b'and some plain ascii') + + +class SMTPDChannelWithDecodeDataTrue(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set decode_data to True + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, 'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + 'utf8 enriched text: żźć\nand some plain ascii') + + +class SMTPDChannelTestWithEnableSMTPUTF8True(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + enable_SMTPUTF8=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_MAIL_command_accepts_SMTPUTF8_when_announced(self): + self.write_line(b'EHLO example') + self.write_line( + 'MAIL from: BODY=8BITMIME SMTPUTF8'.encode( + 'utf-8') + ) + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_process_smtputf8_message(self): + self.write_line(b'EHLO example') + for mail_parameters in [b'', b'BODY=8BITMIME SMTPUTF8']: + self.write_line(b'MAIL from: ' + mail_parameters) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'rcpt to:') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'data') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'c\r\n.') + if mail_parameters == b'': + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + else: + self.assertEqual(self.channel.socket.last, + b'250 SMTPUTF8 message okish\r\n') + + def test_utf8_data(self): + self.write_line(b'EHLO example') + self.write_line( + 'MAIL From: naïve@examplé BODY=8BITMIME SMTPUTF8'.encode('utf-8')) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line('RCPT To:späm@examplé'.encode('utf-8')) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + + def test_MAIL_command_limit_extended_with_SIZE_and_SMTPUTF8(self): + self.write_line(b'ehlo example') + fill_len = (512 + 26 + 10) - len('mail from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 1) + + b'@example>') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_multiple_emails_with_extended_command_length(self): + self.write_line(b'ehlo example') + fill_len = (512 + 26 + 10) - len('mail from:<@example>') + for char in [b'a', b'b', b'c']: + self.write_line(b'MAIL from:<' + char * fill_len + b'a@example>') + self.assertEqual(self.channel.socket.last[0:3], b'500') + self.write_line(b'MAIL from:<' + char * fill_len + b'@example>') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'rcpt to:') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'data') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'test\r\n.') + self.assertEqual(self.channel.socket.last[0:3], b'250') + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = { + "program", "Devnull", "DEBUGSTREAM", "NEWLINE", "COMMASPACE", + "DATA_SIZE_DEFAULT", "usage", "Options", "parseargs", + + } + support.check__all__(self, smtpd, blacklist=blacklist) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.6/test_socket.py b/src/greentest/3.6/test_socket.py new file mode 100644 index 0000000..80dfc40 --- /dev/null +++ b/src/greentest/3.6/test_socket.py @@ -0,0 +1,5641 @@ +import unittest +from test import support + +import errno +import io +import itertools +import socket +import select +import tempfile +import time +import traceback +import queue +import sys +import os +import array +import contextlib +from weakref import proxy +import signal +import math +import pickle +import struct +import random +import string +try: + import multiprocessing +except ImportError: + multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None + +HOST = support.HOST +MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return + +try: + import _thread as thread + import threading +except ImportError: + thread = None + threading = None +try: + import _socket +except ImportError: + _socket = None + + +def _have_socket_can(): + """Check whether CAN sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_rds(): + """Check whether RDS sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_alg(): + """Check whether AF_ALG sockets are supported on this host.""" + try: + s = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +HAVE_SOCKET_CAN = _have_socket_can() + +HAVE_SOCKET_RDS = _have_socket_rds() + +HAVE_SOCKET_ALG = _have_socket_alg() + +# Size in bytes of the int type +SIZEOF_INT = array.array("i").itemsize + +class SocketTCPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.serv.listen() + + def tearDown(self): + self.serv.close() + self.serv = None + +class SocketUDPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.port = support.bind_port(self.serv) + + def tearDown(self): + self.serv.close() + self.serv = None + +class ThreadSafeCleanupTestCase(unittest.TestCase): + """Subclass of unittest.TestCase with thread-safe cleanup methods. + + This subclass protects the addCleanup() and doCleanups() methods + with a recursive lock. + """ + + if threading: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cleanup_lock = threading.RLock() + + def addCleanup(self, *args, **kwargs): + with self._cleanup_lock: + return super().addCleanup(*args, **kwargs) + + def doCleanups(self, *args, **kwargs): + with self._cleanup_lock: + return super().doCleanups(*args, **kwargs) + +class SocketCANTest(unittest.TestCase): + + """To be able to run this test, a `vcan0` CAN interface can be created with + the following commands: + # modprobe vcan + # ip link add dev vcan0 type vcan + # ifconfig vcan0 up + """ + interface = 'vcan0' + bufsize = 128 + + """The CAN frame structure is defined in : + + struct can_frame { + canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ + __u8 can_dlc; /* data length code: 0 .. 8 */ + __u8 data[8] __attribute__((aligned(8))); + }; + """ + can_frame_fmt = "=IB3x8s" + can_frame_size = struct.calcsize(can_frame_fmt) + + """The Broadcast Management Command frame structure is defined + in : + + struct bcm_msg_head { + __u32 opcode; + __u32 flags; + __u32 count; + struct timeval ival1, ival2; + canid_t can_id; + __u32 nframes; + struct can_frame frames[0]; + } + + `bcm_msg_head` must be 8 bytes aligned because of the `frames` member (see + `struct can_frame` definition). Must use native not standard types for packing. + """ + bcm_cmd_msg_fmt = "@3I4l2I" + bcm_cmd_msg_fmt += "x" * (struct.calcsize(bcm_cmd_msg_fmt) % 8) + + def setUp(self): + self.s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + self.addCleanup(self.s.close) + try: + self.s.bind((self.interface,)) + except OSError: + self.skipTest('network interface `%s` does not exist' % + self.interface) + + +class SocketRDSTest(unittest.TestCase): + + """To be able to run this test, the `rds` kernel module must be loaded: + # modprobe rds + """ + bufsize = 8192 + + def setUp(self): + self.serv = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + self.addCleanup(self.serv.close) + try: + self.port = support.bind_port(self.serv) + except OSError: + self.skipTest('unable to bind RDS socket') + + +class ThreadableTest: + """Threadable Test class + + The ThreadableTest class makes it easy to create a threaded + client/server pair from an existing unit test. To create a + new threaded class from an existing unit test, use multiple + inheritance: + + class NewClass (OldClass, ThreadableTest): + pass + + This class defines two new fixture functions with obvious + purposes for overriding: + + clientSetUp () + clientTearDown () + + Any new test functions within the class must then define + tests in pairs, where the test name is preceded with a + '_' to indicate the client portion of the test. Ex: + + def testFoo(self): + # Server portion + + def _testFoo(self): + # Client portion + + Any exceptions raised by the clients during their tests + are caught and transferred to the main thread to alert + the testing framework. + + Note, the server setup function cannot call any blocking + functions that rely on the client thread during setup, + unless serverExplicitReady() is called just before + the blocking call (such as in setting up a client/server + connection and performing the accept() in setUp(). + """ + + def __init__(self): + # Swap the true setup function + self.__setUp = self.setUp + self.__tearDown = self.tearDown + self.setUp = self._setUp + self.tearDown = self._tearDown + + def serverExplicitReady(self): + """This method allows the server to explicitly indicate that + it wants the client thread to proceed. This is useful if the + server is about to execute a blocking routine that is + dependent upon the client thread during its setup routine.""" + self.server_ready.set() + + def _setUp(self): + self.server_ready = threading.Event() + self.client_ready = threading.Event() + self.done = threading.Event() + self.queue = queue.Queue(1) + self.server_crashed = False + + # Do some munging to start the client test. + methodname = self.id() + i = methodname.rfind('.') + methodname = methodname[i+1:] + test_method = getattr(self, '_' + methodname) + self.client_thread = thread.start_new_thread( + self.clientRun, (test_method,)) + + try: + self.__setUp() + except: + self.server_crashed = True + raise + finally: + self.server_ready.set() + self.client_ready.wait() + + def _tearDown(self): + self.__tearDown() + self.done.wait() + + if self.queue.qsize(): + exc = self.queue.get() + raise exc + + def clientRun(self, test_func): + self.server_ready.wait() + try: + self.clientSetUp() + except BaseException as e: + self.queue.put(e) + self.clientTearDown() + return + finally: + self.client_ready.set() + if self.server_crashed: + self.clientTearDown() + return + if not hasattr(test_func, '__call__'): + raise TypeError("test_func must be a callable function") + try: + test_func() + except BaseException as e: + self.queue.put(e) + finally: + self.clientTearDown() + + def clientSetUp(self): + raise NotImplementedError("clientSetUp must be implemented.") + + def clientTearDown(self): + self.done.set() + thread.exit() + +class ThreadedTCPSocketTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketUDPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedCANSocketTest(SocketCANTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketCANTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + try: + self.cli.bind((self.interface,)) + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedRDSSocketTest(SocketRDSTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketRDSTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + try: + # RDS sockets must be bound explicitly to send or receive data + self.cli.bind((HOST, 0)) + self.cli_addr = self.cli.getsockname() + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class SocketConnectedTest(ThreadedTCPSocketTest): + """Socket tests for client-server connection. + + self.cli_conn is a client socket connected to the server. The + setUp() method guarantees that it is connected to the server. + """ + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + ThreadedTCPSocketTest.setUp(self) + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + ThreadedTCPSocketTest.tearDown(self) + + def clientSetUp(self): + ThreadedTCPSocketTest.clientSetUp(self) + self.cli.connect((HOST, self.port)) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + ThreadedTCPSocketTest.clientTearDown(self) + +class SocketPairTest(unittest.TestCase, ThreadableTest): + + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.serv, self.cli = socket.socketpair() + + def tearDown(self): + self.serv.close() + self.serv = None + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +# The following classes are used by the sendmsg()/recvmsg() tests. +# Combining, for instance, ConnectedStreamTestMixin and TCPTestBase +# gives a drop-in replacement for SocketConnectedTest, but different +# address families can be used, and the attributes serv_addr and +# cli_addr will be set to the addresses of the endpoints. + +class SocketTestBase(unittest.TestCase): + """A base class for socket tests. + + Subclasses must provide methods newSocket() to return a new socket + and bindSock(sock) to bind it to an unused address. + + Creates a socket self.serv and sets self.serv_addr to its address. + """ + + def setUp(self): + self.serv = self.newSocket() + self.bindServer() + + def bindServer(self): + """Bind server socket and set self.serv_addr to its address.""" + self.bindSock(self.serv) + self.serv_addr = self.serv.getsockname() + + def tearDown(self): + self.serv.close() + self.serv = None + + +class SocketListeningTestMixin(SocketTestBase): + """Mixin to listen on the server socket.""" + + def setUp(self): + super().setUp() + self.serv.listen() + + +class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, + ThreadableTest): + """Mixin to add client socket and allow client/server tests. + + Client socket is self.cli and its address is self.cli_addr. See + ThreadableTest for usage information. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = self.newClientSocket() + self.bindClient() + + def newClientSocket(self): + """Return a new socket for use as client.""" + return self.newSocket() + + def bindClient(self): + """Bind client socket and set self.cli_addr to its address.""" + self.bindSock(self.cli) + self.cli_addr = self.cli.getsockname() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +class ConnectedStreamTestMixin(SocketListeningTestMixin, + ThreadedSocketTestMixin): + """Mixin to allow client/server stream tests with connected client. + + Server's socket representing connection to client is self.cli_conn + and client's connection to server is self.serv_conn. (Based on + SocketConnectedTest.) + """ + + def setUp(self): + super().setUp() + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + super().tearDown() + + def clientSetUp(self): + super().clientSetUp() + self.cli.connect(self.serv_addr) + self.serv_conn = self.cli + + def clientTearDown(self): + try: + self.serv_conn.close() + self.serv_conn = None + except AttributeError: + pass + super().clientTearDown() + + +class UnixSocketTestBase(SocketTestBase): + """Base class for Unix-domain socket tests.""" + + # This class is used for file descriptor passing tests, so we + # create the sockets in a private directory so that other users + # can't send anything that might be problematic for a privileged + # user running the tests. + + def setUp(self): + self.dir_path = tempfile.mkdtemp() + self.addCleanup(os.rmdir, self.dir_path) + super().setUp() + + def bindSock(self, sock): + path = tempfile.mktemp(dir=self.dir_path) + support.bind_unix_socket(sock, path) + self.addCleanup(support.unlink, path) + +class UnixStreamBase(UnixSocketTestBase): + """Base class for Unix-domain SOCK_STREAM tests.""" + + def newSocket(self): + return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + +class InetTestBase(SocketTestBase): + """Base class for IPv4 socket tests.""" + + host = HOST + + def setUp(self): + super().setUp() + self.port = self.serv_addr[1] + + def bindSock(self, sock): + support.bind_port(sock, host=self.host) + +class TCPTestBase(InetTestBase): + """Base class for TCP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + +class UDPTestBase(InetTestBase): + """Base class for UDP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + +class SCTPStreamBase(InetTestBase): + """Base class for SCTP tests in one-to-one (SOCK_STREAM) mode.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM, + socket.IPPROTO_SCTP) + + +class Inet6TestBase(InetTestBase): + """Base class for IPv6 socket tests.""" + + host = support.HOSTv6 + +class UDP6TestBase(Inet6TestBase): + """Base class for UDP-over-IPv6 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + + +# Test-skipping decorators for use with ThreadableTest. + +def skipWithClientIf(condition, reason): + """Skip decorated test if condition is true, add client_skip decorator. + + If the decorated object is not a class, sets its attribute + "client_skip" to a decorator which will return an empty function + if the test is to be skipped, or the original function if it is + not. This can be used to avoid running the client part of a + skipped test when using ThreadableTest. + """ + def client_pass(*args, **kwargs): + pass + def skipdec(obj): + retval = unittest.skip(reason)(obj) + if not isinstance(obj, type): + retval.client_skip = lambda f: client_pass + return retval + def noskipdec(obj): + if not (isinstance(obj, type) or hasattr(obj, "client_skip")): + obj.client_skip = lambda f: f + return obj + return skipdec if condition else noskipdec + + +def requireAttrs(obj, *attributes): + """Skip decorated test if obj is missing any of the given attributes. + + Sets client_skip attribute as skipWithClientIf() does. + """ + missing = [name for name in attributes if not hasattr(obj, name)] + return skipWithClientIf( + missing, "don't have " + ", ".join(name for name in missing)) + + +def requireSocket(*args): + """Skip decorated test if a socket cannot be created with given arguments. + + When an argument is given as a string, will use the value of that + attribute of the socket module, or skip the test if it doesn't + exist. Sets client_skip attribute as skipWithClientIf() does. + """ + err = None + missing = [obj for obj in args if + isinstance(obj, str) and not hasattr(socket, obj)] + if missing: + err = "don't have " + ", ".join(name for name in missing) + else: + callargs = [getattr(socket, obj) if isinstance(obj, str) else obj + for obj in args] + try: + s = socket.socket(*callargs) + except OSError as e: + # XXX: check errno? + err = str(e) + else: + s.close() + return skipWithClientIf( + err is not None, + "can't create socket({0}): {1}".format( + ", ".join(str(o) for o in args), err)) + + +####################################################################### +## Begin Tests + +class GeneralModuleTests(unittest.TestCase): + + def test_SocketType_is_socketobject(self): + import _socket + self.assertTrue(socket.SocketType is _socket.socket) + s = socket.socket() + self.assertIsInstance(s, socket.SocketType) + s.close() + + def test_repr(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with s: + self.assertIn('fd=%i' % s.fileno(), repr(s)) + self.assertIn('family=%s' % socket.AF_INET, repr(s)) + self.assertIn('type=%s' % socket.SOCK_STREAM, repr(s)) + self.assertIn('proto=0', repr(s)) + self.assertNotIn('raddr', repr(s)) + s.bind(('127.0.0.1', 0)) + self.assertIn('laddr', repr(s)) + self.assertIn(str(s.getsockname()), repr(s)) + self.assertIn('[closed]', repr(s)) + self.assertNotIn('laddr', repr(s)) + + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_csocket_repr(self): + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + try: + expected = ('' + % (s.fileno(), s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + finally: + s.close() + expected = ('' + % (s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + + def test_weakref(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + p = proxy(s) + self.assertEqual(p.fileno(), s.fileno()) + s.close() + s = None + try: + p.fileno() + except ReferenceError: + pass + else: + self.fail('Socket proxy still exists') + + def testSocketError(self): + # Testing socket module exceptions + msg = "Error raising socket exception (%s)." + with self.assertRaises(OSError, msg=msg % 'OSError'): + raise OSError + with self.assertRaises(OSError, msg=msg % 'socket.herror'): + raise socket.herror + with self.assertRaises(OSError, msg=msg % 'socket.gaierror'): + raise socket.gaierror + + def testSendtoErrors(self): + # Testing that sendto doesn't mask failures. See #10169. + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind(('', 0)) + sockname = s.getsockname() + # 2 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'str'") + with self.assertRaises(TypeError) as cm: + s.sendto(5j, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'complex'") + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None) + self.assertIn('not NoneType',str(cm.exception)) + # 3 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', 0, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'str'") + with self.assertRaises(TypeError) as cm: + s.sendto(5j, 0, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'complex'") + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, None) + self.assertIn('not NoneType', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 'bar', sockname) + self.assertIn('an integer is required', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None, None) + self.assertIn('an integer is required', str(cm.exception)) + # wrong number of args + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo') + self.assertIn('(1 given)', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, sockname, 4) + self.assertIn('(4 given)', str(cm.exception)) + + def testCrucialConstants(self): + # Testing for mission critical constants + socket.AF_INET + socket.SOCK_STREAM + socket.SOCK_DGRAM + socket.SOCK_RAW + socket.SOCK_RDM + socket.SOCK_SEQPACKET + socket.SOL_SOCKET + socket.SO_REUSEADDR + + def testHostnameRes(self): + # Testing hostname resolution mechanisms + hostname = socket.gethostname() + try: + ip = socket.gethostbyname(hostname) + except OSError: + # Probably name lookup wasn't set up right; skip this test + self.skipTest('name lookup failure') + self.assertTrue(ip.find('.') >= 0, "Error resolving host to ip.") + try: + hname, aliases, ipaddrs = socket.gethostbyaddr(ip) + except OSError: + # Probably a similar problem as above; skip this test + self.skipTest('name lookup failure') + all_host_names = [hostname, hname] + aliases + fqhn = socket.getfqdn(ip) + if not fqhn in all_host_names: + self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + + def test_host_resolution(self): + for addr in [support.HOST, '10.0.0.1', '255.255.255.255']: + self.assertEqual(socket.gethostbyname(addr), addr) + + # we don't test support.HOSTv6 because there's a chance it doesn't have + # a matching name entry (e.g. 'ip6-localhost') + for host in [support.HOST]: + self.assertIn(host, socket.gethostbyaddr(host)[2]) + + def test_host_resolution_bad_address(self): + # These are all malformed IP addresses and expected not to resolve to + # any result. But some ISPs, e.g. AWS, may successfully resolve these + # IPs. + explanation = ( + "resolving an invalid IP address did not raise OSError; " + "can be caused by a broken DNS server" + ) + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + with self.assertRaises(OSError): + socket.gethostbyname(addr) + with self.assertRaises(OSError, msg=explanation): + socket.gethostbyaddr(addr) + + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") + @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") + def test_sethostname(self): + oldhn = socket.gethostname() + try: + socket.sethostname('new') + except OSError as e: + if e.errno == errno.EPERM: + self.skipTest("test should be run as root") + else: + raise + try: + # running test as root! + self.assertEqual(socket.gethostname(), 'new') + # Should work with bytes objects too + socket.sethostname(b'bar') + self.assertEqual(socket.gethostname(), 'bar') + finally: + socket.sethostname(oldhn) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInterfaceNameIndex(self): + interfaces = socket.if_nameindex() + for index, name in interfaces: + self.assertIsInstance(index, int) + self.assertIsInstance(name, str) + # interface indices are non-zero integers + self.assertGreater(index, 0) + _index = socket.if_nametoindex(name) + self.assertIsInstance(_index, int) + self.assertEqual(index, _index) + _name = socket.if_indextoname(index) + self.assertIsInstance(_name, str) + self.assertEqual(name, _name) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInvalidInterfaceNameIndex(self): + # test nonexistent interface index/name + self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') + # test with invalid values + self.assertRaises(TypeError, socket.if_nametoindex, 0) + self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') + def testRefCountGetNameInfo(self): + # Testing reference count for getnameinfo + try: + # On some versions, this loses a reference + orig = sys.getrefcount(__name__) + socket.getnameinfo(__name__,0) + except TypeError: + if sys.getrefcount(__name__) != orig: + self.fail("socket.getnameinfo loses a reference") + + def testInterpreterCrash(self): + # Making sure getnameinfo doesn't crash the interpreter + try: + # On some versions, this crashes the interpreter. + socket.getnameinfo(('x', 0, 0, 0), 0) + except OSError: + pass + + def testNtoH(self): + # This just checks that htons etc. are their own inverse, + # when looking at the lower 16 or 32 bits. + sizes = {socket.htonl: 32, socket.ntohl: 32, + socket.htons: 16, socket.ntohs: 16} + for func, size in sizes.items(): + mask = (1<") + + def test_unusable_closed_socketio(self): + with socket.socket() as sock: + fp = sock.makefile("rb", buffering=0) + self.assertTrue(fp.readable()) + self.assertFalse(fp.writable()) + self.assertFalse(fp.seekable()) + fp.close() + self.assertRaises(ValueError, fp.readable) + self.assertRaises(ValueError, fp.writable) + self.assertRaises(ValueError, fp.seekable) + + def test_makefile_mode(self): + for mode in 'r', 'rb', 'rw', 'w', 'wb': + with self.subTest(mode=mode): + with socket.socket() as sock: + with sock.makefile(mode) as fp: + self.assertEqual(fp.mode, mode) + + def test_makefile_invalid_mode(self): + for mode in 'rt', 'x', '+', 'a': + with self.subTest(mode=mode): + with socket.socket() as sock: + with self.assertRaisesRegex(ValueError, 'invalid mode'): + sock.makefile(mode) + + def test_pickle(self): + sock = socket.socket() + with sock: + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, sock, protocol) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + family = pickle.loads(pickle.dumps(socket.AF_INET, protocol)) + self.assertEqual(family, socket.AF_INET) + type = pickle.loads(pickle.dumps(socket.SOCK_STREAM, protocol)) + self.assertEqual(type, socket.SOCK_STREAM) + + def test_listen_backlog(self): + for backlog in 0, -1: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv: + srv.bind((HOST, 0)) + srv.listen(backlog) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv: + srv.bind((HOST, 0)) + srv.listen() + + @support.cpython_only + def test_listen_backlog_overflow(self): + # Issue 15989 + import _testcapi + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.bind((HOST, 0)) + self.assertRaises(OverflowError, srv.listen, _testcapi.INT_MAX + 1) + srv.close() + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + def test_flowinfo(self): + self.assertRaises(OverflowError, socket.getnameinfo, + (support.HOSTv6, 0, 0xffffffff), 0) + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + self.assertRaises(OverflowError, s.bind, (support.HOSTv6, 0, -10)) + + def test_str_for_enums(self): + # Make sure that the AF_* and SOCK_* constants have enum-like string + # reprs. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + self.assertEqual(str(s.family), 'AddressFamily.AF_INET') + self.assertEqual(str(s.type), 'SocketKind.SOCK_STREAM') + + @unittest.skipIf(os.name == 'nt', 'Will not work on Windows') + def test_uknown_socket_family_repr(self): + # Test that when created with a family that's not one of the known + # AF_*/SOCK_* constants, socket.family just returns the number. + # + # To do this we fool socket.socket into believing it already has an + # open fd because on this path it doesn't actually verify the family and + # type and populates the socket object. + # + # On Windows this trick won't work, so the test is skipped. + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + with socket.socket(family=42424, type=13331, fileno=fd) as s: + self.assertEqual(s.family, 42424) + self.assertEqual(s.type, 13331) + + @unittest.skipUnless(hasattr(os, 'sendfile'), 'test needs os.sendfile()') + def test__sendfile_use_sendfile(self): + class File: + def __init__(self, fd): + self.fd = fd + + def fileno(self): + return self.fd + with socket.socket() as sock: + fd = os.open(os.curdir, os.O_RDONLY) + os.close(fd) + with self.assertRaises(socket._GiveupOnSendfile): + sock._sendfile_use_sendfile(File(fd)) + with self.assertRaises(OverflowError): + sock._sendfile_use_sendfile(File(2**1000)) + with self.assertRaises(TypeError): + sock._sendfile_use_sendfile(File(None)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +class BasicCANTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_CAN + socket.PF_CAN + socket.CAN_RAW + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCMConstants(self): + socket.CAN_BCM + + # opcodes + socket.CAN_BCM_TX_SETUP # create (cyclic) transmission task + socket.CAN_BCM_TX_DELETE # remove (cyclic) transmission task + socket.CAN_BCM_TX_READ # read properties of (cyclic) transmission task + socket.CAN_BCM_TX_SEND # send one CAN frame + socket.CAN_BCM_RX_SETUP # create RX content filter subscription + socket.CAN_BCM_RX_DELETE # remove RX content filter subscription + socket.CAN_BCM_RX_READ # read properties of RX content filter subscription + socket.CAN_BCM_TX_STATUS # reply to TX_READ request + socket.CAN_BCM_TX_EXPIRED # notification on performed transmissions (count=0) + socket.CAN_BCM_RX_STATUS # reply to RX_READ request + socket.CAN_BCM_RX_TIMEOUT # cyclic message is absent + socket.CAN_BCM_RX_CHANGED # updated CAN frame (detected content change) + + def testCreateSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + pass + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testCreateBCMSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) as s: + pass + + def testBindAny(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.bind(('', )) + + def testTooLongInterfaceName(self): + # most systems limit IFNAMSIZ to 16, take 1024 to be sure + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + self.assertRaisesRegex(OSError, 'interface name too long', + s.bind, ('x' * 1024,)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_LOOPBACK"), + 'socket.CAN_RAW_LOOPBACK required for this test.') + def testLoopback(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + for loopback in (0, 1): + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK, + loopback) + self.assertEqual(loopback, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_FILTER"), + 'socket.CAN_RAW_FILTER required for this test.') + def testFilter(self): + can_id, can_mask = 0x200, 0x700 + can_filter = struct.pack("=II", can_id, can_mask) + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, can_filter) + self.assertEqual(can_filter, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, 8)) + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, bytearray(can_filter)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class CANTest(ThreadedCANSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedCANSocketTest.__init__(self, methodName=methodName) + + @classmethod + def build_can_frame(cls, can_id, data): + """Build a CAN frame.""" + can_dlc = len(data) + data = data.ljust(8, b'\x00') + return struct.pack(cls.can_frame_fmt, can_id, can_dlc, data) + + @classmethod + def dissect_can_frame(cls, frame): + """Dissect a CAN frame.""" + can_id, can_dlc, data = struct.unpack(cls.can_frame_fmt, frame) + return (can_id, can_dlc, data[:can_dlc]) + + def testSendFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + self.assertEqual(addr[0], self.interface) + self.assertEqual(addr[1], socket.AF_CAN) + + def _testSendFrame(self): + self.cf = self.build_can_frame(0x00, b'\x01\x02\x03\x04\x05') + self.cli.send(self.cf) + + def testSendMaxFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + + def _testSendMaxFrame(self): + self.cf = self.build_can_frame(0x00, b'\x07' * 8) + self.cli.send(self.cf) + + def testSendMultiFrames(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf1, cf) + + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf2, cf) + + def _testSendMultiFrames(self): + self.cf1 = self.build_can_frame(0x07, b'\x44\x33\x22\x11') + self.cli.send(self.cf1) + + self.cf2 = self.build_can_frame(0x12, b'\x99\x22\x33') + self.cli.send(self.cf2) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def _testBCM(self): + cf, addr = self.cli.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + can_id, can_dlc, data = self.dissect_can_frame(cf) + self.assertEqual(self.can_id, can_id) + self.assertEqual(self.data, data) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCM(self): + bcm = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) + self.addCleanup(bcm.close) + bcm.connect((self.interface,)) + self.can_id = 0x123 + self.data = bytes([0xc0, 0xff, 0xee]) + self.cf = self.build_can_frame(self.can_id, self.data) + opcode = socket.CAN_BCM_TX_SEND + flags = 0 + count = 0 + ival1_seconds = ival1_usec = ival2_seconds = ival2_usec = 0 + bcm_can_id = 0x0222 + nframes = 1 + assert len(self.cf) == 16 + header = struct.pack(self.bcm_cmd_msg_fmt, + opcode, + flags, + count, + ival1_seconds, + ival1_usec, + ival2_seconds, + ival2_usec, + bcm_can_id, + nframes, + ) + header_plus_frame = header + self.cf + bytes_sent = bcm.send(header_plus_frame) + self.assertEqual(bytes_sent, len(header_plus_frame)) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +class BasicRDSTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_RDS + socket.PF_RDS + + def testCreateSocket(self): + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + pass + + def testSocketBufferSize(self): + bufsize = 16384 + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, bufsize) + s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, bufsize) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class RDSTest(ThreadedRDSSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedRDSSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + super().setUp() + self.evt = threading.Event() + + def testSendAndRecv(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + self.assertEqual(self.cli_addr, addr) + + def _testSendAndRecv(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testPeek(self): + data, addr = self.serv.recvfrom(self.bufsize, socket.MSG_PEEK) + self.assertEqual(self.data, data) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testPeek(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + @requireAttrs(socket.socket, 'recvmsg') + def testSendAndRecvMsg(self): + data, ancdata, msg_flags, addr = self.serv.recvmsg(self.bufsize) + self.assertEqual(self.data, data) + + @requireAttrs(socket.socket, 'sendmsg') + def _testSendAndRecvMsg(self): + self.data = b'hello ' * 10 + self.cli.sendmsg([self.data], (), 0, (HOST, self.port)) + + def testSendAndRecvMulti(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data1, data) + + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data2, data) + + def _testSendAndRecvMulti(self): + self.data1 = b'bacon' + self.cli.sendto(self.data1, 0, (HOST, self.port)) + + self.data2 = b'egg' + self.cli.sendto(self.data2, 0, (HOST, self.port)) + + def testSelect(self): + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testSelect(self): + self.data = b'select' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testCongestion(self): + # wait until the sender is done + self.evt.wait() + + def _testCongestion(self): + # test the behavior in case of congestion + self.data = b'fill' + self.cli.setblocking(False) + try: + # try to lower the receiver's socket buffer size + self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384) + except OSError: + pass + with self.assertRaises(OSError) as cm: + try: + # fill the receiver's socket buffer + while True: + self.cli.sendto(self.data, 0, (HOST, self.port)) + finally: + # signal the receiver we're done + self.evt.set() + # sendto() should have failed with ENOBUFS + self.assertEqual(cm.exception.errno, errno.ENOBUFS) + # and we should have received a congestion notification through poll + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicTCPTest(SocketConnectedTest): + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecv(self): + # Testing large receive over TCP + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.serv_conn.send(MSG) + + def testOverFlowRecv(self): + # Testing receive in chunks over TCP + seg1 = self.cli_conn.recv(len(MSG) - 3) + seg2 = self.cli_conn.recv(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecv(self): + self.serv_conn.send(MSG) + + def testRecvFrom(self): + # Testing large recvfrom() over TCP + msg, addr = self.cli_conn.recvfrom(1024) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.serv_conn.send(MSG) + + def testOverFlowRecvFrom(self): + # Testing recvfrom() in chunks over TCP + seg1, addr = self.cli_conn.recvfrom(len(MSG)-3) + seg2, addr = self.cli_conn.recvfrom(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecvFrom(self): + self.serv_conn.send(MSG) + + def testSendAll(self): + # Testing sendall() with a 2048 byte string over TCP + msg = b'' + while 1: + read = self.cli_conn.recv(1024) + if not read: + break + msg += read + self.assertEqual(msg, b'f' * 2048) + + def _testSendAll(self): + big_chunk = b'f' * 2048 + self.serv_conn.sendall(big_chunk) + + def testFromFd(self): + # Testing fromfd() + fd = self.cli_conn.fileno() + sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + self.assertIsInstance(sock, socket.socket) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testFromFd(self): + self.serv_conn.send(MSG) + + def testDup(self): + # Testing dup() + sock = self.cli_conn.dup() + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDup(self): + self.serv_conn.send(MSG) + + def testShutdown(self): + # Testing shutdown() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + # wait for _testShutdown to finish: on OS X, when the server + # closes the connection the client also becomes disconnected, + # and the client's shutdown call will fail. (Issue #4397.) + self.done.wait() + + def _testShutdown(self): + self.serv_conn.send(MSG) + self.serv_conn.shutdown(2) + + testShutdown_overflow = support.cpython_only(testShutdown) + + @support.cpython_only + def _testShutdown_overflow(self): + import _testcapi + self.serv_conn.send(MSG) + # Issue 15989 + self.assertRaises(OverflowError, self.serv_conn.shutdown, + _testcapi.INT_MAX + 1) + self.assertRaises(OverflowError, self.serv_conn.shutdown, + 2 + (_testcapi.UINT_MAX + 1)) + self.serv_conn.shutdown(2) + + def testDetach(self): + # Testing detach() + fileno = self.cli_conn.fileno() + f = self.cli_conn.detach() + self.assertEqual(f, fileno) + # cli_conn cannot be used anymore... + self.assertTrue(self.cli_conn._closed) + self.assertRaises(OSError, self.cli_conn.recv, 1024) + self.cli_conn.close() + # ...but we can create another socket using the (still open) + # file descriptor + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=f) + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDetach(self): + self.serv_conn.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicUDPTest(ThreadedUDPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedUDPSocketTest.__init__(self, methodName=methodName) + + def testSendtoAndRecv(self): + # Testing sendto() and Recv() over UDP + msg = self.serv.recv(len(MSG)) + self.assertEqual(msg, MSG) + + def _testSendtoAndRecv(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFrom(self): + # Testing recvfrom() over UDP + msg, addr = self.serv.recvfrom(len(MSG)) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFromNegative(self): + # Negative lengths passed to recvfrom should give ValueError. + self.assertRaises(ValueError, self.serv.recvfrom, -1) + + def _testRecvFromNegative(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + +# Tests for the sendmsg()/recvmsg() interface. Where possible, the +# same test code is used with different families and types of socket +# (e.g. stream, datagram), and tests using recvmsg() are repeated +# using recvmsg_into(). +# +# The generic test classes such as SendmsgTests and +# RecvmsgGenericTests inherit from SendrecvmsgBase and expect to be +# supplied with sockets cli_sock and serv_sock representing the +# client's and the server's end of the connection respectively, and +# attributes cli_addr and serv_addr holding their (numeric where +# appropriate) addresses. +# +# The final concrete test classes combine these with subclasses of +# SocketTestBase which set up client and server sockets of a specific +# type, and with subclasses of SendrecvmsgBase such as +# SendrecvmsgDgramBase and SendrecvmsgConnectedBase which map these +# sockets to cli_sock and serv_sock and override the methods and +# attributes of SendrecvmsgBase to fill in destination addresses if +# needed when sending, check for specific flags in msg_flags, etc. +# +# RecvmsgIntoMixin provides a version of doRecvmsg() implemented using +# recvmsg_into(). + +# XXX: like the other datagram (UDP) tests in this module, the code +# here assumes that datagram delivery on the local machine will be +# reliable. + +class SendrecvmsgBase(ThreadSafeCleanupTestCase): + # Base class for sendmsg()/recvmsg() tests. + + # Time in seconds to wait before considering a test failed, or + # None for no timeout. Not all tests actually set a timeout. + fail_timeout = 3.0 + + def setUp(self): + self.misc_event = threading.Event() + super().setUp() + + def sendToServer(self, msg): + # Send msg to the server. + return self.cli_sock.send(msg) + + # Tuple of alternative default arguments for sendmsg() when called + # via sendmsgToServer() (e.g. to include a destination address). + sendmsg_to_server_defaults = () + + def sendmsgToServer(self, *args): + # Call sendmsg() on self.cli_sock with the given arguments, + # filling in any arguments which are not supplied with the + # corresponding items of self.sendmsg_to_server_defaults, if + # any. + return self.cli_sock.sendmsg( + *(args + self.sendmsg_to_server_defaults[len(args):])) + + def doRecvmsg(self, sock, bufsize, *args): + # Call recvmsg() on sock with given arguments and return its + # result. Should be used for tests which can use either + # recvmsg() or recvmsg_into() - RecvmsgIntoMixin overrides + # this method with one which emulates it using recvmsg_into(), + # thus allowing the same test to be used for both methods. + result = sock.recvmsg(bufsize, *args) + self.registerRecvmsgResult(result) + return result + + def registerRecvmsgResult(self, result): + # Called by doRecvmsg() with the return value of recvmsg() or + # recvmsg_into(). Can be overridden to arrange cleanup based + # on the returned ancillary data, for instance. + pass + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer. + self.assertEqual(addr1, addr2) + + # Flags that are normally unset in msg_flags + msg_flags_common_unset = 0 + for name in ("MSG_CTRUNC", "MSG_OOB"): + msg_flags_common_unset |= getattr(socket, name, 0) + + # Flags that are normally set + msg_flags_common_set = 0 + + # Flags set when a complete record has been received (e.g. MSG_EOR + # for SCTP) + msg_flags_eor_indicator = 0 + + # Flags set when a complete record has not been received + # (e.g. MSG_TRUNC for datagram sockets) + msg_flags_non_eor_indicator = 0 + + def checkFlags(self, flags, eor=None, checkset=0, checkunset=0, ignore=0): + # Method to check the value of msg_flags returned by recvmsg[_into](). + # + # Checks that all bits in msg_flags_common_set attribute are + # set in "flags" and all bits in msg_flags_common_unset are + # unset. + # + # The "eor" argument specifies whether the flags should + # indicate that a full record (or datagram) has been received. + # If "eor" is None, no checks are done; otherwise, checks + # that: + # + # * if "eor" is true, all bits in msg_flags_eor_indicator are + # set and all bits in msg_flags_non_eor_indicator are unset + # + # * if "eor" is false, all bits in msg_flags_non_eor_indicator + # are set and all bits in msg_flags_eor_indicator are unset + # + # If "checkset" and/or "checkunset" are supplied, they require + # the given bits to be set or unset respectively, overriding + # what the attributes require for those bits. + # + # If any bits are set in "ignore", they will not be checked, + # regardless of the other inputs. + # + # Will raise Exception if the inputs require a bit to be both + # set and unset, and it is not ignored. + + defaultset = self.msg_flags_common_set + defaultunset = self.msg_flags_common_unset + + if eor: + defaultset |= self.msg_flags_eor_indicator + defaultunset |= self.msg_flags_non_eor_indicator + elif eor is not None: + defaultset |= self.msg_flags_non_eor_indicator + defaultunset |= self.msg_flags_eor_indicator + + # Function arguments override defaults + defaultset &= ~checkunset + defaultunset &= ~checkset + + # Merge arguments with remaining defaults, and check for conflicts + checkset |= defaultset + checkunset |= defaultunset + inboth = checkset & checkunset & ~ignore + if inboth: + raise Exception("contradictory set, unset requirements for flags " + "{0:#x}".format(inboth)) + + # Compare with given msg_flags value + mask = (checkset | checkunset) & ~ignore + self.assertEqual(flags & mask, checkset & mask) + + +class RecvmsgIntoMixin(SendrecvmsgBase): + # Mixin to implement doRecvmsg() using recvmsg_into(). + + def doRecvmsg(self, sock, bufsize, *args): + buf = bytearray(bufsize) + result = sock.recvmsg_into([buf], *args) + self.registerRecvmsgResult(result) + self.assertGreaterEqual(result[0], 0) + self.assertLessEqual(result[0], bufsize) + return (bytes(buf[:result[0]]),) + result[1:] + + +class SendrecvmsgDgramFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for datagram sockets. + + @property + def msg_flags_non_eor_indicator(self): + return super().msg_flags_non_eor_indicator | socket.MSG_TRUNC + + +class SendrecvmsgSCTPFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for SCTP sockets. + + @property + def msg_flags_eor_indicator(self): + return super().msg_flags_eor_indicator | socket.MSG_EOR + + +class SendrecvmsgConnectionlessBase(SendrecvmsgBase): + # Base class for tests on connectionless-mode sockets. Users must + # supply sockets on attributes cli and serv to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.serv + + @property + def cli_sock(self): + return self.cli + + @property + def sendmsg_to_server_defaults(self): + return ([], [], 0, self.serv_addr) + + def sendToServer(self, msg): + return self.cli_sock.sendto(msg, self.serv_addr) + + +class SendrecvmsgConnectedBase(SendrecvmsgBase): + # Base class for tests on connected sockets. Users must supply + # sockets on attributes serv_conn and cli_conn (representing the + # connections *to* the server and the client), to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.cli_conn + + @property + def cli_sock(self): + return self.serv_conn + + def checkRecvmsgAddress(self, addr1, addr2): + # Address is currently "unspecified" for a connected socket, + # so we don't examine it + pass + + +class SendrecvmsgServerTimeoutBase(SendrecvmsgBase): + # Base class to set a timeout on server's socket. + + def setUp(self): + super().setUp() + self.serv_sock.settimeout(self.fail_timeout) + + +class SendmsgTests(SendrecvmsgServerTimeoutBase): + # Tests for sendmsg() which can use any socket type and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsg(self): + # Send a simple message with sendmsg(). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG]), len(MSG)) + + def testSendmsgDataGenerator(self): + # Send from buffer obtained from a generator (not a sequence). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgDataGenerator(self): + self.assertEqual(self.sendmsgToServer((o for o in [MSG])), + len(MSG)) + + def testSendmsgAncillaryGenerator(self): + # Gather (empty) ancillary data from a generator. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgAncillaryGenerator(self): + self.assertEqual(self.sendmsgToServer([MSG], (o for o in [])), + len(MSG)) + + def testSendmsgArray(self): + # Send data from an array instead of the usual bytes object. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgArray(self): + self.assertEqual(self.sendmsgToServer([array.array("B", MSG)]), + len(MSG)) + + def testSendmsgGather(self): + # Send message data from more than one buffer (gather write). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgGather(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + def testSendmsgBadArgs(self): + # Check that sendmsg() rejects invalid arguments. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadArgs(self): + self.assertRaises(TypeError, self.cli_sock.sendmsg) + self.assertRaises(TypeError, self.sendmsgToServer, + b"not in an iterable") + self.assertRaises(TypeError, self.sendmsgToServer, + object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG, object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], 0, object()) + self.sendToServer(b"done") + + def testSendmsgBadCmsg(self): + # Check that invalid ancillary data items are rejected. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(object(), 0, b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, object(), b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, object())]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0)]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b"data", 42)]) + self.sendToServer(b"done") + + @requireAttrs(socket, "CMSG_SPACE") + def testSendmsgBadMultiCmsg(self): + # Check that invalid ancillary data items are rejected when + # more than one item is present. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + @testSendmsgBadMultiCmsg.client_skip + def _testSendmsgBadMultiCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [0, 0, b""]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b""), object()]) + self.sendToServer(b"done") + + def testSendmsgExcessCmsgReject(self): + # Check that sendmsg() rejects excess ancillary data items + # when the number that can be sent is limited. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgExcessCmsgReject(self): + if not hasattr(socket, "CMSG_SPACE"): + # Can only send one item + with self.assertRaises(OSError) as cm: + self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")]) + self.assertIsNone(cm.exception.errno) + self.sendToServer(b"done") + + def testSendmsgAfterClose(self): + # Check that sendmsg() fails on a closed socket. + pass + + def _testSendmsgAfterClose(self): + self.cli_sock.close() + self.assertRaises(OSError, self.sendmsgToServer, [MSG]) + + +class SendmsgStreamTests(SendmsgTests): + # Tests for sendmsg() which require a stream socket and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsgExplicitNoneAddr(self): + # Check that peer address can be specified as None. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgExplicitNoneAddr(self): + self.assertEqual(self.sendmsgToServer([MSG], [], 0, None), len(MSG)) + + def testSendmsgTimeout(self): + # Check that timeout works with sendmsg(). + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + def _testSendmsgTimeout(self): + try: + self.cli_sock.settimeout(0.03) + with self.assertRaises(socket.timeout): + while True: + self.sendmsgToServer([b"a"*512]) + finally: + self.misc_event.set() + + # XXX: would be nice to have more tests for sendmsg flags argument. + + # Linux supports MSG_DONTWAIT when sending, but in general, it + # only works when receiving. Could add other platforms if they + # support it too. + @skipWithClientIf(sys.platform not in {"linux"}, + "MSG_DONTWAIT not known to work on this platform when " + "sending") + def testSendmsgDontWait(self): + # Check that MSG_DONTWAIT in flags causes non-blocking behaviour. + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @testSendmsgDontWait.client_skip + def _testSendmsgDontWait(self): + try: + with self.assertRaises(OSError) as cm: + while True: + self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT) + self.assertIn(cm.exception.errno, + (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + self.misc_event.set() + + +class SendmsgConnectionlessTests(SendmsgTests): + # Tests for sendmsg() which require a connectionless-mode + # (e.g. datagram) socket, and do not involve recvmsg() or + # recvmsg_into(). + + def testSendmsgNoDestAddr(self): + # Check that sendmsg() fails when no destination address is + # given for unconnected socket. + pass + + def _testSendmsgNoDestAddr(self): + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG]) + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG], [], 0, None) + + +class RecvmsgGenericTests(SendrecvmsgBase): + # Tests for recvmsg() which can also be emulated using + # recvmsg_into(), and can use any socket type. + + def testRecvmsg(self): + # Receive a simple message with recvmsg[_into](). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsg(self): + self.sendToServer(MSG) + + def testRecvmsgExplicitDefaults(self): + # Test recvmsg[_into]() with default arguments provided explicitly. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgExplicitDefaults(self): + self.sendToServer(MSG) + + def testRecvmsgShorter(self): + # Receive a message smaller than buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) + 42) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShorter(self): + self.sendToServer(MSG) + + # FreeBSD < 8 doesn't always set the MSG_TRUNC flag when a truncated + # datagram is received (issue #13001). + @support.requires_freebsd_version(8) + def testRecvmsgTrunc(self): + # Receive part of message, check for truncation indicators. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + @support.requires_freebsd_version(8) + def _testRecvmsgTrunc(self): + self.sendToServer(MSG) + + def testRecvmsgShortAncillaryBuf(self): + # Test ancillary data buffer too small to hold any ancillary data. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 1) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShortAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgLongAncillaryBuf(self): + # Test large ancillary data buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgLongAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgAfterClose(self): + # Check that recvmsg[_into]() fails on a closed socket. + self.serv_sock.close() + self.assertRaises(OSError, self.doRecvmsg, self.serv_sock, 1024) + + def _testRecvmsgAfterClose(self): + pass + + def testRecvmsgTimeout(self): + # Check that timeout works. + try: + self.serv_sock.settimeout(0.03) + self.assertRaises(socket.timeout, + self.doRecvmsg, self.serv_sock, len(MSG)) + finally: + self.misc_event.set() + + def _testRecvmsgTimeout(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @requireAttrs(socket, "MSG_PEEK") + def testRecvmsgPeek(self): + # Check that MSG_PEEK in flags enables examination of pending + # data without consuming it. + + # Receive part of data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3, 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + # Ignoring MSG_TRUNC here (so this test is the same for stream + # and datagram sockets). Some wording in POSIX seems to + # suggest that it needn't be set when peeking, but that may + # just be a slip. + self.checkFlags(flags, eor=False, + ignore=getattr(socket, "MSG_TRUNC", 0)) + + # Receive all data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + # Check that the same data can still be received normally. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgPeek.client_skip + def _testRecvmsgPeek(self): + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + def testRecvmsgFromSendmsg(self): + # Test receiving with recvmsg[_into]() when message is sent + # using sendmsg(). + self.serv_sock.settimeout(self.fail_timeout) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgFromSendmsg.client_skip + def _testRecvmsgFromSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + +class RecvmsgGenericStreamTests(RecvmsgGenericTests): + # Tests which require a stream socket and can use either recvmsg() + # or recvmsg_into(). + + def testRecvmsgEOF(self): + # Receive end-of-stream indicator (b"", peer socket closed). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.assertEqual(msg, b"") + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=None) # Might not have end-of-record marker + + def _testRecvmsgEOF(self): + self.cli_sock.close() + + def testRecvmsgOverflow(self): + # Receive a message in more than one chunk. + seg1, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + seg2, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testRecvmsgOverflow(self): + self.sendToServer(MSG) + + +class RecvmsgTests(RecvmsgGenericTests): + # Tests for recvmsg() which can use any socket type. + + def testRecvmsgBadArgs(self): + # Check that recvmsg() rejects invalid arguments. + self.assertRaises(TypeError, self.serv_sock.recvmsg) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + -1, 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + len(MSG), -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + [bytearray(10)], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + object(), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), 0, object()) + + msg, ancdata, flags, addr = self.serv_sock.recvmsg(len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgBadArgs(self): + self.sendToServer(MSG) + + +class RecvmsgIntoTests(RecvmsgIntoMixin, RecvmsgGenericTests): + # Tests for recvmsg_into() which can use any socket type. + + def testRecvmsgIntoBadArgs(self): + # Check that recvmsg_into() rejects invalid arguments. + buf = bytearray(len(MSG)) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + len(MSG), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + buf, 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [object()], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [b"I'm not writable"], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf, object()], 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg_into, + [buf], -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], 0, object()) + + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf], 0, 0) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoBadArgs(self): + self.sendToServer(MSG) + + def testRecvmsgIntoGenerator(self): + # Receive into buffer obtained from a generator (not a sequence). + buf = bytearray(len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + (o for o in [buf])) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoGenerator(self): + self.sendToServer(MSG) + + def testRecvmsgIntoArray(self): + # Receive into an array rather than the usual bytearray. + buf = array.array("B", [0] * len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf]) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf.tobytes(), MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoArray(self): + self.sendToServer(MSG) + + def testRecvmsgIntoScatter(self): + # Receive into multiple buffers (scatter write). + b1 = bytearray(b"----") + b2 = bytearray(b"0123456789") + b3 = bytearray(b"--------------") + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + [b1, memoryview(b2)[2:9], b3]) + self.assertEqual(nbytes, len(b"Mary had a little lamb")) + self.assertEqual(b1, bytearray(b"Mary")) + self.assertEqual(b2, bytearray(b"01 had a 9")) + self.assertEqual(b3, bytearray(b"little lamb---")) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoScatter(self): + self.sendToServer(b"Mary had a little lamb") + + +class CmsgMacroTests(unittest.TestCase): + # Test the functions CMSG_LEN() and CMSG_SPACE(). Tests + # assumptions used by sendmsg() and recvmsg[_into](), which share + # code with these functions. + + # Match the definition in socketmodule.c + try: + import _testcapi + except ImportError: + socklen_t_limit = 0x7fffffff + else: + socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) + + @requireAttrs(socket, "CMSG_LEN") + def testCMSG_LEN(self): + # Test CMSG_LEN() with various valid and invalid values, + # checking the assumptions used by recvmsg() and sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_LEN(0) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(socket.CMSG_LEN(0), array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_LEN(n) + # This is how recvmsg() calculates the data size + self.assertEqual(ret - socket.CMSG_LEN(0), n) + self.assertLessEqual(ret, self.socklen_t_limit) + + self.assertRaises(OverflowError, socket.CMSG_LEN, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_LEN, toobig) + self.assertRaises(OverflowError, socket.CMSG_LEN, sys.maxsize) + + @requireAttrs(socket, "CMSG_SPACE") + def testCMSG_SPACE(self): + # Test CMSG_SPACE() with various valid and invalid values, + # checking the assumptions used by sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + last = socket.CMSG_SPACE(0) + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(last, array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_SPACE(n) + self.assertGreaterEqual(ret, last) + self.assertGreaterEqual(ret, socket.CMSG_LEN(n)) + self.assertGreaterEqual(ret, n + socket.CMSG_LEN(0)) + self.assertLessEqual(ret, self.socklen_t_limit) + last = ret + + self.assertRaises(OverflowError, socket.CMSG_SPACE, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_SPACE, toobig) + self.assertRaises(OverflowError, socket.CMSG_SPACE, sys.maxsize) + + +class SCMRightsTest(SendrecvmsgServerTimeoutBase): + # Tests for file descriptor passing on Unix-domain sockets. + + # Invalid file descriptor value that's unlikely to evaluate to a + # real FD even if one of its bytes is replaced with a different + # value (which shouldn't actually happen). + badfd = -0x5555 + + def newFDs(self, n): + # Return a list of n file descriptors for newly-created files + # containing their list indices as ASCII numbers. + fds = [] + for i in range(n): + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + self.addCleanup(os.close, fd) + os.write(fd, str(i).encode()) + fds.append(fd) + return fds + + def checkFDs(self, fds): + # Check that the file descriptors in the given list contain + # their correct list indices as ASCII numbers. + for n, fd in enumerate(fds): + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(os.read(fd, 1024), str(n).encode()) + + def registerRecvmsgResult(self, result): + self.addCleanup(self.closeRecvmsgFDs, result) + + def closeRecvmsgFDs(self, recvmsg_result): + # Close all file descriptors specified in the ancillary data + # of the given return value from recvmsg() or recvmsg_into(). + for cmsg_level, cmsg_type, cmsg_data in recvmsg_result[1]: + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + for fd in fds: + os.close(fd) + + def createAndSendFDs(self, n): + # Send n new file descriptors created by newFDs() to the + # server, with the constant MSG as the non-ancillary data. + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(n)))]), + len(MSG)) + + def checkRecvmsgFDs(self, numfds, result, maxcmsgs=1, ignoreflags=0): + # Check that constant MSG was received with numfds file + # descriptors in a maximum of maxcmsgs control messages (which + # must contain only complete integers). By default, check + # that MSG_CTRUNC is unset, but ignore any flags in + # ignoreflags. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertIsInstance(ancdata, list) + self.assertLessEqual(len(ancdata), maxcmsgs) + fds = array.array("i") + for item in ancdata: + self.assertIsInstance(item, tuple) + cmsg_level, cmsg_type, cmsg_data = item + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data) % SIZEOF_INT, 0) + fds.frombytes(cmsg_data) + + self.assertEqual(len(fds), numfds) + self.checkFDs(fds) + + def testFDPassSimple(self): + # Pass a single FD (array read from bytes object). + self.checkRecvmsgFDs(1, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testFDPassSimple(self): + self.assertEqual( + self.sendmsgToServer( + [MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(1)).tobytes())]), + len(MSG)) + + def testMultipleFDPass(self): + # Pass multiple FDs in a single array. + self.checkRecvmsgFDs(4, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testMultipleFDPass(self): + self.createAndSendFDs(4) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassCMSG_SPACE(self): + # Test using CMSG_SPACE() to calculate ancillary buffer size. + self.checkRecvmsgFDs( + 4, self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(4 * SIZEOF_INT))) + + @testFDPassCMSG_SPACE.client_skip + def _testFDPassCMSG_SPACE(self): + self.createAndSendFDs(4) + + def testFDPassCMSG_LEN(self): + # Test using CMSG_LEN() to calculate ancillary buffer size. + self.checkRecvmsgFDs(1, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(4 * SIZEOF_INT)), + # RFC 3542 says implementations may set + # MSG_CTRUNC if there isn't enough space + # for trailing padding. + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassCMSG_LEN(self): + self.createAndSendFDs(1) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparate(self): + # Pass two FDs in two separate arrays. Arrays may be combined + # into a single control message by the OS. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), 10240), + maxcmsgs=2) + + @testFDPassSeparate.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparate(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparateMinSpace(self): + # Pass two FDs in two separate arrays, receiving them into the + # minimum space for two arrays. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(SIZEOF_INT)), + maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) + + @testFDPassSeparateMinSpace.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparateMinSpace(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + def sendAncillaryIfPossible(self, msg, ancdata): + # Try to send msg and ancdata to server, but if the system + # call fails, just send msg with no ancillary data. + try: + nbytes = self.sendmsgToServer([msg], ancdata) + except OSError as e: + # Check that it was the system call that failed + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer([msg]) + self.assertEqual(nbytes, len(msg)) + + @unittest.skipIf(sys.platform == "darwin", "see issue #24725") + def testFDPassEmpty(self): + # Try to pass an empty FD array. Can receive either no array + # or an empty array. + self.checkRecvmsgFDs(0, self.doRecvmsg(self.serv_sock, + len(MSG), 10240), + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassEmpty(self): + self.sendAncillaryIfPossible(MSG, [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + b"")]) + + def testFDPassPartialInt(self): + # Try to pass a truncated FD array. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 1) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + def _testFDPassPartialInt(self): + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [self.badfd]).tobytes()[:-1])]) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassPartialIntInMiddle(self): + # Try to pass two FD arrays, the first of which is truncated. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 2) + fds = array.array("i") + # Arrays may have been combined in a single control message + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.assertLessEqual(len(fds), 2) + self.checkFDs(fds) + + @testFDPassPartialIntInMiddle.client_skip + def _testFDPassPartialIntInMiddle(self): + fd0, fd1 = self.newFDs(2) + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0, self.badfd]).tobytes()[:-1]), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]) + + def checkTruncatedHeader(self, result, ignoreflags=0): + # Check that no ancillary data items are returned when data is + # truncated inside the cmsghdr structure. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no buffer size + # is specified. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG)), + # BSD seems to set MSG_CTRUNC only + # if an item has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTruncNoBufSize(self): + self.createAndSendFDs(1) + + def testCmsgTrunc0(self): + # Check that no ancillary data is received when buffer size is 0. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTrunc0(self): + self.createAndSendFDs(1) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + def testCmsgTrunc1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + + def _testCmsgTrunc1(self): + self.createAndSendFDs(1) + + def testCmsgTrunc2Int(self): + # The cmsghdr structure has at least three members, two of + # which are ints, so we still shouldn't see any ancillary + # data. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + SIZEOF_INT * 2)) + + def _testCmsgTrunc2Int(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Minus1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(0) - 1)) + + def _testCmsgTruncLen0Minus1(self): + self.createAndSendFDs(1) + + # The following tests try to truncate the control message in the + # middle of the FD array. + + def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): + # Check that file descriptor data is truncated to between + # mindata and maxdata bytes when received with buffer size + # ancbuf, and that any complete file descriptor numbers are + # valid. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + if mindata == 0 and ancdata == []: + return + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertGreaterEqual(len(cmsg_data), mindata) + self.assertLessEqual(len(cmsg_data), maxdata) + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.checkFDs(fds) + + def testCmsgTruncLen0(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + + def _testCmsgTruncLen0(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Plus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + + def _testCmsgTruncLen0Plus1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), + maxdata=SIZEOF_INT) + + def _testCmsgTruncLen1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen2Minus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, + maxdata=(2 * SIZEOF_INT) - 1) + + def _testCmsgTruncLen2Minus1(self): + self.createAndSendFDs(2) + + +class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase): + # Test sendmsg() and recvmsg[_into]() using the ancillary data + # features of the RFC 3542 Advanced Sockets API for IPv6. + # Currently we can only handle certain data items (e.g. traffic + # class, hop limit, MTU discovery and fragmentation settings) + # without resorting to unportable means such as the struct module, + # but the tests here are aimed at testing the ancillary data + # handling in sendmsg() and recvmsg() rather than the IPv6 API + # itself. + + # Test value to use when setting hop limit of packet + hop_limit = 2 + + # Test value to use when setting traffic class of packet. + # -1 means "use kernel default". + traffic_class = -1 + + def ancillaryMapping(self, ancdata): + # Given ancillary data list ancdata, return a mapping from + # pairs (cmsg_level, cmsg_type) to corresponding cmsg_data. + # Check that no (level, type) pair appears more than once. + d = {} + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertNotIn((cmsg_level, cmsg_type), d) + d[(cmsg_level, cmsg_type)] = cmsg_data + return d + + def checkHopLimit(self, ancbufsize, maxhop=255, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space. Check that data is MSG, ancillary data is not + # truncated (but ignore any flags in ignoreflags), and hop + # limit is between 0 and maxhop inclusive. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + self.assertIsInstance(ancdata[0], tuple) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimit(self): + # Test receiving the packet hop limit as ancillary data. + self.checkHopLimit(ancbufsize=10240) + + @testRecvHopLimit.client_skip + def _testRecvHopLimit(self): + # Need to wait until server has asked to receive ancillary + # data, as implementations are not required to buffer it + # otherwise. + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimitCMSG_SPACE(self): + # Test receiving hop limit, using CMSG_SPACE to calculate buffer size. + self.checkHopLimit(ancbufsize=socket.CMSG_SPACE(SIZEOF_INT)) + + @testRecvHopLimitCMSG_SPACE.client_skip + def _testRecvHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Could test receiving into buffer sized using CMSG_LEN, but RFC + # 3542 says portable applications must provide space for trailing + # padding. Implementations may set MSG_CTRUNC if there isn't + # enough space for the padding. + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSetHopLimit(self): + # Test setting hop limit on outgoing packet and receiving it + # at the other end. + self.checkHopLimit(ancbufsize=10240, maxhop=self.hop_limit) + + @testSetHopLimit.client_skip + def _testSetHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + def checkTrafficClassAndHopLimit(self, ancbufsize, maxhop=255, + ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space. Check that data is MSG, ancillary + # data is not truncated (but ignore any flags in ignoreflags), + # and traffic class and hop limit are in range (hop limit no + # more than maxhop). + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + self.assertEqual(len(ancdata), 2) + ancmap = self.ancillaryMapping(ancdata) + + tcdata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_TCLASS)] + self.assertEqual(len(tcdata), SIZEOF_INT) + a = array.array("i") + a.frombytes(tcdata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + hldata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT)] + self.assertEqual(len(hldata), SIZEOF_INT) + a = array.array("i") + a.frombytes(hldata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimit(self): + # Test receiving traffic class and hop limit as ancillary data. + self.checkTrafficClassAndHopLimit(ancbufsize=10240) + + @testRecvTrafficClassAndHopLimit.client_skip + def _testRecvTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + # Test receiving traffic class and hop limit, using + # CMSG_SPACE() to calculate buffer size. + self.checkTrafficClassAndHopLimit( + ancbufsize=socket.CMSG_SPACE(SIZEOF_INT) * 2) + + @testRecvTrafficClassAndHopLimitCMSG_SPACE.client_skip + def _testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSetTrafficClassAndHopLimit(self): + # Test setting traffic class and hop limit on outgoing packet, + # and receiving them at the other end. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testSetTrafficClassAndHopLimit.client_skip + def _testSetTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testOddCmsgSize(self): + # Try to send ancillary data with first item one byte too + # long. Fall back to sending with correct size if this fails, + # and check that second item was handled correctly. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testOddCmsgSize.client_skip + def _testOddCmsgSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + try: + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class]).tobytes() + b"\x00"), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + except OSError as e: + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + self.assertEqual(nbytes, len(MSG)) + + # Tests for proper handling of truncated ancillary data + + def checkHopLimitTruncatedHeader(self, ancbufsize, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space, which should be too small to contain the ancillary + # data header (if ancbufsize is None, pass no second argument + # to recvmsg()). Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and no ancillary data is + # returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + args = () if ancbufsize is None else (ancbufsize,) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), *args) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no ancillary + # buffer size is provided. + self.checkHopLimitTruncatedHeader(ancbufsize=None, + # BSD seems to set + # MSG_CTRUNC only if an item + # has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + @testCmsgTruncNoBufSize.client_skip + def _testCmsgTruncNoBufSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc0(self): + # Check that no ancillary data is received when ancillary + # buffer size is zero. + self.checkHopLimitTruncatedHeader(ancbufsize=0, + ignoreflags=socket.MSG_CTRUNC) + + @testSingleCmsgTrunc0.client_skip + def _testSingleCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=1) + + @testSingleCmsgTrunc1.client_skip + def _testSingleCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc2Int(self): + self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) + + @testSingleCmsgTrunc2Int.client_skip + def _testSingleCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncLen0Minus1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) + + @testSingleCmsgTruncLen0Minus1.client_skip + def _testSingleCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncInData(self): + # Test truncation of a control message inside its associated + # data. The message may be returned with its data truncated, + # or not returned at all. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + self.assertLessEqual(len(ancdata), 1) + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + @testSingleCmsgTruncInData.client_skip + def _testSingleCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + def checkTruncatedSecondHeader(self, ancbufsize, ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space, which should be large enough to + # contain the first item, but too small to contain the header + # of the second. Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and only one ancillary + # data item is returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertIn(cmsg_type, {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT}) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + # Try the above test with various buffer sizes. + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc0(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT), + ignoreflags=socket.MSG_CTRUNC) + + @testSecondCmsgTrunc0.client_skip + def _testSecondCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) + + @testSecondCmsgTrunc1.client_skip + def _testSecondCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc2Int(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + 2 * SIZEOF_INT) + + @testSecondCmsgTrunc2Int.client_skip + def _testSecondCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTruncLen0Minus1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(0) - 1) + + @testSecondCmsgTruncLen0Minus1.client_skip + def _testSecondCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecomdCmsgTruncInData(self): + # Test truncation of the second of two control messages inside + # its associated data. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + cmsg_types = {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT} + + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + self.assertEqual(ancdata, []) + + @testSecomdCmsgTruncInData.client_skip + def _testSecomdCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + +# Derive concrete test classes for different socket types. + +class SendrecvmsgUDPTestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDPTest(SendmsgConnectionlessTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDPTest(RecvmsgTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDPTest(RecvmsgIntoTests, SendrecvmsgUDPTestBase): + pass + + +class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDP6TestBase): + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer, ignoring scope ID + self.assertEqual(addr1[:-1], addr2[:-1]) + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDP6Test(SendmsgConnectionlessTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDP6Test(RecvmsgTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDP6Test(RecvmsgIntoTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgRFC3542AncillaryUDP6Test(RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoRFC3542AncillaryUDP6Test(RecvmsgIntoMixin, + RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + + +class SendrecvmsgTCPTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, TCPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgTCPTest(SendmsgStreamTests, SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgTCPTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoTCPTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + + +class SendrecvmsgSCTPStreamTestBase(SendrecvmsgSCTPFlagsBase, + SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, SCTPStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgSCTPStreamTest(SendmsgStreamTests, SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCTPStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + +@requireAttrs(socket.socket, "recvmsg_into") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCTPStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgIntoSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + + +class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, UnixStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUnixStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUnixStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCMRightsStreamTest(SCMRightsTest, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg_into") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCMRightsStreamTest(RecvmsgIntoMixin, SCMRightsTest, + SendrecvmsgUnixStreamTestBase): + pass + + +# Test interrupting the interruptible send/receive methods with a +# signal when a timeout is set. These tests avoid having multiple +# threads alive during the test so that the OS cannot deliver the +# signal to the wrong one. + +class InterruptedTimeoutBase(unittest.TestCase): + # Base class for interrupted send/receive tests. Installs an + # empty handler for SIGALRM and removes it on teardown, along with + # any scheduled alarms. + + def setUp(self): + super().setUp() + orig_alrm_handler = signal.signal(signal.SIGALRM, + lambda signum, frame: 1 / 0) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(self.setAlarm, 0) + + # Timeout for socket operations + timeout = 4.0 + + # Provide setAlarm() method to schedule delivery of SIGALRM after + # given number of seconds, or cancel it if zero, and an + # appropriate time value to use. Use setitimer() if available. + if hasattr(signal, "setitimer"): + alarm_time = 0.05 + + def setAlarm(self, seconds): + signal.setitimer(signal.ITIMER_REAL, seconds) + else: + # Old systems may deliver the alarm up to one second early + alarm_time = 2 + + def setAlarm(self, seconds): + signal.alarm(seconds) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase): + # Test interrupting the recv*() methods with signals when a + # timeout is set. + + def setUp(self): + super().setUp() + self.serv.settimeout(self.timeout) + + def checkInterruptedRecv(self, func, *args, **kwargs): + # Check that func(*args, **kwargs) raises + # errno of EINTR when interrupted by a signal. + self.setAlarm(self.alarm_time) + with self.assertRaises(ZeroDivisionError) as cm: + func(*args, **kwargs) + + def testInterruptedRecvTimeout(self): + self.checkInterruptedRecv(self.serv.recv, 1024) + + def testInterruptedRecvIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recv_into, bytearray(1024)) + + def testInterruptedRecvfromTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom, 1024) + + def testInterruptedRecvfromIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom_into, bytearray(1024)) + + @requireAttrs(socket.socket, "recvmsg") + def testInterruptedRecvmsgTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg, 1024) + + @requireAttrs(socket.socket, "recvmsg_into") + def testInterruptedRecvmsgIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg_into, [bytearray(1024)]) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +@unittest.skipUnless(thread, 'Threading required for this test.') +class InterruptedSendTimeoutTest(InterruptedTimeoutBase, + ThreadSafeCleanupTestCase, + SocketListeningTestMixin, TCPTestBase): + # Test interrupting the interruptible send*() methods with signals + # when a timeout is set. + + def setUp(self): + super().setUp() + self.serv_conn = self.newSocket() + self.addCleanup(self.serv_conn.close) + # Use a thread to complete the connection, but wait for it to + # terminate before running the test, so that there is only one + # thread to accept the signal. + cli_thread = threading.Thread(target=self.doConnect) + cli_thread.start() + self.cli_conn, addr = self.serv.accept() + self.addCleanup(self.cli_conn.close) + cli_thread.join() + self.serv_conn.settimeout(self.timeout) + + def doConnect(self): + self.serv_conn.connect(self.serv_addr) + + def checkInterruptedSend(self, func, *args, **kwargs): + # Check that func(*args, **kwargs), run in a loop, raises + # OSError with an errno of EINTR when interrupted by a + # signal. + with self.assertRaises(ZeroDivisionError) as cm: + while True: + self.setAlarm(self.alarm_time) + func(*args, **kwargs) + + # Issue #12958: The following tests have problems on OS X prior to 10.7 + @support.requires_mac_ver(10, 7) + def testInterruptedSendTimeout(self): + self.checkInterruptedSend(self.serv_conn.send, b"a"*512) + + @support.requires_mac_ver(10, 7) + def testInterruptedSendtoTimeout(self): + # Passing an actual address here as Python's wrapper for + # sendto() doesn't allow passing a zero-length one; POSIX + # requires that the address is ignored since the socket is + # connection-mode, however. + self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512, + self.serv_addr) + + @support.requires_mac_ver(10, 7) + @requireAttrs(socket.socket, "sendmsg") + def testInterruptedSendmsgTimeout(self): + self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512]) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class TCPCloserTest(ThreadedTCPSocketTest): + + def testClose(self): + conn, addr = self.serv.accept() + conn.close() + + sd = self.cli + read, write, err = select.select([sd], [], [], 1.0) + self.assertEqual(read, [sd]) + self.assertEqual(sd.recv(1), b'') + + # Calling close() many times should be safe. + conn.close() + conn.close() + + def _testClose(self): + self.cli.connect((HOST, self.port)) + time.sleep(1.0) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BasicSocketPairTest(SocketPairTest): + + def __init__(self, methodName='runTest'): + SocketPairTest.__init__(self, methodName=methodName) + + def _check_defaults(self, sock): + self.assertIsInstance(sock, socket.socket) + if hasattr(socket, 'AF_UNIX'): + self.assertEqual(sock.family, socket.AF_UNIX) + else: + self.assertEqual(sock.family, socket.AF_INET) + self.assertEqual(sock.type, socket.SOCK_STREAM) + self.assertEqual(sock.proto, 0) + + def _testDefaults(self): + self._check_defaults(self.cli) + + def testDefaults(self): + self._check_defaults(self.serv) + + def testRecv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.cli.send(MSG) + + def testSend(self): + self.serv.send(MSG) + + def _testSend(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NonBlockingTCPTests(ThreadedTCPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def testSetBlocking(self): + # Testing whether set blocking works + self.serv.setblocking(True) + self.assertIsNone(self.serv.gettimeout()) + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error setting non-blocking mode.") + + def _testSetBlocking(self): + pass + + @support.cpython_only + def testSetBlocking_overflow(self): + # Issue 15989 + import _testcapi + if _testcapi.UINT_MAX >= _testcapi.ULONG_MAX: + self.skipTest('needs UINT_MAX < ULONG_MAX') + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.serv.setblocking(_testcapi.UINT_MAX + 1) + self.assertIsNone(self.serv.gettimeout()) + + _testSetBlocking_overflow = support.cpython_only(_testSetBlocking) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'test needs socket.SOCK_NONBLOCK') + @support.requires_linux_version(2, 6, 28) + def testInitNonBlocking(self): + # reinit server socket + self.serv.close() + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM | + socket.SOCK_NONBLOCK) + self.port = support.bind_port(self.serv) + self.serv.listen() + # actual testing + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error creating with non-blocking mode.") + + def _testInitNonBlocking(self): + pass + + def testInheritFlags(self): + # Issue #7995: when calling accept() on a listening socket with a + # timeout, the resulting socket should not be non-blocking. + self.serv.settimeout(10) + try: + conn, addr = self.serv.accept() + message = conn.recv(len(MSG)) + finally: + conn.close() + self.serv.settimeout(None) + + def _testInheritFlags(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + time.sleep(0.5) + self.cli.send(MSG) + + def testAccept(self): + # Testing non-blocking accept + self.serv.setblocking(0) + try: + conn, addr = self.serv.accept() + except OSError: + pass + else: + self.fail("Error trying to do non-blocking accept.") + read, write, err = select.select([self.serv], [], []) + if self.serv in read: + conn, addr = self.serv.accept() + self.assertIsNone(conn.gettimeout()) + conn.close() + else: + self.fail("Error trying to do accept after select.") + + def _testAccept(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + + def testConnect(self): + # Testing non-blocking connect + conn, addr = self.serv.accept() + conn.close() + + def _testConnect(self): + self.cli.settimeout(10) + self.cli.connect((HOST, self.port)) + + def testRecv(self): + # Testing non-blocking recv + conn, addr = self.serv.accept() + conn.setblocking(0) + try: + msg = conn.recv(len(MSG)) + except OSError: + pass + else: + self.fail("Error trying to do non-blocking recv.") + read, write, err = select.select([conn], [], []) + if conn in read: + msg = conn.recv(len(MSG)) + conn.close() + self.assertEqual(msg, MSG) + else: + self.fail("Error during select call to non-blocking socket.") + + def _testRecv(self): + self.cli.connect((HOST, self.port)) + time.sleep(0.1) + self.cli.send(MSG) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class FileObjectClassTestCase(SocketConnectedTest): + """Unit tests for the object returned by socket.makefile() + + self.read_file is the io object returned by makefile() on + the client connection. You can read from this file to + get output from the server. + + self.write_file is the io object returned by makefile() on the + server connection. You can write to this file to send output + to the client. + """ + + bufsize = -1 # Use default buffer size + encoding = 'utf-8' + errors = 'strict' + newline = None + + read_mode = 'rb' + read_msg = MSG + write_mode = 'wb' + write_msg = MSG + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def setUp(self): + self.evt1, self.evt2, self.serv_finished, self.cli_finished = [ + threading.Event() for i in range(4)] + SocketConnectedTest.setUp(self) + self.read_file = self.cli_conn.makefile( + self.read_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def tearDown(self): + self.serv_finished.set() + self.read_file.close() + self.assertTrue(self.read_file.closed) + self.read_file = None + SocketConnectedTest.tearDown(self) + + def clientSetUp(self): + SocketConnectedTest.clientSetUp(self) + self.write_file = self.serv_conn.makefile( + self.write_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def clientTearDown(self): + self.cli_finished.set() + self.write_file.close() + self.assertTrue(self.write_file.closed) + self.write_file = None + SocketConnectedTest.clientTearDown(self) + + def testReadAfterTimeout(self): + # Issue #7322: A file object must disallow further reads + # after a timeout has occurred. + self.cli_conn.settimeout(1) + self.read_file.read(3) + # First read raises a timeout + self.assertRaises(socket.timeout, self.read_file.read, 1) + # Second read is disallowed + with self.assertRaises(OSError) as ctx: + self.read_file.read(1) + self.assertIn("cannot read from timed out object", str(ctx.exception)) + + def _testReadAfterTimeout(self): + self.write_file.write(self.write_msg[0:3]) + self.write_file.flush() + self.serv_finished.wait() + + def testSmallRead(self): + # Performing small file read test + first_seg = self.read_file.read(len(self.read_msg)-3) + second_seg = self.read_file.read(3) + msg = first_seg + second_seg + self.assertEqual(msg, self.read_msg) + + def _testSmallRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testFullRead(self): + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testFullRead(self): + self.write_file.write(self.write_msg) + self.write_file.close() + + def testUnbufferedRead(self): + # Performing unbuffered file read test + buf = type(self.read_msg)() + while 1: + char = self.read_file.read(1) + if not char: + break + buf += char + self.assertEqual(buf, self.read_msg) + + def _testUnbufferedRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testReadline(self): + # Performing file readline test + line = self.read_file.readline() + self.assertEqual(line, self.read_msg) + + def _testReadline(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testCloseAfterMakefile(self): + # The file returned by makefile should keep the socket open. + self.cli_conn.close() + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testCloseAfterMakefile(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileAfterMakefileClose(self): + self.read_file.close() + msg = self.cli_conn.recv(len(MSG)) + if isinstance(self.read_msg, str): + msg = msg.decode() + self.assertEqual(msg, self.read_msg) + + def _testMakefileAfterMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testClosedAttr(self): + self.assertTrue(not self.read_file.closed) + + def _testClosedAttr(self): + self.assertTrue(not self.write_file.closed) + + def testAttributes(self): + self.assertEqual(self.read_file.mode, self.read_mode) + self.assertEqual(self.read_file.name, self.cli_conn.fileno()) + + def _testAttributes(self): + self.assertEqual(self.write_file.mode, self.write_mode) + self.assertEqual(self.write_file.name, self.serv_conn.fileno()) + + def testRealClose(self): + self.read_file.close() + self.assertRaises(ValueError, self.read_file.fileno) + self.cli_conn.close() + self.assertRaises(OSError, self.cli_conn.getsockname) + + def _testRealClose(self): + pass + + +class UnbufferedFileObjectClassTestCase(FileObjectClassTestCase): + + """Repeat the tests from FileObjectClassTestCase with bufsize==0. + + In this case (and in this case only), it should be possible to + create a file object, read a line from it, create another file + object, read another line from it, without loss of data in the + first file object's buffer. Note that http.client relies on this + when reading multiple requests from the same socket.""" + + bufsize = 0 # Use unbuffered mode + + def testUnbufferedReadline(self): + # Read a line, create a new file object, read another line with it + line = self.read_file.readline() # first line + self.assertEqual(line, b"A. " + self.write_msg) # first line + self.read_file = self.cli_conn.makefile('rb', 0) + line = self.read_file.readline() # second line + self.assertEqual(line, b"B. " + self.write_msg) # second line + + def _testUnbufferedReadline(self): + self.write_file.write(b"A. " + self.write_msg) + self.write_file.write(b"B. " + self.write_msg) + self.write_file.flush() + + def testMakefileClose(self): + # The file returned by makefile should keep the socket open... + self.cli_conn.close() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, self.read_msg) + # ...until the file is itself closed + self.read_file.close() + self.assertRaises(OSError, self.cli_conn.recv, 1024) + + def _testMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileCloseSocketDestroy(self): + refcount_before = sys.getrefcount(self.cli_conn) + self.read_file.close() + refcount_after = sys.getrefcount(self.cli_conn) + self.assertEqual(refcount_before - 1, refcount_after) + + def _testMakefileCloseSocketDestroy(self): + pass + + # Non-blocking ops + # NOTE: to set `read_file` as non-blocking, we must call + # `cli_conn.setblocking` and vice-versa (see setUp / clientSetUp). + + def testSmallReadNonBlocking(self): + self.cli_conn.setblocking(False) + self.assertEqual(self.read_file.readinto(bytearray(10)), None) + self.assertEqual(self.read_file.read(len(self.read_msg) - 3), None) + self.evt1.set() + self.evt2.wait(1.0) + first_seg = self.read_file.read(len(self.read_msg) - 3) + if first_seg is None: + # Data not arrived (can happen under Windows), wait a bit + time.sleep(0.5) + first_seg = self.read_file.read(len(self.read_msg) - 3) + buf = bytearray(10) + n = self.read_file.readinto(buf) + self.assertEqual(n, 3) + msg = first_seg + buf[:n] + self.assertEqual(msg, self.read_msg) + self.assertEqual(self.read_file.readinto(bytearray(16)), None) + self.assertEqual(self.read_file.read(1), None) + + def _testSmallReadNonBlocking(self): + self.evt1.wait(1.0) + self.write_file.write(self.write_msg) + self.write_file.flush() + self.evt2.set() + # Avoid cloding the socket before the server test has finished, + # otherwise system recv() will return 0 instead of EWOULDBLOCK. + self.serv_finished.wait(5.0) + + def testWriteNonBlocking(self): + self.cli_finished.wait(5.0) + # The client thread can't skip directly - the SkipTest exception + # would appear as a failure. + if self.serv_skipped: + self.skipTest(self.serv_skipped) + + def _testWriteNonBlocking(self): + self.serv_skipped = None + self.serv_conn.setblocking(False) + # Try to saturate the socket buffer pipe with repeated large writes. + BIG = b"x" * support.SOCK_MAX_SIZE + LIMIT = 10 + # The first write() succeeds since a chunk of data can be buffered + n = self.write_file.write(BIG) + self.assertGreater(n, 0) + for i in range(LIMIT): + n = self.write_file.write(BIG) + if n is None: + # Succeeded + break + self.assertGreater(n, 0) + else: + # Let us know that this test didn't manage to establish + # the expected conditions. This is not a failure in itself but, + # if it happens repeatedly, the test should be fixed. + self.serv_skipped = "failed to saturate the socket buffer" + + +class LineBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 1 # Default-buffered for reading; line-buffered for writing + + +class SmallBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 2 # Exercise the buffering code + + +class UnicodeReadFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'wb' + write_msg = MSG + newline = '' + + +class UnicodeWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'rb' + read_msg = MSG + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class UnicodeReadWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class NetworkConnectionTest(object): + """Prove network connection.""" + + def clientSetUp(self): + # We're inherited below by BasicTCPTest2, which also inherits + # BasicTCPTest, which defines self.port referenced below. + self.cli = socket.create_connection((HOST, self.port)) + self.serv_conn = self.cli + +class BasicTCPTest2(NetworkConnectionTest, BasicTCPTest): + """Tests that NetworkConnection does not break existing TCP functionality. + """ + +class NetworkConnectionNoServer(unittest.TestCase): + + class MockSocket(socket.socket): + def connect(self, *args): + raise socket.timeout('timed out') + + @contextlib.contextmanager + def mocked_socket_module(self): + """Return a socket which times out on connect""" + old_socket = socket.socket + socket.socket = self.MockSocket + try: + yield + finally: + socket.socket = old_socket + + def test_connect(self): + port = support.find_unused_port() + cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(cli.close) + with self.assertRaises(OSError) as cm: + cli.connect((HOST, port)) + self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + + def test_create_connection(self): + # Issue #9792: errors raised by create_connection() should have + # a proper errno attribute. + port = support.find_unused_port() + with self.assertRaises(OSError) as cm: + socket.create_connection((HOST, port)) + + # Issue #16257: create_connection() calls getaddrinfo() against + # 'localhost'. This may result in an IPV6 addr being returned + # as well as an IPV4 one: + # >>> socket.getaddrinfo('localhost', port, 0, SOCK_STREAM) + # >>> [(2, 2, 0, '', ('127.0.0.1', 41230)), + # (26, 2, 0, '', ('::1', 41230, 0, 0))] + # + # create_connection() enumerates through all the addresses returned + # and if it doesn't successfully bind to any of them, it propagates + # the last exception it encountered. + # + # On Solaris, ENETUNREACH is returned in this circumstance instead + # of ECONNREFUSED. So, if that errno exists, add it to our list of + # expected errnos. + expected_errnos = [ errno.ECONNREFUSED, ] + if hasattr(errno, 'ENETUNREACH'): + expected_errnos.append(errno.ENETUNREACH) + + self.assertIn(cm.exception.errno, expected_errnos) + + def test_create_connection_timeout(self): + # Issue #9792: create_connection() should not recast timeout errors + # as generic socket errors. + with self.mocked_socket_module(): + with self.assertRaises(socket.timeout): + socket.create_connection((HOST, 1234)) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.source_port = support.find_unused_port() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def _justAccept(self): + conn, addr = self.serv.accept() + conn.close() + + testFamily = _justAccept + def _testFamily(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.family, 2) + + testSourceAddress = _justAccept + def _testSourceAddress(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30, + source_address=('', self.source_port)) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.getsockname()[1], self.source_port) + # The port number being used is sufficient to show that the bind() + # call happened. + + testTimeoutDefault = _justAccept + def _testTimeoutDefault(self): + # passing no explicit timeout uses socket's global default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(42) + try: + self.cli = socket.create_connection((HOST, self.port)) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), 42) + + testTimeoutNone = _justAccept + def _testTimeoutNone(self): + # None timeout means the same as sock.settimeout(None) + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + self.cli = socket.create_connection((HOST, self.port), timeout=None) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), None) + + testTimeoutValueNamed = _justAccept + def _testTimeoutValueNamed(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.assertEqual(self.cli.gettimeout(), 30) + + testTimeoutValueNonamed = _justAccept + def _testTimeoutValueNonamed(self): + self.cli = socket.create_connection((HOST, self.port), 30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.gettimeout(), 30) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class NetworkConnectionBehaviourTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def testInsideTimeout(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + time.sleep(3) + conn.send(b"done!") + testOutsideTimeout = testInsideTimeout + + def _testInsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port)) + data = sock.recv(5) + self.assertEqual(data, b"done!") + + def _testOutsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port), timeout=1) + self.assertRaises(socket.timeout, lambda: sock.recv(5)) + + +class TCPTimeoutTest(SocketTCPTest): + + def testTCPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.accept() + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (TCP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of error (TCP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (TCP)") + if not ok: + self.fail("accept() returned success when we did not expect it") + + @unittest.skipUnless(hasattr(signal, 'alarm'), + 'test needs signal.alarm()') + def testInterruptedTimeout(self): + # XXX I don't know how to do this test on MSWindows or any other + # plaform that doesn't support signal.alarm() or os.kill(), though + # the bug should have existed on all platforms. + self.serv.settimeout(5.0) # must be longer than alarm + class Alarm(Exception): + pass + def alarm_handler(signal, frame): + raise Alarm + old_alarm = signal.signal(signal.SIGALRM, alarm_handler) + try: + signal.alarm(2) # POSIX allows alarm to be up to 1 second early + try: + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of Alarm") + except Alarm: + pass + except: + self.fail("caught other exception instead of Alarm:" + " %s(%s):\n%s" % + (sys.exc_info()[:2] + (traceback.format_exc(),))) + else: + self.fail("nothing caught") + finally: + signal.alarm(0) # shut off alarm + except Alarm: + self.fail("got Alarm in wrong place") + finally: + # no alarm can be pending. Safe to restore old handler. + signal.signal(signal.SIGALRM, old_alarm) + +class UDPTimeoutTest(SocketUDPTest): + + def testUDPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.recv(1024) + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (UDP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.recv(1024) + except socket.timeout: + self.fail("caught timeout instead of error (UDP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (UDP)") + if not ok: + self.fail("recv() returned success when we did not expect it") + +class TestExceptions(unittest.TestCase): + + def testExceptionTree(self): + self.assertTrue(issubclass(OSError, Exception)) + self.assertTrue(issubclass(socket.herror, OSError)) + self.assertTrue(issubclass(socket.gaierror, OSError)) + self.assertTrue(issubclass(socket.timeout, OSError)) + + def test_setblocking_invalidfd(self): + # Regression test for issue #28471 + + sock0 = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + sock = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, 0, sock0.fileno()) + sock0.close() + self.addCleanup(sock.detach) + + with self.assertRaises(OSError): + sock.setblocking(False) + + +@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +class TestLinuxAbstractNamespace(unittest.TestCase): + + UNIX_PATH_MAX = 108 + + def testLinuxAbstractNamespace(self): + address = b"\x00python-test-hello\x00\xff" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s1: + s1.bind(address) + s1.listen() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s2: + s2.connect(s1.getsockname()) + with s1.accept()[0] as s3: + self.assertEqual(s1.getsockname(), address) + self.assertEqual(s2.getpeername(), address) + + def testMaxName(self): + address = b"\x00" + b"h" * (self.UNIX_PATH_MAX - 1) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(address) + self.assertEqual(s.getsockname(), address) + + def testNameOverflow(self): + address = "\x00" + "h" * self.UNIX_PATH_MAX + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + self.assertRaises(OSError, s.bind, address) + + def testStrName(self): + # Check that an abstract name can be passed as a string. + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s.bind("\x00python\x00test\x00") + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + finally: + s.close() + + def testBytearrayName(self): + # Check that an abstract name can be passed as a bytearray. + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(bytearray(b"\x00python\x00test\x00")) + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + +@unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'test needs socket.AF_UNIX') +class TestUnixDomain(unittest.TestCase): + + def setUp(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + def tearDown(self): + self.sock.close() + + def encoded(self, path): + # Return the given path encoded in the file system encoding, + # or skip the test if this is not possible. + try: + return os.fsencode(path) + except UnicodeEncodeError: + self.skipTest( + "Pathname {0!a} cannot be represented in file " + "system encoding {1!r}".format( + path, sys.getfilesystemencoding())) + + def bind(self, sock, path): + # Bind the socket + try: + support.bind_unix_socket(sock, path) + except OSError as e: + if str(e) == "AF_UNIX path too long": + self.skipTest( + "Pathname {0!a} is too long to serve as an AF_UNIX path" + .format(path)) + else: + raise + + def testUnbound(self): + # Issue #30205 + self.assertIn(self.sock.getsockname(), ('', None)) + + def testStrAddr(self): + # Test binding to and retrieving a normal string pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testBytesAddr(self): + # Test binding to a bytes pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, self.encoded(path)) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testSurrogateescapeBind(self): + # Test binding to a valid non-ASCII pathname, with the + # non-ASCII bytes supplied using surrogateescape encoding. + path = os.path.abspath(support.TESTFN_UNICODE) + b = self.encoded(path) + self.bind(self.sock, b.decode("ascii", "surrogateescape")) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testUnencodableAddr(self): + # Test binding to a pathname that cannot be encoded in the + # file system encoding. + if support.TESTFN_UNENCODABLE is None: + self.skipTest("No unencodable filename available") + path = os.path.abspath(support.TESTFN_UNENCODABLE) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + +@unittest.skipUnless(thread, 'Threading required for this test.') +class BufferIOTest(SocketConnectedTest): + """ + Test the buffer versions of socket.recv() and socket.send(). + """ + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecvIntoArray(self): + buf = array.array("B", [0] * len(MSG)) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + buf = buf.tobytes() + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvIntoBytearray(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoBytearray = _testRecvIntoArray + + def testRecvIntoMemoryview(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoMemoryview = _testRecvIntoArray + + def testRecvFromIntoArray(self): + buf = array.array("B", [0] * len(MSG)) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + buf = buf.tobytes() + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvFromIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoBytearray(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoBytearray = _testRecvFromIntoArray + + def testRecvFromIntoMemoryview(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoMemoryview = _testRecvFromIntoArray + + def testRecvFromIntoSmallBuffer(self): + # See issue #20246. + buf = bytearray(8) + self.assertRaises(ValueError, self.cli_conn.recvfrom_into, buf, 1024) + + def _testRecvFromIntoSmallBuffer(self): + self.serv_conn.send(MSG) + + def testRecvFromIntoEmptyBuffer(self): + buf = bytearray() + self.cli_conn.recvfrom_into(buf) + self.cli_conn.recvfrom_into(buf, 0) + + _testRecvFromIntoEmptyBuffer = _testRecvFromIntoArray + + +TIPC_STYPE = 2000 +TIPC_LOWER = 200 +TIPC_UPPER = 210 + +def isTipcAvailable(): + """Check if the TIPC module is loaded + + The TIPC module is not loaded automatically on Ubuntu and probably + other Linux distros. + """ + if not hasattr(socket, "AF_TIPC"): + return False + try: + f = open("/proc/modules") + except (FileNotFoundError, IsADirectoryError, PermissionError): + # It's ok if the file does not exist, is a directory or if we + # have not the permission to read it. + return False + with f: + for line in f: + if line.startswith("tipc "): + return True + return False + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCTest(unittest.TestCase): + def testRDM(self): + srv = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + cli = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + self.addCleanup(srv.close) + self.addCleanup(cli.close) + + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + srv.bind(srvaddr) + + sendaddr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + cli.sendto(MSG, sendaddr) + + msg, recvaddr = srv.recvfrom(1024) + + self.assertEqual(cli.getsockname(), recvaddr) + self.assertEqual(msg, MSG) + + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCThreadableTest(unittest.TestCase, ThreadableTest): + def __init__(self, methodName = 'runTest'): + unittest.TestCase.__init__(self, methodName = methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.srv = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.srv.close) + self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + self.srv.bind(srvaddr) + self.srv.listen() + self.serverExplicitReady() + self.conn, self.connaddr = self.srv.accept() + self.addCleanup(self.conn.close) + + def clientSetUp(self): + # There is a hittable race between serverExplicitReady() and the + # accept() call; sleep a little while to avoid it, otherwise + # we could get an exception + time.sleep(0.1) + self.cli = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.cli.close) + addr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + self.cli.connect(addr) + self.cliaddr = self.cli.getsockname() + + def testStream(self): + msg = self.conn.recv(1024) + self.assertEqual(msg, MSG) + self.assertEqual(self.cliaddr, self.connaddr) + + def _testStream(self): + self.cli.send(MSG) + self.cli.close() + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class ContextManagersTest(ThreadedTCPSocketTest): + + def _testSocketClass(self): + # base test + with socket.socket() as sock: + self.assertFalse(sock._closed) + self.assertTrue(sock._closed) + # close inside with block + with socket.socket() as sock: + sock.close() + self.assertTrue(sock._closed) + # exception inside with block + with socket.socket() as sock: + self.assertRaises(OSError, sock.sendall, b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionBase(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionBase(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + self.assertFalse(sock._closed) + sock.sendall(b'foo') + self.assertEqual(sock.recv(1024), b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionClose(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionClose(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + sock.close() + self.assertTrue(sock._closed) + self.assertRaises(OSError, sock.sendall, b'foo') + + +class InheritanceTest(unittest.TestCase): + @unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), + "SOCK_CLOEXEC not defined") + @support.requires_linux_version(2, 6, 28) + def test_SOCK_CLOEXEC(self): + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: + self.assertTrue(s.type & socket.SOCK_CLOEXEC) + self.assertFalse(s.get_inheritable()) + + def test_default_inheritable(self): + sock = socket.socket() + with sock: + self.assertEqual(sock.get_inheritable(), False) + + def test_dup(self): + sock = socket.socket() + with sock: + newsock = sock.dup() + sock.close() + with newsock: + self.assertEqual(newsock.get_inheritable(), False) + + def test_set_inheritable(self): + sock = socket.socket() + with sock: + sock.set_inheritable(True) + self.assertEqual(sock.get_inheritable(), True) + + sock.set_inheritable(False) + self.assertEqual(sock.get_inheritable(), False) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + + @unittest.skipUnless(hasattr(socket, "socketpair"), + "need socket.socketpair()") + def test_socketpair(self): + s1, s2 = socket.socketpair() + self.addCleanup(s1.close) + self.addCleanup(s2.close) + self.assertEqual(s1.get_inheritable(), False) + self.assertEqual(s2.get_inheritable(), False) + + +@unittest.skipUnless(hasattr(socket, "SOCK_NONBLOCK"), + "SOCK_NONBLOCK not defined") +class NonblockConstantTest(unittest.TestCase): + def checkNonblock(self, s, nonblock=True, timeout=0.0): + if nonblock: + self.assertTrue(s.type & socket.SOCK_NONBLOCK) + self.assertEqual(s.gettimeout(), timeout) + else: + self.assertFalse(s.type & socket.SOCK_NONBLOCK) + self.assertEqual(s.gettimeout(), None) + + @support.requires_linux_version(2, 6, 28) + def test_SOCK_NONBLOCK(self): + # a lot of it seems silly and redundant, but I wanted to test that + # changing back and forth worked ok + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) as s: + self.checkNonblock(s) + s.setblocking(1) + self.checkNonblock(s, False) + s.setblocking(0) + self.checkNonblock(s) + s.settimeout(None) + self.checkNonblock(s, False) + s.settimeout(2.0) + self.checkNonblock(s, timeout=2.0) + s.setblocking(1) + self.checkNonblock(s, False) + # defaulttimeout + t = socket.getdefaulttimeout() + socket.setdefaulttimeout(0.0) + with socket.socket() as s: + self.checkNonblock(s) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(2.0) + with socket.socket() as s: + self.checkNonblock(s, timeout=2.0) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(t) + + +@unittest.skipUnless(os.name == "nt", "Windows specific") +@unittest.skipUnless(multiprocessing, "need multiprocessing") +class TestSocketSharing(SocketTCPTest): + # This must be classmethod and not staticmethod or multiprocessing + # won't be able to bootstrap it. + @classmethod + def remoteProcessServer(cls, q): + # Recreate socket from shared data + sdata = q.get() + message = q.get() + + s = socket.fromshare(sdata) + s2, c = s.accept() + + # Send the message + s2.sendall(message) + s2.close() + s.close() + + def testShare(self): + # Transfer the listening server socket to another process + # and service it from there. + + # Create process: + q = multiprocessing.Queue() + p = multiprocessing.Process(target=self.remoteProcessServer, args=(q,)) + p.start() + + # Get the shared socket data + data = self.serv.share(p.pid) + + # Pass the shared socket to the other process + addr = self.serv.getsockname() + self.serv.close() + q.put(data) + + # The data that the server will send us + message = b"slapmahfro" + q.put(message) + + # Connect + s = socket.create_connection(addr) + # listen for the data + m = [] + while True: + data = s.recv(100) + if not data: + break + m.append(data) + s.close() + received = b"".join(m) + self.assertEqual(received, message) + p.join() + + def testShareLength(self): + data = self.serv.share(os.getpid()) + self.assertRaises(ValueError, socket.fromshare, data[:-1]) + self.assertRaises(ValueError, socket.fromshare, data+b"foo") + + def compareSockets(self, org, other): + # socket sharing is expected to work only for blocking socket + # since the internal python timeout value isn't transferred. + self.assertEqual(org.gettimeout(), None) + self.assertEqual(org.gettimeout(), other.gettimeout()) + + self.assertEqual(org.family, other.family) + self.assertEqual(org.type, other.type) + # If the user specified "0" for proto, then + # internally windows will have picked the correct value. + # Python introspection on the socket however will still return + # 0. For the shared socket, the python value is recreated + # from the actual value, so it may not compare correctly. + if org.proto != 0: + self.assertEqual(org.proto, other.proto) + + def testShareLocal(self): + data = self.serv.share(os.getpid()) + s = socket.fromshare(data) + try: + self.compareSockets(self.serv, s) + finally: + s.close() + + def testTypes(self): + families = [socket.AF_INET, socket.AF_INET6] + types = [socket.SOCK_STREAM, socket.SOCK_DGRAM] + for f in families: + for t in types: + try: + source = socket.socket(f, t) + except OSError: + continue # This combination is not supported + try: + data = source.share(os.getpid()) + shared = socket.fromshare(data) + try: + self.compareSockets(source, shared) + finally: + shared.close() + finally: + source.close() + + +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendfileUsingSendTest(ThreadedTCPSocketTest): + """ + Test the send() implementation of socket.sendfile(). + """ + + FILESIZE = (10 * 1024 * 1024) # 10MB + BUFSIZE = 8192 + FILEDATA = b"" + TIMEOUT = 2 + + @classmethod + def setUpClass(cls): + def chunks(total, step): + assert total >= step + while total > step: + yield step + total -= step + if total: + yield total + + chunk = b"".join([random.choice(string.ascii_letters).encode() + for i in range(cls.BUFSIZE)]) + with open(support.TESTFN, 'wb') as f: + for csize in chunks(cls.FILESIZE, cls.BUFSIZE): + f.write(chunk) + with open(support.TESTFN, 'rb') as f: + cls.FILEDATA = f.read() + assert len(cls.FILEDATA) == cls.FILESIZE + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + + def accept_conn(self): + self.serv.settimeout(self.TIMEOUT) + conn, addr = self.serv.accept() + conn.settimeout(self.TIMEOUT) + self.addCleanup(conn.close) + return conn + + def recv_data(self, conn): + received = [] + while True: + chunk = conn.recv(self.BUFSIZE) + if not chunk: + break + received.append(chunk) + return b''.join(received) + + def meth_from_sock(self, sock): + # Depending on the mixin class being run return either send() + # or sendfile() method implementation. + return getattr(sock, "_sendfile_use_send") + + # regular file + + def _testRegularFile(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, self.FILESIZE) + self.assertEqual(file.tell(), self.FILESIZE) + + def testRegularFile(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # non regular file + + def _testNonRegularFile(self): + address = self.serv.getsockname() + file = io.BytesIO(self.FILEDATA) + with socket.create_connection(address) as sock, file as file: + sent = sock.sendfile(file) + self.assertEqual(sent, self.FILESIZE) + self.assertEqual(file.tell(), self.FILESIZE) + self.assertRaises(socket._GiveupOnSendfile, + sock._sendfile_use_sendfile, file) + + def testNonRegularFile(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # empty file + + def _testEmptyFileSend(self): + address = self.serv.getsockname() + filename = support.TESTFN + "2" + with open(filename, 'wb'): + self.addCleanup(support.unlink, filename) + file = open(filename, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, 0) + self.assertEqual(file.tell(), 0) + + def testEmptyFileSend(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(data, b"") + + # offset + + def _testOffset(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file, offset=5000) + self.assertEqual(sent, self.FILESIZE - 5000) + self.assertEqual(file.tell(), self.FILESIZE) + + def testOffset(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE - 5000) + self.assertEqual(data, self.FILEDATA[5000:]) + + # count + + def _testCount(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 5000007 + meth = self.meth_from_sock(sock) + sent = meth(file, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count) + + def testCount(self): + count = 5000007 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[:count]) + + # count small + + def _testCountSmall(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 1 + meth = self.meth_from_sock(sock) + sent = meth(file, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count) + + def testCountSmall(self): + count = 1 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[:count]) + + # count + offset + + def _testCountWithOffset(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 100007 + meth = self.meth_from_sock(sock) + sent = meth(file, offset=2007, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count + 2007) + + def testCountWithOffset(self): + count = 100007 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[2007:count+2007]) + + # non blocking sockets are not supposed to work + + def _testNonBlocking(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + sock.setblocking(False) + meth = self.meth_from_sock(sock) + self.assertRaises(ValueError, meth, file) + self.assertRaises(ValueError, sock.sendfile, file) + + def testNonBlocking(self): + conn = self.accept_conn() + if conn.recv(8192): + self.fail('was not supposed to receive any data') + + # timeout (non-triggered) + + def _testWithTimeout(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, self.FILESIZE) + + def testWithTimeout(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # timeout (triggered) + + def _testWithTimeoutTriggeredSend(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=0.01) as sock, \ + file as file: + meth = self.meth_from_sock(sock) + self.assertRaises(socket.timeout, meth, file) + + def testWithTimeoutTriggeredSend(self): + conn = self.accept_conn() + conn.recv(88192) + + # errors + + def _test_errors(self): + pass + + def test_errors(self): + with open(support.TESTFN, 'rb') as file: + with socket.socket(type=socket.SOCK_DGRAM) as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex( + ValueError, "SOCK_STREAM", meth, file) + with open(support.TESTFN, 'rt') as file: + with socket.socket() as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex( + ValueError, "binary mode", meth, file) + with open(support.TESTFN, 'rb') as file: + with socket.socket() as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex(TypeError, "positive integer", + meth, file, count='2') + self.assertRaisesRegex(TypeError, "positive integer", + meth, file, count=0.1) + self.assertRaisesRegex(ValueError, "positive integer", + meth, file, count=0) + self.assertRaisesRegex(ValueError, "positive integer", + meth, file, count=-1) + + +@unittest.skipUnless(thread, 'Threading required for this test.') +@unittest.skipUnless(hasattr(os, "sendfile"), + 'os.sendfile() required for this test.') +class SendfileUsingSendfileTest(SendfileUsingSendTest): + """ + Test the sendfile() implementation of socket.sendfile(). + """ + def meth_from_sock(self, sock): + return getattr(sock, "_sendfile_use_sendfile") + + +@unittest.skipUnless(HAVE_SOCKET_ALG, 'AF_ALG required') +class LinuxKernelCryptoAPI(unittest.TestCase): + # tests for AF_ALG + def create_alg(self, typ, name): + sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + try: + sock.bind((typ, name)) + except FileNotFoundError as e: + # type / algorithm is not available + sock.close() + raise unittest.SkipTest(str(e), typ, name) + else: + return sock + + def test_sha256(self): + expected = bytes.fromhex("ba7816bf8f01cfea414140de5dae2223b00361a396" + "177a9cb410ff61f20015ad") + with self.create_alg('hash', 'sha256') as algo: + op, _ = algo.accept() + with op: + op.sendall(b"abc") + self.assertEqual(op.recv(512), expected) + + op, _ = algo.accept() + with op: + op.send(b'a', socket.MSG_MORE) + op.send(b'b', socket.MSG_MORE) + op.send(b'c', socket.MSG_MORE) + op.send(b'') + self.assertEqual(op.recv(512), expected) + + def test_hmac_sha1(self): + expected = bytes.fromhex("effcdf6ae5eb2fa2d27416d5f184df9c259a7c79") + with self.create_alg('hash', 'hmac(sha1)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, b"Jefe") + op, _ = algo.accept() + with op: + op.sendall(b"what do ya want for nothing?") + self.assertEqual(op.recv(512), expected) + + # Although it should work with 3.19 and newer the test blocks on + # Ubuntu 15.10 with Kernel 4.2.0-19. + @support.requires_linux_version(4, 3) + def test_aes_cbc(self): + key = bytes.fromhex('06a9214036b8a15b512e03d534120006') + iv = bytes.fromhex('3dafba429d9eb430b422da802c9fac41') + msg = b"Single block msg" + ciphertext = bytes.fromhex('e353779c1079aeb82708942dbe77181a') + msglen = len(msg) + with self.create_alg('skcipher', 'cbc(aes)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) + op, _ = algo.accept() + with op: + op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, + flags=socket.MSG_MORE) + op.sendall(msg) + self.assertEqual(op.recv(msglen), ciphertext) + + op, _ = algo.accept() + with op: + op.sendmsg_afalg([ciphertext], + op=socket.ALG_OP_DECRYPT, iv=iv) + self.assertEqual(op.recv(msglen), msg) + + # long message + multiplier = 1024 + longmsg = [msg] * multiplier + op, _ = algo.accept() + with op: + op.sendmsg_afalg(longmsg, + op=socket.ALG_OP_ENCRYPT, iv=iv) + enc = op.recv(msglen * multiplier) + self.assertEqual(len(enc), msglen * multiplier) + self.assertTrue(enc[:msglen], ciphertext) + + op, _ = algo.accept() + with op: + op.sendmsg_afalg([enc], + op=socket.ALG_OP_DECRYPT, iv=iv) + dec = op.recv(msglen * multiplier) + self.assertEqual(len(dec), msglen * multiplier) + self.assertEqual(dec, msg * multiplier) + + @support.requires_linux_version(4, 9) # see issue29324 + def test_aead_aes_gcm(self): + key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') + iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') + plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') + assoc = bytes.fromhex('24825602bd12a984e0092d3e448eda5f') + expected_ct = bytes.fromhex('93fe7d9e9bfd10348a5606e5cafa7354') + expected_tag = bytes.fromhex('0032a1dc85f1c9786925a2e71d8272dd') + + taglen = len(expected_tag) + assoclen = len(assoc) + + with self.create_alg('aead', 'gcm(aes)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_AEAD_AUTHSIZE, + None, taglen) + + # send assoc, plain and tag buffer in separate steps + op, _ = algo.accept() + with op: + op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, + assoclen=assoclen, flags=socket.MSG_MORE) + op.sendall(assoc, socket.MSG_MORE) + op.sendall(plain) + res = op.recv(assoclen + len(plain) + taglen) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # now with msg + op, _ = algo.accept() + with op: + msg = assoc + plain + op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, + assoclen=assoclen) + res = op.recv(assoclen + len(plain) + taglen) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # create anc data manually + pack_uint32 = struct.Struct('I').pack + op, _ = algo.accept() + with op: + msg = assoc + plain + op.sendmsg( + [msg], + ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], + [socket.SOL_ALG, socket.ALG_SET_IV, pack_uint32(len(iv)) + iv], + [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], + ) + ) + res = op.recv(len(msg) + taglen) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # decrypt and verify + op, _ = algo.accept() + with op: + msg = assoc + expected_ct + expected_tag + op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, + assoclen=assoclen) + res = op.recv(len(msg) - taglen) + self.assertEqual(plain, res[assoclen:]) + + @support.requires_linux_version(4, 3) # see test_aes_cbc + def test_drbg_pr_sha256(self): + # deterministic random bit generator, prediction resistance, sha256 + with self.create_alg('rng', 'drbg_pr_sha256') as algo: + extra_seed = os.urandom(32) + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, extra_seed) + op, _ = algo.accept() + with op: + rn = op.recv(32) + self.assertEqual(len(rn), 32) + + def test_sendmsg_afalg_args(self): + sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + with sock: + with self.assertRaises(TypeError): + sock.sendmsg_afalg() + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=None) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(1) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, assoclen=None) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, assoclen=-1) + + +def test_main(): + tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest, + TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ] + + tests.extend([ + NonBlockingTCPTests, + FileObjectClassTestCase, + UnbufferedFileObjectClassTestCase, + LineBufferedFileObjectClassTestCase, + SmallBufferedFileObjectClassTestCase, + UnicodeReadFileObjectClassTestCase, + UnicodeWriteFileObjectClassTestCase, + UnicodeReadWriteFileObjectClassTestCase, + NetworkConnectionNoServer, + NetworkConnectionAttributesTest, + NetworkConnectionBehaviourTest, + ContextManagersTest, + InheritanceTest, + NonblockConstantTest + ]) + tests.append(BasicSocketPairTest) + tests.append(TestUnixDomain) + tests.append(TestLinuxAbstractNamespace) + tests.extend([TIPCTest, TIPCThreadableTest]) + tests.extend([BasicCANTest, CANTest]) + tests.extend([BasicRDSTest, RDSTest]) + tests.append(LinuxKernelCryptoAPI) + tests.extend([ + CmsgMacroTests, + SendmsgUDPTest, + RecvmsgUDPTest, + RecvmsgIntoUDPTest, + SendmsgUDP6Test, + RecvmsgUDP6Test, + RecvmsgRFC3542AncillaryUDP6Test, + RecvmsgIntoRFC3542AncillaryUDP6Test, + RecvmsgIntoUDP6Test, + SendmsgTCPTest, + RecvmsgTCPTest, + RecvmsgIntoTCPTest, + SendmsgSCTPStreamTest, + RecvmsgSCTPStreamTest, + RecvmsgIntoSCTPStreamTest, + SendmsgUnixStreamTest, + RecvmsgUnixStreamTest, + RecvmsgIntoUnixStreamTest, + RecvmsgSCMRightsStreamTest, + RecvmsgIntoSCMRightsStreamTest, + # These are slow when setitimer() is not available + InterruptedRecvTimeoutTest, + InterruptedSendTimeoutTest, + TestSocketSharing, + SendfileUsingSendTest, + SendfileUsingSendfileTest, + ]) + + thread_info = support.threading_setup() + support.run_unittest(*tests) + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.6/test_ssl.py b/src/greentest/3.6/test_ssl.py new file mode 100644 index 0000000..54644e1 --- /dev/null +++ b/src/greentest/3.6/test_ssl.py @@ -0,0 +1,3663 @@ +# Test the support for SSL and sockets + +import sys +import unittest +from test import support +import socket +import select +import time +import datetime +import gc +import os +import errno +import pprint +import tempfile +import urllib.request +import traceback +import asyncore +import weakref +import platform +import functools +try: + import ctypes +except ImportError: + ctypes = None + +ssl = support.import_module("ssl") + +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = support.HOST +IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL') +IS_OPENSSL_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0) + + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/make_ssl_certs.py. +# Other certificates are simply fetched from the Internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = os.fsencode(CERTFILE) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = os.fsencode(ONLYCERT) +BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = os.fsencode(CAPATH) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") + + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SIGNED_CERTFILE2 = data_file("keycert4.pem") +# Same certificate as pycacert.pem, but without extra text in file +SIGNING_CA = data_file("capath", "ceff1710.0") +# cert with all kinds of subject alt names +ALLSANFILE = data_file("allsans.pem") + +REMOTE_HOST = "self-signed.pythontest.net" + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +NONEXISTINGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") + +DHFILE = data_file("dh1024.pem") +BYTES_DHFILE = os.fsencode(DHFILE) + +# Not defined in all versions of OpenSSL +OP_NO_COMPRESSION = getattr(ssl, "OP_NO_COMPRESSION", 0) +OP_SINGLE_DH_USE = getattr(ssl, "OP_SINGLE_DH_USE", 0) +OP_SINGLE_ECDH_USE = getattr(ssl, "OP_SINGLE_ECDH_USE", 0) +OP_CIPHER_SERVER_PREFERENCE = getattr(ssl, "OP_CIPHER_SERVER_PREFERENCE", 0) + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + +def can_clear_options(): + # 0.9.8m or higher + return ssl._OPENSSL_API_VERSION >= (0, 9, 8, 13, 15) + +def no_sslv2_implies_sslv3_hello(): + # 0.9.7h or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) + +def have_verify_flags(): + # 0.9.8 or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 8, 0, 15) + +def utc_offset(): #NOTE: ignore issues like #1647654 + # local time = utc time + utc offset + if time.daylight and time.localtime().tm_isdst > 0: + return -time.altzone # seconds + return -time.timezone + +def asn1time(cert_time): + # Some versions of OpenSSL ignore seconds, see #18207 + # 0.9.8.i + if ssl._OPENSSL_API_VERSION == (0, 9, 8, 9, 15): + fmt = "%b %d %H:%M:%S %Y GMT" + dt = datetime.datetime.strptime(cert_time, fmt) + dt = dt.replace(second=0) + cert_time = dt.strftime(fmt) + # %d adds leading zero but ASN1_TIME_print() uses leading space + if cert_time[4] == "0": + cert_time = cert_time[:4] + " " + cert_time[5:] + + return cert_time + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): + if hasattr(ssl, 'PROTOCOL_SSLv2'): + @functools.wraps(func) + def f(*args, **kwargs): + try: + ssl.SSLContext(ssl.PROTOCOL_SSLv2) + except ssl.SSLError: + if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and + platform.linux_distribution() == ('debian', 'squeeze/sid', '')): + raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") + return func(*args, **kwargs) + return f + else: + return func + +needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") + + +def test_wrap_socket(sock, ssl_version=ssl.PROTOCOL_TLS, *, + cert_reqs=ssl.CERT_NONE, ca_certs=None, + ciphers=None, certfile=None, keyfile=None, + **kwargs): + context = ssl.SSLContext(ssl_version) + if cert_reqs is not None: + context.verify_mode = cert_reqs + if ca_certs is not None: + context.load_verify_locations(ca_certs) + if certfile is not None or keyfile is not None: + context.load_cert_chain(certfile, keyfile) + if ciphers is not None: + context.set_ciphers(ciphers) + return context.wrap_socket(sock, **kwargs) + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + if ssl.HAS_ECDH: + ssl.OP_SINGLE_ECDH_USE + if ssl.OPENSSL_VERSION_INFO >= (1, 0): + ssl.OP_NO_COMPRESSION + self.assertIn(ssl.HAS_SNI, {True, False}) + self.assertIn(ssl.HAS_ECDH, {True, False}) + ssl.OP_NO_SSLv2 + ssl.OP_NO_SSLv3 + ssl.OP_NO_TLSv1 + ssl.OP_NO_TLSv1_3 + if ssl.OPENSSL_VERSION_INFO >= (1, 0, 1): + ssl.OP_NO_TLSv1_1 + ssl.OP_NO_TLSv1_2 + + def test_str_for_enums(self): + # Make sure that the PROTOCOL_* constants have enum-like string + # reprs. + proto = ssl.PROTOCOL_TLS + self.assertEqual(str(proto), '_SSLMethod.PROTOCOL_TLS') + ctx = ssl.SSLContext(proto) + self.assertIs(ctx.protocol, proto) + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + + data, is_cryptographic = ssl.RAND_pseudo_bytes(16) + self.assertEqual(len(data), 16) + self.assertEqual(is_cryptographic, v == 1) + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + + # negative num is invalid + self.assertRaises(ValueError, ssl.RAND_bytes, -5) + self.assertRaises(ValueError, ssl.RAND_pseudo_bytes, -5) + + if hasattr(ssl, 'RAND_egd'): + self.assertRaises(TypeError, ssl.RAND_egd, 1) + self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1) + ssl.RAND_add("this is a random string", 75.0) + ssl.RAND_add(b"this is a random bytes object", 75.0) + ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) + + @unittest.skipUnless(os.name == 'posix', 'requires posix') + def test_random_fork(self): + status = ssl.RAND_status() + if not status: + self.fail("OpenSSL's PRNG has insufficient randomness") + + rfd, wfd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(rfd) + child_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(child_random), 16) + os.write(wfd, child_random) + os.close(wfd) + except BaseException: + os._exit(1) + else: + os._exit(0) + else: + os.close(wfd) + self.addCleanup(os.close, rfd) + _, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + child_random = os.read(rfd, 16) + self.assertEqual(len(child_random), 16) + parent_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(parent_random), 16) + + self.assertNotEqual(child_random, parent_random) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + p = ssl._ssl._test_decode_cert(CERTFILE) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['issuer'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + # Note the next three asserts will fail if the keys are regenerated + self.assertEqual(p['notAfter'], asn1time('Oct 5 23:01:56 2020 GMT')) + self.assertEqual(p['notBefore'], asn1time('Oct 8 23:01:56 2010 GMT')) + self.assertEqual(p['serialNumber'], 'D7C7381919AFC24E') + self.assertEqual(p['subject'], + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)) + ) + self.assertEqual(p['subjectAltName'], (('DNS', 'localhost'),)) + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + + def test_parse_all_sans(self): + p = ssl._ssl._test_decode_cert(ALLSANFILE) + self.assertEqual(p['subjectAltName'], + ( + ('DNS', 'allsans'), + ('othername', ''), + ('othername', ''), + ('email', 'user@example.org'), + ('DNS', 'www.example.org'), + ('DirName', + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'dirname example'),))), + ('URI', 'https://www.python.org/'), + ('IP Address', '127.0.0.1'), + ('IP Address', '0:0:0:0:0:0:0:1\n'), + ('Registered ID', '1.2.3.4.5') + ) + ) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, int) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 0.9 + self.assertGreaterEqual(n, 0x900000) + # < 3.0 + self.assertLess(n, 0x30000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 0) + self.assertLess(major, 3) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + # Version string as returned by {Open,Libre}SSL, the format might change + if IS_LIBRESSL: + self.assertTrue(s.startswith("LibreSSL {:d}".format(major)), + (s, t, hex(n))) + else: + self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)), + (s, t, hex(n))) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = test_wrap_socket(s) + wr = weakref.ref(ss) + with support.check_warnings(("", ResourceWarning)): + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # OSError raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s) as ss: + self.assertRaises(OSError, ss.recv, 1) + self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) + self.assertRaises(OSError, ss.recvfrom, 1) + self.assertRaises(OSError, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(OSError, ss.send, b'x') + self.assertRaises(OSError, ss.sendto, b'x', ('0.0.0.0', 0)) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with test_wrap_socket(s) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + def test_errors_sslwrap(self): + sock = socket.socket() + self.assertRaisesRegex(ValueError, + "certfile must be specified", + ssl.wrap_socket, sock, keyfile=CERTFILE) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True, certfile="") + with ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) as s: + self.assertRaisesRegex(ValueError, "can't connect in server-side mode", + s.connect, (HOST, 8080)) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, + certfile=CERTFILE, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, + certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def bad_cert_test(self, certfile): + """Check that trying to use the given client certificate fails""" + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + certfile) + sock = socket.socket() + self.addCleanup(sock.close) + with self.assertRaises(ssl.SSLError): + test_wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) + + def test_empty_cert(self): + """Wrapping with an empty cert file""" + self.bad_cert_test("nullcert.pem") + + def test_malformed_cert(self): + """Wrapping with a badly formatted certificate (syntax error)""" + self.bad_cert_test("badcert.pem") + + def test_malformed_key(self): + """Wrapping with a badly formatted key (syntax error)""" + self.bad_cert_test("badkey.pem") + + def test_match_hostname(self): + def ok(cert, hostname): + ssl.match_hostname(cert, hostname) + def fail(cert, hostname): + self.assertRaises(ssl.CertificateError, + ssl.match_hostname, cert, hostname) + + # -- Hostname matching -- + + cert = {'subject': ((('commonName', 'example.com'),),)} + ok(cert, 'example.com') + ok(cert, 'ExAmple.cOm') + fail(cert, 'www.example.com') + fail(cert, '.example.com') + fail(cert, 'example.org') + fail(cert, 'exampleXcom') + + cert = {'subject': ((('commonName', '*.a.com'),),)} + ok(cert, 'foo.a.com') + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + # only match one left-most wildcard + cert = {'subject': ((('commonName', 'f*.com'),),)} + ok(cert, 'foo.com') + ok(cert, 'f.com') + fail(cert, 'bar.com') + fail(cert, 'foo.a.com') + fail(cert, 'bar.foo.com') + + # NULL bytes are bad, CVE-2013-4073 + cert = {'subject': ((('commonName', + 'null.python.org\x00example.org'),),)} + ok(cert, 'null.python.org\x00example.org') # or raise an error? + fail(cert, 'example.org') + fail(cert, 'null.python.org') + + # error cases with wildcards + cert = {'subject': ((('commonName', '*.*.a.com'),),)} + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + cert = {'subject': ((('commonName', 'a.*.com'),),)} + fail(cert, 'a.foo.com') + fail(cert, 'a..com') + fail(cert, 'a.com') + + # wildcard doesn't match IDNA prefix 'xn--' + idna = 'püthon.python.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, idna) + cert = {'subject': ((('commonName', 'x*.python.org'),),)} + fail(cert, idna) + cert = {'subject': ((('commonName', 'xn--p*.python.org'),),)} + fail(cert, idna) + + # wildcard in first fragment and IDNA A-labels in sequent fragments + # are supported. + idna = 'www*.pythön.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, 'www.pythön.org'.encode("idna").decode("ascii")) + ok(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'ftp.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'pythön.org'.encode("idna").decode("ascii")) + + # Slightly fake real-world example + cert = {'notAfter': 'Jun 26 21:41:46 2011 GMT', + 'subject': ((('commonName', 'linuxfrz.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('DNS', 'linuxfr.com'), + ('othername', ''))} + ok(cert, 'linuxfr.org') + ok(cert, 'linuxfr.com') + # Not a "DNS" entry + fail(cert, '') + # When there is a subjectAltName, commonName isn't used + fail(cert, 'linuxfrz.org') + + # A pristine real-world example + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),), + (('commonName', 'mail.google.com'),))} + ok(cert, 'mail.google.com') + fail(cert, 'gmail.com') + # Only commonName is considered + fail(cert, 'California') + + # -- IPv4 matching -- + cert = {'subject': ((('commonName', 'example.com'),),), + 'subjectAltName': (('DNS', 'example.com'), + ('IP Address', '10.11.12.13'), + ('IP Address', '14.15.16.17'))} + ok(cert, '10.11.12.13') + ok(cert, '14.15.16.17') + fail(cert, '14.15.16.18') + fail(cert, 'example.net') + + # -- IPv6 matching -- + cert = {'subject': ((('commonName', 'example.com'),),), + 'subjectAltName': (('DNS', 'example.com'), + ('IP Address', '2001:0:0:0:0:0:0:CAFE\n'), + ('IP Address', '2003:0:0:0:0:0:0:BABA\n'))} + ok(cert, '2001::cafe') + ok(cert, '2003::baba') + fail(cert, '2003::bebe') + fail(cert, 'example.net') + + # -- Miscellaneous -- + + # Neither commonName nor subjectAltName + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),))} + fail(cert, 'mail.google.com') + + # No DNS entry in subjectAltName but a commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('commonName', 'mail.google.com'),)), + 'subjectAltName': (('othername', 'blabla'), )} + ok(cert, 'mail.google.com') + + # No DNS entry subjectAltName and no commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),)), + 'subjectAltName': (('othername', 'blabla'),)} + fail(cert, 'google.com') + + # Empty cert / no cert + self.assertRaises(ValueError, ssl.match_hostname, None, 'example.com') + self.assertRaises(ValueError, ssl.match_hostname, {}, 'example.com') + + # Issue #17980: avoid denials of service by refusing more than one + # wildcard per fragment. + cert = {'subject': ((('commonName', 'a*b.com'),),)} + ok(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b.co*'),),)} + fail(cert, 'axxb.com') + cert = {'subject': ((('commonName', 'a*b*.com'),),)} + with self.assertRaises(ssl.CertificateError) as cm: + ssl.match_hostname(cert, 'axxbxxc.com') + self.assertIn("too many wildcards", str(cm.exception)) + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with socket.socket() as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + s.bind(('127.0.0.1', 0)) + s.listen() + c = socket.socket(socket.AF_INET) + c.connect(s.getsockname()) + with test_wrap_socket(c, do_handshake_on_connect=False) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + s.close() + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + def test_dealloc_warn(self): + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + r = repr(ss) + with self.assertWarns(ResourceWarning) as cm: + ss = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (set, bool)) + if isinstance(trust, set): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegex(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegex(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + test_wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + def cert_time_ok(self, timestring, timestamp): + self.assertEqual(ssl.cert_time_to_seconds(timestring), timestamp) + + def cert_time_fail(self, timestring): + with self.assertRaises(ValueError): + ssl.cert_time_to_seconds(timestring) + + @unittest.skipUnless(utc_offset(), + 'local time needs to be different from UTC') + def test_cert_time_to_seconds_timezone(self): + # Issue #19940: ssl.cert_time_to_seconds() returns wrong + # results if local timezone is not UTC + self.cert_time_ok("May 9 00:00:00 2007 GMT", 1178668800.0) + self.cert_time_ok("Jan 5 09:34:43 2018 GMT", 1515144883.0) + + def test_cert_time_to_seconds(self): + timestring = "Jan 5 09:34:43 2018 GMT" + ts = 1515144883.0 + self.cert_time_ok(timestring, ts) + # accept keyword parameter, assert its name + self.assertEqual(ssl.cert_time_to_seconds(cert_time=timestring), ts) + # accept both %e and %d (space or zero generated by strftime) + self.cert_time_ok("Jan 05 09:34:43 2018 GMT", ts) + # case-insensitive + self.cert_time_ok("JaN 5 09:34:43 2018 GmT", ts) + self.cert_time_fail("Jan 5 09:34 2018 GMT") # no seconds + self.cert_time_fail("Jan 5 09:34:43 2018") # no GMT + self.cert_time_fail("Jan 5 09:34:43 2018 UTC") # not GMT timezone + self.cert_time_fail("Jan 35 09:34:43 2018 GMT") # invalid day + self.cert_time_fail("Jon 5 09:34:43 2018 GMT") # invalid month + self.cert_time_fail("Jan 5 24:00:00 2018 GMT") # invalid hour + self.cert_time_fail("Jan 5 09:60:43 2018 GMT") # invalid minute + + newyear_ts = 1230768000.0 + # leap seconds + self.cert_time_ok("Dec 31 23:59:60 2008 GMT", newyear_ts) + # same timestamp + self.cert_time_ok("Jan 1 00:00:00 2009 GMT", newyear_ts) + + self.cert_time_ok("Jan 5 09:34:59 2018 GMT", 1515144899) + # allow 60th second (even if it is not a leap second) + self.cert_time_ok("Jan 5 09:34:60 2018 GMT", 1515144900) + # allow 2nd leap second for compatibility with time.strptime() + self.cert_time_ok("Jan 5 09:34:61 2018 GMT", 1515144901) + self.cert_time_fail("Jan 5 09:34:62 2018 GMT") # invalid seconds + + # no special treatement for the special value: + # 99991231235959Z (rfc 5280) + self.cert_time_ok("Dec 31 23:59:59 9999 GMT", 253402300799.0) + + @support.run_with_locale('LC_ALL', '') + def test_cert_time_to_seconds_locale(self): + # `cert_time_to_seconds()` should be locale independent + + def local_february_name(): + return time.strftime('%b', (1, 2, 3, 4, 5, 6, 0, 0, 0)) + + if local_february_name().lower() == 'feb': + self.skipTest("locale-specific month name needs to be " + "different from C locale") + + # locale-independent + self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) + self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + + def test_connect_ex_error(self): + server = socket.socket(socket.AF_INET) + self.addCleanup(server.close) + port = support.bind_port(server) # Reserve port but don't listen + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + rc = s.connect_ex((HOST, port)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + + +class ContextTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_constructor(self): + for protocol in PROTOCOLS: + ssl.SSLContext(protocol) + ctx = ssl.SSLContext() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + @skip_if_broken_ubuntu_ssl + def test_protocol(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.protocol, proto) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @unittest.skipIf(ssl.OPENSSL_VERSION_INFO < (1, 0, 2, 0, 0), 'OpenSSL too old') + def test_get_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers('AESGCM') + names = set(d['name'] for d in ctx.get_ciphers()) + self.assertIn('AES256-GCM-SHA384', names) + self.assertIn('AES128-GCM-SHA256', names) + + @skip_if_broken_ubuntu_ssl + def test_options(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + # SSLContext also enables these by default + default |= (OP_NO_COMPRESSION | OP_CIPHER_SERVER_PREFERENCE | + OP_SINGLE_DH_USE | OP_SINGLE_ECDH_USE) + self.assertEqual(default, ctx.options) + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + if can_clear_options(): + ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) + self.assertEqual(default, ctx.options) + ctx.options = 0 + # Ubuntu has OP_NO_SSLv3 forced on by default + self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + else: + with self.assertRaises(ValueError): + ctx.options = 0 + + def test_verify_mode(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read() + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read() + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegex(ssl.SSLError, "no start line"): + ctx.load_verify_locations(cadata="broken") + with self.assertRaisesRegex(ssl.SSLError, "not enough data"): + ctx.load_verify_locations(cadata=b"broken") + + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_dh_params(DHFILE) + if os.name != 'nt': + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(FileNotFoundError) as cm: + ctx.load_dh_params(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + @skip_if_broken_ubuntu_ssl + def test_session_stats(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + @needs_sni + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @needs_sni + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': asn1time('Mar 29 12:29:49 2033 GMT'), + 'notBefore': asn1time('Mar 30 12:29:49 2003 GMT'), + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def _assert_context_options(self, ctx): + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + if OP_NO_COMPRESSION != 0: + self.assertEqual(ctx.options & OP_NO_COMPRESSION, + OP_NO_COMPRESSION) + if OP_SINGLE_DH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_DH_USE, + OP_SINGLE_DH_USE) + if OP_SINGLE_ECDH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_ECDH_USE, + OP_SINGLE_ECDH_USE) + if OP_CIPHER_SERVER_PREFERENCE != 0: + self.assertEqual(ctx.options & OP_CIPHER_SERVER_PREFERENCE, + OP_CIPHER_SERVER_PREFERENCE) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) + + + with open(SIGNING_CA) as f: + cadata = f.read() + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self._assert_context_options(ctx) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self._assert_context_options(ctx) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_SSLv23) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + def test_check_hostname(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertFalse(ctx.check_hostname) + + # Requires CERT_REQUIRED or CERT_OPTIONAL + with self.assertRaises(ValueError): + ctx.check_hostname = True + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + + def test_context_client_server(self): + # PROTOCOL_TLS_CLIENT has sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + # PROTOCOL_TLS_SERVER has different but also sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + self.assertEqual(cm.exception.library, 'PEM') + self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) + self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen() + c = socket.socket() + c.connect(s.getsockname()) + c.setblocking(False) + with ctx.wrap_socket(c, False, do_handshake_on_connect=False) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertTrue(s.startswith("The operation did not complete (read)"), s) + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + +class MemoryBIOTests(unittest.TestCase): + + def test_read_write(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + self.assertEqual(bio.read(), b'') + bio.write(b'foo') + bio.write(b'bar') + self.assertEqual(bio.read(), b'foobar') + self.assertEqual(bio.read(), b'') + bio.write(b'baz') + self.assertEqual(bio.read(2), b'ba') + self.assertEqual(bio.read(1), b'z') + self.assertEqual(bio.read(1), b'') + + def test_eof(self): + bio = ssl.MemoryBIO() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertFalse(bio.eof) + bio.write(b'foo') + self.assertFalse(bio.eof) + bio.write_eof() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(2), b'fo') + self.assertFalse(bio.eof) + self.assertEqual(bio.read(1), b'o') + self.assertTrue(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertTrue(bio.eof) + + def test_pending(self): + bio = ssl.MemoryBIO() + self.assertEqual(bio.pending, 0) + bio.write(b'foo') + self.assertEqual(bio.pending, 3) + for i in range(3): + bio.read(1) + self.assertEqual(bio.pending, 3-i-1) + for i in range(3): + bio.write(b'x') + self.assertEqual(bio.pending, i+1) + bio.read() + self.assertEqual(bio.pending, 0) + + def test_buffer_types(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + bio.write(bytearray(b'bar')) + self.assertEqual(bio.read(), b'bar') + bio.write(memoryview(b'baz')) + self.assertEqual(bio.read(), b'baz') + + def test_error_types(self): + bio = ssl.MemoryBIO() + self.assertRaises(TypeError, bio.write, 'foo') + self.assertRaises(TypeError, bio.write, None) + self.assertRaises(TypeError, bio.write, True) + self.assertRaises(TypeError, bio.write, 1) + + +@unittest.skipUnless(_have_threads, "Needs threading module") +class SimpleBackgroundTests(unittest.TestCase): + + """Tests that connect to a simple server running in the background""" + + def setUp(self): + server = ThreadedEchoServer(SIGNED_CERTFILE) + self.server_addr = (HOST, server.port) + server.__enter__() + self.addCleanup(server.__exit__, None, None, None) + + def test_connect(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + self.assertFalse(s.server_side) + + # this should succeed because we specify the root cert + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) as s: + s.connect(self.server_addr) + self.assertTrue(s.getpeercert()) + self.assertFalse(s.server_side) + + def test_connect_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) + self.addCleanup(s.close) + self.assertEqual(0, s.connect_ex(self.server_addr)) + self.assertTrue(s.getpeercert()) + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.setblocking(False) + rc = s.connect_ex(self.server_addr) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + + def test_connect_with_context(self): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + # Same with a server hostname + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="dummy") as s: + s.connect(self.server_addr) + ctx.verify_mode = ssl.CERT_REQUIRED + # This should succeed because we specify the root cert + ctx.load_verify_locations(SIGNING_CA) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_with_context_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_cadata(self): + with open(SIGNING_CA) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + ss.connect(self.server_addr) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + s = socket.socket(socket.AF_INET) + s.connect(self.server_addr) + s.setblocking(False) + s = test_wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + self.addCleanup(s.close) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + _test_get_server_certificate(self, *self.server_addr, cert=SIGNING_CA) + + def test_get_server_certificate_fail(self): + # Connection failure crashes ThreadedEchoServer, so run this in an + # independent test method + _test_get_server_certificate_fail(self, *self.server_addr) + + def test_ciphers(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(self.server_addr) + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(self.server_addr) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = test_wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(self.server_addr) + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + self.assertEqual(len(ctx.get_ca_certs()), 1) + + @needs_sni + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s) as ss: + ss.connect(self.server_addr) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + + def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): + # A simple IO loop. Call func(*args) depending on the error we get + # (WANT_READ or WANT_WRITE) move data between the socket and the BIOs. + timeout = kwargs.get('timeout', 10) + count = 0 + while True: + errno = None + count += 1 + try: + ret = func(*args) + except ssl.SSLError as e: + if e.errno not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + raise + errno = e.errno + # Get any data from the outgoing BIO irrespective of any error, and + # send it to the socket. + buf = outgoing.read() + sock.sendall(buf) + # If there's no error, we're done. For WANT_READ, we need to get + # data from the socket and put it in the incoming BIO. + if errno is None: + break + elif errno == ssl.SSL_ERROR_WANT_READ: + buf = sock.recv(32768) + if buf: + incoming.write(buf) + else: + incoming.write_eof() + if support.verbose: + sys.stdout.write("Needed %d calls to complete %s().\n" + % (count, func.__name__)) + return ret + + def test_bio_handshake(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(SIGNING_CA) + ctx.check_hostname = True + sslobj = ctx.wrap_bio(incoming, outgoing, False, 'localhost') + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNone(sslobj.version()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # If the server shuts down the TCP connection without sending a + # secure shutdown message, this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + + def test_bio_read_write_data(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'FOO\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf, b'foo\n') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + + +class NetworkedTests(unittest.TestCase): + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with support.transient_internet(REMOTE_HOST): + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'Needs IPv6') + def test_get_server_certificate_ipv6(self): + with support.transient_internet('ipv6.google.com'): + _test_get_server_certificate(self, 'ipv6.google.com', 443) + _test_get_server_certificate_fail(self, 'ipv6.google.com', 443) + + +def _test_get_server_certificate(test, host, port, cert=None): + pem = ssl.get_server_certificate((host, port)) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + +def _test_get_server_certificate_fail(test, host, port): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + +if _have_threads: + from test.ssl_servers import make_https_server + + class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(1) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) + self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) + except (ssl.SSLError, ConnectionResetError, OSError) as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # OSError may occur with wrong protocols, e.g. both + # sides use PROTOCOL_TLS_SERVER. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + # + # bpo-31323: Store the exception as string to prevent + # a reference leak: server -> conn_errors -> exception + # -> traceback -> self (ConnectionHandler) -> server + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.server.stop() + self.close() + return False + else: + self.server.shared_ciphers.append(self.sslconn.shared_ciphers()) + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + sys.stdout.write(" cert binary is " + str(len(cert_binary)) + " bytes\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + sys.stdout.write(" server: selected protocol is now " + + str(self.sslconn.selected_npn_protocol()) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + try: + self.sock = self.sslconn.unwrap() + except OSError: + # Many tests shut the TCP connection down + # without an SSL shutdown. This causes + # unwrap() to raise OSError with errno=0! + pass + else: + self.sslconn = None + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except OSError: + if self.server.chatty: + handle_error("Test server failure:\n") + self.close() + self.running = False + # normally, we'd just stop here, but for the test + # harness, we want to stop the server + self.server.stop() + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + npn_protocols=None, alpn_protocols=None, + ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLSv1) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if npn_protocols: + self.context.set_npn_protocols(npn_protocols) + if alpn_protocols: + self.context.set_alpn_protocols(alpn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = support.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_npn_protocols = [] + self.selected_alpn_protocols = [] + self.shared_ciphers = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.sock.settimeout(0.05) + self.sock.listen() + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except socket.timeout: + pass + except KeyboardInterrupt: + self.stop() + self.sock.close() + + def stop(self): + self.active = False + + class AsyncoreEchoServer(threading.Thread): + + # this one's based on asyncore.dispatcher + + class EchoServer (asyncore.dispatcher): + + class ConnectionHandler(asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = test_wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accepted(self, sock_obj, addr): + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + # make sure that ConnectionHandler is removed from socket_map + asyncore.close_all(ignore_all=True) + + def start (self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + + def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None, + session=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=sni_name, session=session) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_alpn_protocol': s.selected_alpn_protocol(), + 'client_npn_protocol': s.selected_npn_protocol(), + 'version': s.version(), + 'session_reused': s.session_reused, + 'session': s.session, + }) + s.close() + stats['server_alpn_protocols'] = server.selected_alpn_protocols + stats['server_npn_protocols'] = server.selected_npn_protocols + stats['server_shared_ciphers'] = server.shared_ciphers + return stats + + def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_SSLv23: + client_context.set_ciphers("ALL") + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(CERTFILE) + ctx.load_verify_locations(CERTFILE) + try: + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except OSError as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) + + + class ThreadedTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + for protocol in PROTOCOLS: + if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}: + continue + with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): + context = ssl.SSLContext(protocol) + context.load_cert_chain(CERTFILE) + server_params_test(context, context, + chatty=True, connectionchatty=True) + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # server_context.load_verify_locations(SIGNING_CA) + server_context.load_cert_chain(SIGNED_CERTFILE2) + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_SERVER): + server_params_test(client_context=client_context, + server_context=server_context, + chatty=True, connectionchatty=True, + sni_name='fakehostname') + + client_context.check_hostname = False + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True, + sni_name='fakehostname') + self.assertIn('called a function you should not call', + str(e.exception)) + + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_SERVER): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=server_context, + chatty=True, connectionchatty=True) + self.assertIn('called a function you should not call', + str(e.exception)) + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True) + self.assertIn('called a function you should not call', + str(e.exception)) + + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + s = context.wrap_socket(socket.socket(), + do_handshake_on_connect=False) + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + s.close() + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(SIGNING_CA) + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + with self.assertRaisesRegex(ssl.SSLError, + "certificate verify failed"): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="localhost") as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: + with self.assertRaisesRegex(ssl.CertificateError, + "hostname 'invalid' doesn't match 'localhost'"): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with socket.socket() as s: + with self.assertRaisesRegex(ValueError, + "check_hostname requires server_hostname"): + context.wrap_socket(s) + + def test_wrong_cert(self): + """Connecting when the server rejects the client's certificate + + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with a wrong client certificate fails. + """ + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + "wrongcert.pem") + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_REQUIRED, + cacerts=CERTFILE, chatty=False, + connectionchatty=False) + with server, \ + socket.socket() as sock, \ + test_wrap_socket(sock, + certfile=certfile, + ssl_version=ssl.PROTOCOL_TLSv1) as s: + try: + # Expect either an SSL error about the server rejecting + # the connection, or a low-level connection reset (which + # sometimes happens on Windows) + s.connect((HOST, server.port)) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = support.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen() + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with socket.socket() as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = test_wrap_socket(c) + except OSError: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") + def test_protocol_sslv2(self): + """Connecting to an SSLv2 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) + # SSLv23 client with specific SSL options + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv2) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + def test_protocol_sslv23(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) + except OSError as x: + # this fails on some older versions of OpenSSL (0.9.7l, for instance) + if support.verbose: + sys.stdout.write( + " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" + % str(x)) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1') + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + + # Server with specific SSL options + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), + "OpenSSL is compiled without SSLv3 support") + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, + False, client_options=ssl.OP_NO_SSLv2) + + @skip_if_broken_ubuntu_ssl + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), + "TLS version 1.1 not supported.") + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), + "TLS version 1.2 not supported.") + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv23, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(1) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = test_wrap_socket(s, ssl_version=ssl.PROTOCOL_TLSv1) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using socketserver to create and manage SSL connections.""" + server = make_https_server(self, certfile=CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + with open(CERTFILE, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = 'https://localhost:%d/%s' % ( + server.port, os.path.split(CERTFILE)[1]) + context = ssl.create_default_context(cafile=CERTFILE) + f = urllib.request.urlopen(url, context=context) + try: + dlen = f.info().get("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = test_wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, expect success?, *args, return value func) + send_methods = [ + ('send', s.send, True, [], len), + ('sendto', s.sendto, False, ["some.address"], len), + ('sendall', s.sendall, True, [], lambda x: None), + ] + # (name, method, whether to expect success, *args) + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = "PREFIX_" + + for (meth_name, send_meth, expect_success, args, + ret_val_meth) in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + ret = send_meth(indata, *args) + msg = "sending with {}".format(meth_name) + self.assertEqual(ret, ret_val_meth(indata), msg=msg) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # read(-1, buffer) is supported, even though read(-1) is not + data = b"data" + s.send(data) + buffer = bytearray(len(data)) + self.assertEqual(s.read(-1, buffer), len(data)) + self.assertEqual(buffer, data) # sendall accepts bytes-like objects + + if ctypes is not None: + ubyte = ctypes.c_ubyte * len(data) + byteslike = ubyte.from_buffer_copy(data) + s.sendall(byteslike) + self.assertEqual(s.read(), data) + + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, bytearray(100)) + + s.write(b"over\n") + + self.assertRaises(ValueError, s.recv, -1) + self.assertRaises(ValueError, s.read, -1) + + s.close() + + def test_recv_zero(self): + server = ThreadedEchoServer(CERTFILE) + server.__enter__() + self.addCleanup(server.__exit__, None, None) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + # recv/read(0) should return no data + s.send(b"data") + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.read(0), b"") + self.assertEqual(s.read(), b"data") + + # Should not block if the other end sends no data + s.setblocking(False) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.recv_into(bytearray()), 0) + + def test_nonblocking_send(self): + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + s.setblocking(False) + + # If we keep sending data, at some point the buffers + # will be full and the call will block + buf = bytearray(8192) + def fill_buffer(): + while True: + s.send(buf) + self.assertRaises((ssl.SSLWantWriteError, + ssl.SSLWantReadError), fill_buffer) + + # Now read all the output and discard it + s.setblocking(True) + s.close() + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen() + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + test_wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = test_wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + server = context.wrap_socket(server, server_side=True) + self.assertTrue(server.server_side) + + evt = threading.Event() + remote = None + peer = None + def serve(): + nonlocal remote, peer + server.listen() + # Block on the accept and wait on the connection to close. + evt.set() + remote, peer = server.accept() + remote.recv(1) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = context.wrap_socket(socket.socket()) + client.connect((host, port)) + client_addr = client.getsockname() + client.close() + t.join() + remote.close() + server.close() + # Sanity checks. + self.assertIsInstance(remote, ssl.SSLSocket) + self.assertEqual(peer, client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_default_ciphers(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + try: + # Force a set of weak ciphers on our client context + context.set_ciphers("DES") + except ssl.SSLError: + self.skipTest("no DES cipher available") + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_SSLv23, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) + self.assertIn("no shared cipher", server.conn_errors[0]) + + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLSv1, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + self.assertIs(s.version(), None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1') + self.assertIs(s.version(), None) + + @unittest.skipUnless(ssl.HAS_TLSv1_3, + "test requires TLSv1.3 enabled OpenSSL") + def test_tls1_3(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + context.load_cert_chain(CERTFILE) + # disable all but TLS 1.3 + context.options |= ( + ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_2 + ) + with ThreadedEchoServer(context=context) as server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + self.assertIn(s.cipher()[0], [ + 'TLS13-AES-256-GCM-SHA384', + 'TLS13-CHACHA20-POLY1305-SHA256', + 'TLS13-AES-128-GCM-SHA256', + ]) + + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.load_cert_chain(CERTFILE) + # TLSv1.3 defaults to PFS key agreement and no longer has KEA in + # cipher name. + context.options |= ssl.OP_NO_TLSv1_3 + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + if ssl.OPENSSL_VERSION_INFO < (1, 0, 0): + context.set_ciphers("ECCdraft:ECDH") + with ThreadedEchoServer(context=context) as server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got channel binding data: {0!r}\n" + .format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + s.close() + + # now, again + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got another channel binding data: {0!r}\n" + .format(new_cb_data)) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + s.close() + + def test_compression(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['compression'], None) + + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + context.load_dh_params(DHFILE) + context.set_ciphers("kEDH") + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: + self.fail("Non-DH cipher: " + cipher[0]) + + def test_selected_alpn_protocol(self): + # selected_alpn_protocol() is None unless ALPN is used. + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support required") + def test_selected_alpn_protocol_if_server_uses_alpn(self): + # selected_alpn_protocol() is None unless ALPN is used by the client. + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_verify_locations(CERTFILE) + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(['foo', 'bar']) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support needed for this test") + def test_alpn_protocols(self): + server_protocols = ['foo', 'bar', 'milkshake'] + protocol_tests = [ + (['foo', 'bar'], 'foo'), + (['bar', 'foo'], 'foo'), + (['milkshake'], 'milkshake'), + (['http/3.0', 'http/4.0'], None) + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + server_context.load_cert_chain(CERTFILE) + server_context.set_alpn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) + client_context.load_cert_chain(CERTFILE) + client_context.set_alpn_protocols(client_protocols) + + try: + stats = server_params_test(client_context, + server_context, + chatty=True, + connectionchatty=True) + except ssl.SSLError as e: + stats = e + + if (expected is None and IS_OPENSSL_1_1 + and ssl.OPENSSL_VERSION_INFO < (1, 1, 0, 6)): + # OpenSSL 1.1.0 to 1.1.0e raises handshake error + self.assertIsInstance(stats, ssl.SSLError) + else: + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_alpn_protocol'] + self.assertEqual(client_result, expected, + msg % (client_result, "client")) + server_result = stats['server_alpn_protocols'][-1] \ + if len(stats['server_alpn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, + msg % (server_result, "server")) + + def test_selected_npn_protocol(self): + # selected_npn_protocol() is None unless NPN is used + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context.load_cert_chain(CERTFILE) + stats = server_params_test(context, context, + chatty=True, connectionchatty=True) + self.assertIs(stats['client_npn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test") + def test_npn_protocols(self): + server_protocols = ['http/1.1', 'spdy/2'] + protocol_tests = [ + (['http/1.1', 'spdy/2'], 'http/1.1'), + (['spdy/2', 'http/1.1'], 'http/1.1'), + (['spdy/2', 'test'], 'spdy/2'), + (['abc', 'def'], 'abc') + ] + for client_protocols, expected in protocol_tests: + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(CERTFILE) + server_context.set_npn_protocols(server_protocols) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.load_cert_chain(CERTFILE) + client_context.set_npn_protocols(client_protocols) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True) + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_npn_protocol'] + self.assertEqual(client_result, expected, msg % (client_result, "client")) + server_result = stats['server_npn_protocols'][-1] \ + if len(stats['server_npn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, msg % (server_result, "server")) + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + @needs_sni + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, 'localhost') + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, 'localhost') + self.assertEqual(calls, []) + + @needs_sni + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @needs_sni + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1/0 + server_context.set_servername_callback(cb_raising) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertIn("ZeroDivisionError", stderr.getvalue()) + + @needs_sni + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertIn("TypeError", stderr.getvalue()) + + def test_shared_ciphers(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + if ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): + client_context.set_ciphers("AES128:AES256") + server_context.set_ciphers("AES256") + alg1 = "AES256" + alg2 = "AES-256" + else: + client_context.set_ciphers("AES:3DES") + server_context.set_ciphers("3DES") + alg1 = "3DES" + alg2 = "DES-CBC3" + + stats = server_params_test(client_context, server_context) + ciphers = stats['server_shared_ciphers'][0] + self.assertGreater(len(ciphers), 0) + for name, tls_version, bits in ciphers: + if not alg1 in name.split("-") and alg2 not in name: + self.fail(name) + + def test_read_write_after_close_raises_valuerror(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + + with server: + s = context.wrap_socket(socket.socket()) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + def test_sendfile(self): + TEST_DATA = b"x" * 512 + with open(support.TESTFN, 'wb') as f: + f.write(TEST_DATA) + self.addCleanup(support.unlink, support.TESTFN) + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + with open(support.TESTFN, 'rb') as file: + s.sendfile(file) + self.assertEqual(s.recv(1024), TEST_DATA) + + def test_session(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + server_context.load_cert_chain(SIGNED_CERTFILE) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + client_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_verify_locations(SIGNING_CA) + + # first connection without session + stats = server_params_test(client_context, server_context) + session = stats['session'] + self.assertTrue(session.id) + self.assertGreater(session.time, 0) + self.assertGreater(session.timeout, 0) + self.assertTrue(session.has_ticket) + if ssl.OPENSSL_VERSION_INFO > (1, 0, 1): + self.assertGreater(session.ticket_lifetime_hint, 0) + self.assertFalse(stats['session_reused']) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 1) + self.assertEqual(sess_stat['hits'], 0) + + # reuse session + stats = server_params_test(client_context, server_context, session=session) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 2) + self.assertEqual(sess_stat['hits'], 1) + self.assertTrue(stats['session_reused']) + session2 = stats['session'] + self.assertEqual(session2.id, session.id) + self.assertEqual(session2, session) + self.assertIsNot(session2, session) + self.assertGreaterEqual(session2.time, session.time) + self.assertGreaterEqual(session2.timeout, session.timeout) + + # another one without session + stats = server_params_test(client_context, server_context) + self.assertFalse(stats['session_reused']) + session3 = stats['session'] + self.assertNotEqual(session3.id, session.id) + self.assertNotEqual(session3, session) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 3) + self.assertEqual(sess_stat['hits'], 1) + + # reuse session again + stats = server_params_test(client_context, server_context, session=session) + self.assertTrue(stats['session_reused']) + session4 = stats['session'] + self.assertEqual(session4.id, session.id) + self.assertEqual(session4, session) + self.assertGreaterEqual(session4.time, session.time) + self.assertGreaterEqual(session4.timeout, session.timeout) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 4) + self.assertEqual(sess_stat['hits'], 2) + + def test_session_handling(self): + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(CERTFILE) + context.load_cert_chain(CERTFILE) + + context2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context2.verify_mode = ssl.CERT_REQUIRED + context2.load_verify_locations(CERTFILE) + context2.load_cert_chain(CERTFILE) + + # TODO: session reuse does not work with TLS 1.3 + context.options |= ssl.OP_NO_TLSv1_3 + context2.options |= ssl.OP_NO_TLSv1_3 + + server = ThreadedEchoServer(context=context, chatty=False) + with server: + with context.wrap_socket(socket.socket()) as s: + # session is None before handshake + self.assertEqual(s.session, None) + self.assertEqual(s.session_reused, None) + s.connect((HOST, server.port)) + session = s.session + self.assertTrue(session) + with self.assertRaises(TypeError) as e: + s.session = object + self.assertEqual(str(e.exception), 'Value is not a SSLSession.') + + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + # cannot set session after handshake + with self.assertRaises(ValueError) as e: + s.session = session + self.assertEqual(str(e.exception), + 'Cannot set session after handshake.') + + with context.wrap_socket(socket.socket()) as s: + # can set session before handshake and before the + # connection was established + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(s.session.id, session.id) + self.assertEqual(s.session, session) + self.assertEqual(s.session_reused, True) + + with context2.wrap_socket(socket.socket()) as s: + # cannot re-use session with a different SSLContext + with self.assertRaises(ValueError) as e: + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(str(e.exception), + 'Session refers to a different SSLContext.') + + +def test_main(verbose=False): + if support.verbose: + import warnings + plats = { + 'Linux': platform.linux_distribution, + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', + r'dist\(\) and linux_distribution\(\) ' + 'functions are deprecated .*', + PendingDeprecationWarning, + ) + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + tests = [ + ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests, + SimpleBackgroundTests, + ] + + if support.is_resource_enabled('network'): + tests.append(NetworkedTests) + + if _have_threads: + thread_info = support.threading_setup() + if thread_info: + tests.append(ThreadedTests) + + try: + support.run_unittest(*tests) + finally: + if _have_threads: + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.6/test_subprocess.py b/src/greentest/3.6/test_subprocess.py new file mode 100644 index 0000000..2db0503 --- /dev/null +++ b/src/greentest/3.6/test_subprocess.py @@ -0,0 +1,3001 @@ +import unittest +from unittest import mock +from test import support +import subprocess +import sys +import platform +import signal +import io +import os +import errno +import tempfile +import time +import selectors +import sysconfig +import select +import shutil +import gc +import textwrap + +try: + import ctypes +except ImportError: + ctypes = None +else: + import ctypes.util + +try: + import threading +except ImportError: + threading = None + +try: + import _testcapi +except ImportError: + _testcapi = None + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +mswindows = (sys.platform == "win32") + +# +# Depends on the following external programs: Python +# + +if mswindows: + SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), ' + 'os.O_BINARY);') +else: + SETBINARY = '' + +NONEXISTING_CMD = ('nonexisting_i_hope',) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + # Try to minimize the number of children we have so this test + # doesn't crash on some buildbots (Alphas in particular). + support.reap_children() + + def tearDown(self): + for inst in subprocess._active: + inst.wait() + subprocess._cleanup() + self.assertFalse(subprocess._active, "subprocess._active not empty") + self.doCleanups() + support.reap_children() + + def assertStderrEqual(self, stderr, expected, msg=None): + # In a debug build, stuff like "[6580 refs]" is printed to stderr at + # shutdown time. That frustrates tests trying to check stderr produced + # from a spawned Python process. + actual = support.strip_python_stderr(stderr) + # strip_python_stderr also strips whitespace, so we do too. + expected = expected.strip() + self.assertEqual(actual, expected, msg) + + +class PopenTestException(Exception): + pass + + +class PopenExecuteChildRaises(subprocess.Popen): + """Popen subclass for testing cleanup of subprocess.PIPE filehandles when + _execute_child fails. + """ + def _execute_child(self, *args, **kwargs): + raise PopenTestException("Forced Exception for Test") + + +class ProcessTestCase(BaseTestCase): + + def test_io_buffered_by_default(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + self.assertIsInstance(p.stdin, io.BufferedIOBase) + self.assertIsInstance(p.stdout, io.BufferedIOBase) + self.assertIsInstance(p.stderr, io.BufferedIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_io_unbuffered_works(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0) + try: + self.assertIsInstance(p.stdin, io.RawIOBase) + self.assertIsInstance(p.stdout, io.RawIOBase) + self.assertIsInstance(p.stderr, io.RawIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_call_seq(self): + # call() function with sequence argument + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(rc, 47) + + def test_call_timeout(self): + # call() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.call waits for the + # child. + self.assertRaises(subprocess.TimeoutExpired, subprocess.call, + [sys.executable, "-c", "while True: pass"], + timeout=0.1) + + def test_check_call_zero(self): + # check_call() function with zero return code + rc = subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(0)"]) + self.assertEqual(rc, 0) + + def test_check_call_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(c.exception.returncode, 47) + + def test_check_output(self): + # check_output() function with zero return code + output = subprocess.check_output( + [sys.executable, "-c", "print('BDFL')"]) + self.assertIn(b'BDFL', output) + + def test_check_output_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_output( + [sys.executable, "-c", "import sys; sys.exit(5)"]) + self.assertEqual(c.exception.returncode, 5) + + def test_check_output_stderr(self): + # check_output() function stderr redirected to stdout + output = subprocess.check_output( + [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], + stderr=subprocess.STDOUT) + self.assertIn(b'BDFL', output) + + def test_check_output_stdin_arg(self): + # check_output() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + stdin=tf) + self.assertIn(b'PEAR', output) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + input=b'pear') + self.assertIn(b'PEAR', output) + + def test_check_output_stdout_arg(self): + # check_output() refuses to accept 'stdout' argument + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdout=sys.stdout) + self.fail("Expected ValueError when stdout arg supplied.") + self.assertIn('stdout', c.exception.args[0]) + + def test_check_output_stdin_with_input_arg(self): + # check_output() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdin=tf, input=b'hare') + self.fail("Expected ValueError when stdin and input args supplied.") + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + # check_output() function with timeout arg + with self.assertRaises(subprocess.TimeoutExpired) as c: + output = subprocess.check_output( + [sys.executable, "-c", + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"], + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3) + self.fail("Expected TimeoutExpired.") + self.assertEqual(c.exception.output, b'BDFL') + + def test_call_kwargs(self): + # call() function with keyword args + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + rc = subprocess.call([sys.executable, "-c", + 'import sys, os;' + 'sys.exit(os.getenv("FRUIT")=="banana")'], + env=newenv) + self.assertEqual(rc, 1) + + def test_invalid_args(self): + # Popen() called with invalid arguments should raise TypeError + # but Popen.__del__ should not complain (issue #12085) + with support.captured_stderr() as s: + self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) + argcount = subprocess.Popen.__init__.__code__.co_argcount + too_many_args = [0] * (argcount + 1) + self.assertRaises(TypeError, subprocess.Popen, *too_many_args) + self.assertEqual(s.getvalue(), '') + + def test_stdin_none(self): + # .stdin is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + p.wait() + self.assertEqual(p.stdin, None) + + def test_stdout_none(self): + # .stdout is None when not redirected, and the child's stdout will + # be inherited from the parent. In order to test this we run a + # subprocess in a subprocess: + # this_test + # \-- subprocess created by this test (parent) + # \-- subprocess created by the parent subprocess (child) + # The parent doesn't specify stdout, so the child will use the + # parent's stdout. This test checks that the message printed by the + # child goes to the parent stdout. The parent also checks that the + # child's stdout is None. See #11963. + code = ('import sys; from subprocess import Popen, PIPE;' + 'p = Popen([sys.executable, "-c", "print(\'test_stdout_none\')"],' + ' stdin=PIPE, stderr=PIPE);' + 'p.wait(); assert p.stdout is None;') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test_stdout_none') + + def test_stderr_none(self): + # .stderr is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + p.wait() + self.assertEqual(p.stderr, None) + + def _assert_python(self, pre_args, **kwargs): + # We include sys.exit() to prevent the test runner from hanging + # whenever python is found. + args = pre_args + ["import sys; sys.exit(47)"] + p = subprocess.Popen(args, **kwargs) + p.wait() + self.assertEqual(47, p.returncode) + + def test_executable(self): + # Check that the executable argument works. + # + # On Unix (non-Mac and non-Windows), Python looks at args[0] to + # determine where its standard library is, so we need the directory + # of args[0] to be valid for the Popen() call to Python to succeed. + # See also issue #16170 and issue #7774. + doesnotexist = os.path.join(os.path.dirname(sys.executable), + "doesnotexist") + self._assert_python([doesnotexist, "-c"], executable=sys.executable) + + def test_executable_takes_precedence(self): + # Check that the executable argument takes precedence over args[0]. + # + # Verify first that the call succeeds without the executable arg. + pre_args = [sys.executable, "-c"] + self._assert_python(pre_args) + self.assertRaises((FileNotFoundError, PermissionError), + self._assert_python, pre_args, + executable="doesnotexist") + + @unittest.skipIf(mswindows, "executable argument replaces shell") + def test_executable_replaces_shell(self): + # Check that the executable argument replaces the default shell + # when shell=True. + self._assert_python([], executable=sys.executable, shell=True) + + # For use in the test_cwd* tests below. + def _normalize_cwd(self, cwd): + # Normalize an expected cwd (for Tru64 support). + # We can't use os.path.realpath since it doesn't expand Tru64 {memb} + # strings. See bug #1063571. + with support.change_cwd(cwd): + return os.getcwd() + + # For use in the test_cwd* tests below. + def _split_python_path(self): + # Return normalized (python_dir, python_base). + python_path = os.path.realpath(sys.executable) + return os.path.split(python_path) + + # For use in the test_cwd* tests below. + def _assert_cwd(self, expected_cwd, python_arg, **kwargs): + # Invoke Python via Popen, and assert that (1) the call succeeds, + # and that (2) the current working directory of the child process + # matches *expected_cwd*. + p = subprocess.Popen([python_arg, "-c", + "import os, sys; " + "sys.stdout.write(os.getcwd()); " + "sys.exit(47)"], + stdout=subprocess.PIPE, + **kwargs) + self.addCleanup(p.stdout.close) + p.wait() + self.assertEqual(47, p.returncode) + normcase = os.path.normcase + self.assertEqual(normcase(expected_cwd), + normcase(p.stdout.read().decode("utf-8"))) + + def test_cwd(self): + # Check that cwd changes the cwd for the child process. + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + + def test_cwd_with_pathlike(self): + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + + class _PathLikeObj: + def __fspath__(self): + return temp_dir + + self._assert_cwd(temp_dir, sys.executable, cwd=_PathLikeObj()) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_arg(self): + # Check that Popen looks for args[0] relative to cwd if args[0] + # is relative. + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + with support.temp_cwd('test_cwd_with_relative_arg', quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python]) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, rel_python, cwd=python_dir) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_executable(self): + # Check that Popen looks for executable relative to cwd if executable + # is relative (and that executable takes precedence over args[0]). + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + doesntexist = "somethingyoudonthave" + with support.temp_cwd('test_cwd_with_relative_executable', quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python, + cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, doesntexist, executable=rel_python, + cwd=python_dir) + + def test_cwd_with_absolute_arg(self): + # Check that Popen can find the executable when the cwd is wrong + # if args[0] is an absolute path. + python_dir, python_base = self._split_python_path() + abs_python = os.path.join(python_dir, python_base) + rel_python = os.path.join(os.curdir, python_base) + with support.temp_dir() as wrong_dir: + # Before calling with an absolute path, confirm that using a + # relative path fails. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + wrong_dir = self._normalize_cwd(wrong_dir) + self._assert_cwd(wrong_dir, abs_python, cwd=wrong_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + def test_executable_with_cwd(self): + python_dir, python_base = self._split_python_path() + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, "somethingyoudonthave", + executable=sys.executable, cwd=python_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + @unittest.skipIf(sysconfig.is_python_build(), + "need an installed Python. See #7774") + def test_executable_without_cwd(self): + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + self._assert_cwd(os.getcwd(), "somethingyoudonthave", + executable=sys.executable) + + def test_stdin_pipe(self): + # stdin redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.stdin.write(b"pear") + p.stdin.close() + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_filedes(self): + # stdin is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + os.write(d, b"pear") + os.lseek(d, 0, 0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=d) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_fileobj(self): + # stdin is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b"pear") + tf.seek(0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=tf) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdout_pipe(self): + # stdout redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=subprocess.PIPE) + with p: + self.assertEqual(p.stdout.read(), b"orange") + + def test_stdout_filedes(self): + # stdout is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=d) + p.wait() + os.lseek(d, 0, 0) + self.assertEqual(os.read(d, 1024), b"orange") + + def test_stdout_fileobj(self): + # stdout is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=tf) + p.wait() + tf.seek(0) + self.assertEqual(tf.read(), b"orange") + + def test_stderr_pipe(self): + # stderr redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=subprocess.PIPE) + with p: + self.assertStderrEqual(p.stderr.read(), b"strawberry") + + def test_stderr_filedes(self): + # stderr is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=d) + p.wait() + os.lseek(d, 0, 0) + self.assertStderrEqual(os.read(d, 1024), b"strawberry") + + def test_stderr_fileobj(self): + # stderr is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"strawberry") + + def test_stderr_redirect_with_no_stdout_redirect(self): + # test stderr=STDOUT while stdout=None (not set) + + # - grandchild prints to stderr + # - child redirects grandchild's stderr to its stdout + # - the parent should get grandchild's stderr in child's stdout + p = subprocess.Popen([sys.executable, "-c", + 'import sys, subprocess;' + 'rc = subprocess.call([sys.executable, "-c",' + ' "import sys;"' + ' "sys.stderr.write(\'42\')"],' + ' stderr=subprocess.STDOUT);' + 'sys.exit(rc)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + #NOTE: stdout should get stderr from grandchild + self.assertStderrEqual(stdout, b'42') + self.assertStderrEqual(stderr, b'') # should be empty + self.assertEqual(p.returncode, 0) + + def test_stdout_stderr_pipe(self): + # capture stdout and stderr to the same pipe + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + with p: + self.assertStderrEqual(p.stdout.read(), b"appleorange") + + def test_stdout_stderr_file(self): + # capture stdout and stderr to the same open file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=tf, + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"appleorange") + + def test_stdout_filedes_of_stdout(self): + # stdout is set to 1 (#1531862). + # To avoid printing the text on stdout, we do something similar to + # test_stdout_none (see above). The parent subprocess calls the child + # subprocess passing stdout=1, and this test uses stdout=PIPE in + # order to capture and check the output of the parent. See #11963. + code = ('import sys, subprocess; ' + 'rc = subprocess.call([sys.executable, "-c", ' + ' "import os, sys; sys.exit(os.write(sys.stdout.fileno(), ' + 'b\'test with stdout=1\'))"], stdout=1); ' + 'assert rc == 18') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test with stdout=1') + + def test_stdout_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'for i in range(10240):' + 'print("x" * 1024)'], + stdout=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdout, None) + + def test_stderr_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys\n' + 'for i in range(10240):' + 'sys.stderr.write("x" * 1024)'], + stderr=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stderr, None) + + def test_stdin_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdin.read(1)'], + stdin=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdin, None) + + def test_env(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "orange" + with subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange") + + # Windows requires at least the SYSTEMROOT environment variable to start + # Python + @unittest.skipIf(sys.platform == 'win32', + 'cannot test an empty env on Windows') + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') is not None, + 'the python library cannot be loaded ' + 'with an empty environment') + def test_empty_env(self): + with subprocess.Popen([sys.executable, "-c", + 'import os; ' + 'print(list(os.environ.keys()))'], + stdout=subprocess.PIPE, + env={}) as p: + stdout, stderr = p.communicate() + self.assertIn(stdout.strip(), + (b"[]", + # Mac OS X adds __CF_USER_TEXT_ENCODING variable to an empty + # environment + b"['__CF_USER_TEXT_ENCODING']")) + + def test_invalid_cmd(self): + # null character in the command name + cmd = sys.executable + '\0' + with self.assertRaises(ValueError): + subprocess.Popen([cmd, "-c", "pass"]) + + # null character in the command argument + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass#\0"]) + + def test_invalid_env(self): + # null character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # null character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the enviroment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + with subprocess.Popen([sys.executable, "-c", + 'import sys, os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange=lemon") + + def test_communicate_stdin(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.communicate(b"pear") + self.assertEqual(p.returncode, 1) + + def test_communicate_stdout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("pineapple")'], + stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, b"pineapple") + self.assertEqual(stderr, None) + + def test_communicate_stderr(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("pineapple")'], + stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple");' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(b"banana") + self.assertEqual(stdout, b"banana") + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate_timeout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stderr.write("pineapple\\n");' + 'time.sleep(1);' + 'sys.stderr.write("pear\\n");' + 'sys.stdout.write(sys.stdin.read())'], + universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, "banana", + timeout=0.3) + # Make sure we can keep waiting for it, and that we get the whole output + # after it completes. + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr.encode(), b"pineapple\npear\n") + + def test_communicate_timeout_large_output(self): + # Test an expiring timeout while the child is outputting lots of data. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));'], + stdout=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, timeout=0.4) + (stdout, _) = p.communicate() + self.assertEqual(len(stdout), 4 * 64 * 1024) + + # Test for the fd leak reported in http://bugs.python.org/issue2791. + def test_communicate_pipe_fd_leak(self): + for stdin_pipe in (False, True): + for stdout_pipe in (False, True): + for stderr_pipe in (False, True): + options = {} + if stdin_pipe: + options['stdin'] = subprocess.PIPE + if stdout_pipe: + options['stdout'] = subprocess.PIPE + if stderr_pipe: + options['stderr'] = subprocess.PIPE + if not options: + continue + p = subprocess.Popen((sys.executable, "-c", "pass"), **options) + p.communicate() + if p.stdin is not None: + self.assertTrue(p.stdin.closed) + if p.stdout is not None: + self.assertTrue(p.stdout.closed) + if p.stderr is not None: + self.assertTrue(p.stderr.closed) + + def test_communicate_returns(self): + # communicate() should return None if no redirection is active + p = subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(47)"]) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertEqual(stderr, None) + + def test_communicate_pipe_buf(self): + # communicate() with writes larger than pipe_buf + # This test will probably deadlock rather than fail, if + # communicate() does not work properly. + x, y = os.pipe() + os.close(x) + os.close(y) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read(47));' + 'sys.stderr.write("x" * %d);' + 'sys.stdout.write(sys.stdin.read())' % + support.PIPE_MAX_SIZE], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + string_to_write = b"a" * support.PIPE_MAX_SIZE + (stdout, stderr) = p.communicate(string_to_write) + self.assertEqual(stdout, string_to_write) + + def test_writes_before_communicate(self): + # stdin.write before communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.stdin.write(b"banana") + (stdout, stderr) = p.communicate(b"split") + self.assertEqual(stdout, b"bananasplit") + self.assertStderrEqual(stderr, b"") + + def test_universal_newlines(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(sys.stdin.readline().encode());' + 'buf.flush();' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(sys.stdin.read().encode());' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + with p: + p.stdin.write("line1\n") + p.stdin.flush() + self.assertEqual(p.stdout.readline(), "line1\n") + p.stdin.write("line3\n") + p.stdin.close() + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.readline(), + "line2\n") + self.assertEqual(p.stdout.read(6), + "line3\n") + self.assertEqual(p.stdout.read(), + "line4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate(self): + # universal newlines through communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, + "line2\nline4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate_stdin(self): + # universal newlines through communicate(), with only stdin + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.readline() + assert s == "line1\\n", repr(s) + s = sys.stdin.read() + assert s == "line3\\n", repr(s) + ''')], + stdin=subprocess.PIPE, + universal_newlines=1) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_input_none(self): + # Test communicate(input=None) with universal newlines. + # + # We set stdout to PIPE because, as of this writing, a different + # code path is tested when the number of pipes is zero or one. + p = subprocess.Popen([sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + p.communicate() + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_stdin_stdout_stderr(self): + # universal newlines through communicate(), with stdin, stdout, stderr + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.buffer.readline() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line2\\r") + sys.stderr.buffer.write(b"eline2\\n") + s = sys.stdin.buffer.read() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line4\\n") + sys.stdout.buffer.write(b"line5\\r\\n") + sys.stderr.buffer.write(b"eline6\\r") + sys.stderr.buffer.write(b"eline7\\r\\nz") + ''')], + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) + # Python debug build push something like "[42442 refs]\n" + # to stderr at exit of subprocess. + # Don't use assertStderrEqual because it strips CR and LF from output. + self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + + def test_universal_newlines_communicate_encodings(self): + # Check that universal newlines mode works for various encodings, + # in particular for encodings in the UTF-16 and UTF-32 families. + # See issue #15595. + # + # UTF-16 and UTF-32-BE are sufficient to check both with BOM and + # without, and UTF-16 and UTF-32. + for encoding in ['utf-16', 'utf-32-be']: + code = ("import sys; " + r"sys.stdout.buffer.write('1\r\n2\r3\n4'.encode('%s'))" % + encoding) + args = [sys.executable, '-c', code] + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding=encoding) + stdout, stderr = popen.communicate(input='') + self.assertEqual(stdout, '1\n2\n3\n4') + + def test_communicate_errors(self): + for errors, expected in [ + ('ignore', ''), + ('replace', '\ufffd\ufffd'), + ('surrogateescape', '\udc80\udc80'), + ('backslashreplace', '\\x80\\x80'), + ]: + code = ("import sys; " + r"sys.stdout.buffer.write(b'[\x80\x80]')") + args = [sys.executable, '-c', code] + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding='utf-8', + errors=errors) + stdout, stderr = popen.communicate(input='') + self.assertEqual(stdout, '[{}]'.format(expected)) + + def test_no_leaking(self): + # Make sure we leak no resources + if not mswindows: + max_handles = 1026 # too much for most UNIX systems + else: + max_handles = 2050 # too much for (at least some) Windows setups + handles = [] + tmpdir = tempfile.mkdtemp() + try: + for i in range(max_handles): + try: + tmpfile = os.path.join(tmpdir, support.TESTFN) + handles.append(os.open(tmpfile, os.O_WRONLY|os.O_CREAT)) + except OSError as e: + if e.errno != errno.EMFILE: + raise + break + else: + self.skipTest("failed to reach the file descriptor limit " + "(tried %d)" % max_handles) + # Close a couple of them (should be enough for a subprocess) + for i in range(10): + os.close(handles.pop()) + # Loop creating some subprocesses. If one of them leaks some fds, + # the next loop iteration will fail by reaching the max fd limit. + for i in range(15): + p = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write(sys.stdin.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + data = p.communicate(b"lime")[0] + self.assertEqual(data, b"lime") + finally: + for h in handles: + os.close(h) + shutil.rmtree(tmpdir) + + def test_list2cmdline(self): + self.assertEqual(subprocess.list2cmdline(['a b c', 'd', 'e']), + '"a b c" d e') + self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']), + 'ab\\"c \\ d') + self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']), + 'ab\\"c " \\\\" d') + self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']), + 'a\\\\\\b "de fg" h') + self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']), + 'a\\\\\\"b c d') + self.assertEqual(subprocess.list2cmdline(['a\\\\b c', 'd', 'e']), + '"a\\\\b c" d e') + self.assertEqual(subprocess.list2cmdline(['a\\\\b\\ c', 'd', 'e']), + '"a\\\\b\\ c" d e') + self.assertEqual(subprocess.list2cmdline(['ab', '']), + 'ab ""') + + def test_poll(self): + p = subprocess.Popen([sys.executable, "-c", + "import os; os.read(0, 1)"], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + self.assertIsNone(p.poll()) + os.write(p.stdin.fileno(), b'A') + p.wait() + # Subsequent invocations should just return the returncode + self.assertEqual(p.poll(), 0) + + def test_wait(self): + p = subprocess.Popen([sys.executable, "-c", "pass"]) + self.assertEqual(p.wait(), 0) + # Subsequent invocations should just return the returncode + self.assertEqual(p.wait(), 0) + + def test_wait_timeout(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(0.3)"]) + with self.assertRaises(subprocess.TimeoutExpired) as c: + p.wait(timeout=0.0001) + self.assertIn("0.0001", str(c.exception)) # For coverage of __str__. + # Some heavily loaded buildbots (sparc Debian 3.x) require this much + # time to start. + self.assertEqual(p.wait(timeout=3), 0) + + def test_wait_endtime(self): + """Confirm that the deprecated endtime parameter warns.""" + p = subprocess.Popen([sys.executable, "-c", "pass"]) + try: + with self.assertWarns(DeprecationWarning) as warn_cm: + p.wait(endtime=time.time()+0.01) + except subprocess.TimeoutExpired: + pass # We're not testing endtime timeout behavior. + finally: + p.kill() + self.assertIn('test_subprocess.py', warn_cm.filename) + self.assertIn('endtime', str(warn_cm.warning)) + + def test_invalid_bufsize(self): + # an invalid type of the bufsize argument should raise + # TypeError. + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], "orange") + + def test_bufsize_is_none(self): + # bufsize=None should be the same as bufsize=0. + p = subprocess.Popen([sys.executable, "-c", "pass"], None) + self.assertEqual(p.wait(), 0) + # Again with keyword arg + p = subprocess.Popen([sys.executable, "-c", "pass"], bufsize=None) + self.assertEqual(p.wait(), 0) + + def _test_bufsize_equal_one(self, line, expected, universal_newlines): + # subprocess may deadlock with bufsize=1, see issue #21332 + with subprocess.Popen([sys.executable, "-c", "import sys;" + "sys.stdout.write(sys.stdin.readline());" + "sys.stdout.flush()"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + bufsize=1, + universal_newlines=universal_newlines) as p: + p.stdin.write(line) # expect that it flushes the line in text mode + os.close(p.stdin.fileno()) # close it without flushing the buffer + read_line = p.stdout.readline() + with support.SuppressCrashReport(): + try: + p.stdin.close() + except OSError: + pass + p.stdin = None + self.assertEqual(p.returncode, 0) + self.assertEqual(read_line, expected) + + def test_bufsize_equal_one_text_mode(self): + # line is flushed in text mode with bufsize=1. + # we should get the full line in return + line = "line\n" + self._test_bufsize_equal_one(line, line, universal_newlines=True) + + def test_bufsize_equal_one_binary_mode(self): + # line is not flushed in binary mode with bufsize=1. + # we should get empty response + line = b'line' + os.linesep.encode() # assume ascii-based locale + self._test_bufsize_equal_one(line, b'', universal_newlines=False) + + def test_leaking_fds_on_error(self): + # see bug #5179: Popen leaks file descriptors to PIPEs if + # the child fails to execute; this will eventually exhaust + # the maximum number of open fds. 1024 seems a very common + # value for that limit, but Windows has 2048, so we loop + # 1024 times (each call leaked two fds). + for i in range(1024): + with self.assertRaises(OSError) as c: + subprocess.Popen(NONEXISTING_CMD, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # ignore errors that indicate the command was not found + if c.exception.errno not in (errno.ENOENT, errno.EACCES): + raise c.exception + + def test_nonexisting_with_pipes(self): + # bpo-30121: Popen with pipes must close properly pipes on error. + # Previously, os.close() was called with a Windows handle which is not + # a valid file descriptor. + # + # Run the test in a subprocess to control how the CRT reports errors + # and to get stderr content. + try: + import msvcrt + msvcrt.CrtSetReportMode + except (AttributeError, ImportError): + self.skipTest("need msvcrt.CrtSetReportMode") + + code = textwrap.dedent(f""" + import msvcrt + import subprocess + + cmd = {NONEXISTING_CMD!r} + + for report_type in [msvcrt.CRT_WARN, + msvcrt.CRT_ERROR, + msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(report_type, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(report_type, msvcrt.CRTDBG_FILE_STDERR) + + try: + subprocess.Popen([cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError: + pass + """) + cmd = [sys.executable, "-c", code] + proc = subprocess.Popen(cmd, + stderr=subprocess.PIPE, + universal_newlines=True) + with proc: + stderr = proc.communicate()[1] + self.assertEqual(stderr, "") + self.assertEqual(proc.returncode, 0) + + @unittest.skipIf(threading is None, "threading required") + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(NONEXISTING_CMD, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + + @unittest.skipIf(threading is None, "threading required") + def test_threadsafe_wait(self): + """Issue21291: Popen.wait() needs to be threadsafe for returncode.""" + proc = subprocess.Popen([sys.executable, '-c', + 'import time; time.sleep(12)']) + self.assertEqual(proc.returncode, None) + results = [] + + def kill_proc_timer_thread(): + results.append(('thread-start-poll-result', proc.poll())) + # terminate it from the thread and wait for the result. + proc.kill() + proc.wait() + results.append(('thread-after-kill-and-wait', proc.returncode)) + # this wait should be a no-op given the above. + proc.wait() + results.append(('thread-after-second-wait', proc.returncode)) + + # This is a timing sensitive test, the failure mode is + # triggered when both the main thread and this thread are in + # the wait() call at once. The delay here is to allow the + # main thread to most likely be blocked in its wait() call. + t = threading.Timer(0.2, kill_proc_timer_thread) + t.start() + + if mswindows: + expected_errorcode = 1 + else: + # Should be -9 because of the proc.kill() from the thread. + expected_errorcode = -9 + + # Wait for the process to finish; the thread should kill it + # long before it finishes on its own. Supplying a timeout + # triggers a different code path for better coverage. + proc.wait(timeout=20) + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in wait from main thread") + + # This should be a no-op with no change in returncode. + proc.wait() + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in second main wait.") + + t.join() + # Ensure that all of the thread results are as expected. + # When a race condition occurs in wait(), the returncode could + # be set by the wrong thread that doesn't actually have it + # leading to an incorrect value. + self.assertEqual([('thread-start-poll-result', None), + ('thread-after-kill-and-wait', expected_errorcode), + ('thread-after-second-wait', expected_errorcode)], + results) + + def test_issue8780(self): + # Ensure that stdout is inherited from the parent + # if stdout=PIPE is not used + code = ';'.join(( + 'import subprocess, sys', + 'retcode = subprocess.call(' + "[sys.executable, '-c', 'print(\"Hello World!\")'])", + 'assert retcode == 0')) + output = subprocess.check_output([sys.executable, '-c', code]) + self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + + def test_handles_closed_on_exception(self): + # If CreateProcess exits with an error, ensure the + # duplicate output handles are released + ifhandle, ifname = tempfile.mkstemp() + ofhandle, ofname = tempfile.mkstemp() + efhandle, efname = tempfile.mkstemp() + try: + subprocess.Popen (["*"], stdin=ifhandle, stdout=ofhandle, + stderr=efhandle) + except OSError: + os.close(ifhandle) + os.remove(ifname) + os.close(ofhandle) + os.remove(ofname) + os.close(efhandle) + os.remove(efname) + self.assertFalse(os.path.exists(ifname)) + self.assertFalse(os.path.exists(ofname)) + self.assertFalse(os.path.exists(efname)) + + def test_communicate_epipe(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.communicate(b"x" * 2**20) + + def test_communicate_epipe_only_stdin(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + p.wait() + p.communicate(b"x" * 2**20) + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), + "Requires signal.SIGUSR1") + @unittest.skipUnless(hasattr(os, 'kill'), + "Requires os.kill") + @unittest.skipUnless(hasattr(os, 'getppid'), + "Requires os.getppid") + def test_communicate_eintr(self): + # Issue #12493: communicate() should handle EINTR + def handler(signum, frame): + pass + old_handler = signal.signal(signal.SIGUSR1, handler) + self.addCleanup(signal.signal, signal.SIGUSR1, old_handler) + + args = [sys.executable, "-c", + 'import os, signal;' + 'os.kill(os.getppid(), signal.SIGUSR1)'] + for stream in ('stdout', 'stderr'): + kw = {stream: subprocess.PIPE} + with subprocess.Popen(args, **kw) as process: + # communicate() will be interrupted by SIGUSR1 + process.communicate() + + + # This test is Linux-ish specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + def test_failed_child_execute_fd_leak(self): + """Test for the fork() failure fd leak reported in issue16327.""" + fd_directory = '/proc/%d/fd' % os.getpid() + fds_before_popen = os.listdir(fd_directory) + with self.assertRaises(PopenTestException): + PopenExecuteChildRaises( + [sys.executable, '-c', 'pass'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # NOTE: This test doesn't verify that the real _execute_child + # does not close the file descriptors itself on the way out + # during an exception. Code inspection has confirmed that. + + fds_after_exception = os.listdir(fd_directory) + self.assertEqual(fds_before_popen, fds_after_exception) + + @unittest.skipIf(mswindows, "behavior currently not supported on Windows") + def test_file_not_found_includes_filename(self): + with self.assertRaises(FileNotFoundError) as c: + subprocess.call(['/opt/nonexistent_binary', 'with', 'some', 'args']) + self.assertEqual(c.exception.filename, '/opt/nonexistent_binary') + + @unittest.skipIf(mswindows, "behavior currently not supported on Windows") + def test_file_not_found_with_bad_cwd(self): + with self.assertRaises(FileNotFoundError) as c: + subprocess.Popen(['exit', '0'], cwd='/some/nonexistent/directory') + self.assertEqual(c.exception.filename, '/some/nonexistent/directory') + + +class RunFuncTestCase(BaseTestCase): + def run_python(self, code, **kwargs): + """Run Python code in a subprocess using subprocess.run""" + argv = [sys.executable, "-c", code] + return subprocess.run(argv, **kwargs) + + def test_returncode(self): + # call() function with sequence argument + cp = self.run_python("import sys; sys.exit(47)") + self.assertEqual(cp.returncode, 47) + with self.assertRaises(subprocess.CalledProcessError): + cp.check_returncode() + + def test_check(self): + with self.assertRaises(subprocess.CalledProcessError) as c: + self.run_python("import sys; sys.exit(47)", check=True) + self.assertEqual(c.exception.returncode, 47) + + def test_check_zero(self): + # check_returncode shouldn't raise when returncode is zero + cp = self.run_python("import sys; sys.exit(0)", check=True) + self.assertEqual(cp.returncode, 0) + + def test_timeout(self): + # run() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.run waits for the + # child. + with self.assertRaises(subprocess.TimeoutExpired): + self.run_python("while True: pass", timeout=0.0001) + + def test_capture_stdout(self): + # capture stdout with zero return code + cp = self.run_python("print('BDFL')", stdout=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stdout) + + def test_capture_stderr(self): + cp = self.run_python("import sys; sys.stderr.write('BDFL')", + stderr=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stderr) + + def test_check_output_stdin_arg(self): + # run() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + stdin=tf, stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + input=b'pear', stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_stdin_with_input_arg(self): + # run() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError, + msg="Expected ValueError when stdin and input args supplied.") as c: + output = self.run_python("print('will not be run')", + stdin=tf, input=b'hare') + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + with self.assertRaises(subprocess.TimeoutExpired) as c: + cp = self.run_python(( + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"), + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3, stdout=subprocess.PIPE) + self.assertEqual(c.exception.output, b'BDFL') + # output is aliased to stdout + self.assertEqual(c.exception.stdout, b'BDFL') + + def test_run_kwargs(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + cp = self.run_python(('import sys, os;' + 'sys.exit(33 if os.getenv("FRUIT")=="banana" else 31)'), + env=newenv) + self.assertEqual(cp.returncode, 33) + + +@unittest.skipIf(mswindows, "POSIX specific tests") +class POSIXProcessTestCase(BaseTestCase): + + def setUp(self): + super().setUp() + self._nonexistent_dir = "/_this/pa.th/does/not/exist" + + def _get_chdir_exception(self): + try: + os.chdir(self._nonexistent_dir) + except OSError as e: + # This avoids hard coding the errno value or the OS perror() + # string and instead capture the exception that we want to see + # below for comparison. + desired_exception = e + desired_exception.strerror += ': ' + repr(self._nonexistent_dir) + else: + self.fail("chdir to nonexistent directory %s succeeded." % + self._nonexistent_dir) + return desired_exception + + def test_exception_cwd(self): + """Test error in the child raised in the parent for a bad cwd.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + cwd=self._nonexistent_dir) + except OSError as e: + # Test that the child process chdir failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_executable(self): + """Test error in the child raised in the parent for a bad executable.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + executable=self._nonexistent_dir) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_args_0(self): + """Test error in the child raised in the parent for a bad args[0].""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([self._nonexistent_dir, "-c", ""]) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + # We mock the __del__ method for Popen in the next two tests + # because it does cleanup based on the pid returned by fork_exec + # along with issuing a resource warning if it still exists. Since + # we don't actually spawn a process in these tests we can forego + # the destructor. An alternative would be to set _child_created to + # False before the destructor is called but there is no easy way + # to do that + class PopenNoDestructor(subprocess.Popen): + def __del__(self): + pass + + @mock.patch("subprocess._posixsubprocess.fork_exec") + def test_exception_errpipe_normal(self, fork_exec): + """Test error passing done through errpipe_write in the good case""" + def proper_error(*args): + errpipe_write = args[13] + # Write the hex for the error code EISDIR: 'is a directory' + err_code = '{:x}'.format(errno.EISDIR).encode() + os.write(errpipe_write, b"OSError:" + err_code + b":") + return 0 + + fork_exec.side_effect = proper_error + + with self.assertRaises(IsADirectoryError): + self.PopenNoDestructor(["non_existent_command"]) + + @mock.patch("subprocess._posixsubprocess.fork_exec") + def test_exception_errpipe_bad_data(self, fork_exec): + """Test error passing done through errpipe_write where its not + in the expected format""" + error_data = b"\xFF\x00\xDE\xAD" + def bad_error(*args): + errpipe_write = args[13] + # Anything can be in the pipe, no assumptions should + # be made about its encoding, so we'll write some + # arbitrary hex bytes to test it out + os.write(errpipe_write, error_data) + return 0 + + fork_exec.side_effect = bad_error + + with self.assertRaises(subprocess.SubprocessError) as e: + self.PopenNoDestructor(["non_existent_command"]) + + self.assertIn(repr(error_data), str(e.exception)) + + + def test_restore_signals(self): + # Code coverage for both values of restore_signals to make sure it + # at least does not blow up. + # A test for behavior would be complex. Contributions welcome. + subprocess.call([sys.executable, "-c", ""], restore_signals=True) + subprocess.call([sys.executable, "-c", ""], restore_signals=False) + + def test_start_new_session(self): + # For code coverage of calling setsid(). We don't care if we get an + # EPERM error from it depending on the test execution environment, that + # still indicates that it was called. + try: + output = subprocess.check_output( + [sys.executable, "-c", + "import os; print(os.getpgid(os.getpid()))"], + start_new_session=True) + except OSError as e: + if e.errno != errno.EPERM: + raise + else: + parent_pgid = os.getpgid(os.getpid()) + child_pgid = int(output) + self.assertNotEqual(parent_pgid, child_pgid) + + def test_run_abort(self): + # returncode handles signal termination + with support.SuppressCrashReport(): + p = subprocess.Popen([sys.executable, "-c", + 'import os; os.abort()']) + p.wait() + self.assertEqual(-p.returncode, signal.SIGABRT) + + def test_CalledProcessError_str_signal(self): + err = subprocess.CalledProcessError(-int(signal.SIGABRT), "fake cmd") + error_string = str(err) + # We're relying on the repr() of the signal.Signals intenum to provide + # the word signal, the signal name and the numeric value. + self.assertIn("signal", error_string.lower()) + # We're not being specific about the signal name as some signals have + # multiple names and which name is revealed can vary. + self.assertIn("SIG", error_string) + self.assertIn(str(signal.SIGABRT), error_string) + + def test_CalledProcessError_str_unknown_signal(self): + err = subprocess.CalledProcessError(-9876543, "fake cmd") + error_string = str(err) + self.assertIn("unknown signal 9876543.", error_string) + + def test_CalledProcessError_str_non_zero(self): + err = subprocess.CalledProcessError(2, "fake cmd") + error_string = str(err) + self.assertIn("non-zero exit status 2.", error_string) + + def test_preexec(self): + # DISCLAIMER: Setting environment variables is *not* a good use + # of a preexec_fn. This is merely a test. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + preexec_fn=lambda: os.putenv("FRUIT", "apple")) + with p: + self.assertEqual(p.stdout.read(), b"apple") + + def test_preexec_exception(self): + def raise_it(): + raise ValueError("What if two swallows carried a coconut?") + try: + p = subprocess.Popen([sys.executable, "-c", ""], + preexec_fn=raise_it) + except subprocess.SubprocessError as e: + self.assertTrue( + subprocess._posixsubprocess, + "Expected a ValueError from the preexec_fn") + except ValueError as e: + self.assertIn("coconut", e.args[0]) + else: + self.fail("Exception raised by preexec_fn did not make it " + "to the parent process.") + + class _TestExecuteChildPopen(subprocess.Popen): + """Used to test behavior at the end of _execute_child.""" + def __init__(self, testcase, *args, **kwargs): + self._testcase = testcase + subprocess.Popen.__init__(self, *args, **kwargs) + + def _execute_child(self, *args, **kwargs): + try: + subprocess.Popen._execute_child(self, *args, **kwargs) + finally: + # Open a bunch of file descriptors and verify that + # none of them are the same as the ones the Popen + # instance is using for stdin/stdout/stderr. + devzero_fds = [os.open("/dev/zero", os.O_RDONLY) + for _ in range(8)] + try: + for fd in devzero_fds: + self._testcase.assertNotIn( + fd, (self.stdin.fileno(), self.stdout.fileno(), + self.stderr.fileno()), + msg="At least one fd was closed early.") + finally: + for fd in devzero_fds: + os.close(fd) + + @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") + def test_preexec_errpipe_does_not_double_close_pipes(self): + """Issue16140: Don't double close pipes on preexec error.""" + + def raise_it(): + raise subprocess.SubprocessError( + "force the _execute_child() errpipe_data path.") + + with self.assertRaises(subprocess.SubprocessError): + self._TestExecuteChildPopen( + self, [sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=raise_it) + + def test_preexec_gc_module_failure(self): + # This tests the code that disables garbage collection if the child + # process will execute any Python. + def raise_runtime_error(): + raise RuntimeError("this shouldn't escape") + enabled = gc.isenabled() + orig_gc_disable = gc.disable + orig_gc_isenabled = gc.isenabled + try: + gc.disable() + self.assertFalse(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertFalse(gc.isenabled(), + "Popen enabled gc when it shouldn't.") + + gc.enable() + self.assertTrue(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertTrue(gc.isenabled(), "Popen left gc disabled.") + + gc.disable = raise_runtime_error + self.assertRaises(RuntimeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + + del gc.isenabled # force an AttributeError + self.assertRaises(AttributeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + finally: + gc.disable = orig_gc_disable + gc.isenabled = orig_gc_isenabled + if not enabled: + gc.disable() + + @unittest.skipIf( + sys.platform == 'darwin', 'setrlimit() seems to fail on OS X') + def test_preexec_fork_failure(self): + # The internal code did not preserve the previous exception when + # re-enabling garbage collection + try: + from resource import getrlimit, setrlimit, RLIMIT_NPROC + except ImportError as err: + self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD + limits = getrlimit(RLIMIT_NPROC) + [_, hard] = limits + setrlimit(RLIMIT_NPROC, (0, hard)) + self.addCleanup(setrlimit, RLIMIT_NPROC, limits) + try: + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + except BlockingIOError: + # Forking should raise EAGAIN, translated to BlockingIOError + pass + else: + self.skipTest('RLIMIT_NPROC had no effect; probably superuser') + + def test_args_string(self): + # args is a string + fd, fname = tempfile.mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!%s\n" % support.unix_shell) + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + p = subprocess.Popen(fname) + p.wait() + os.remove(fname) + self.assertEqual(p.returncode, 47) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + startupinfo=47) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + creationflags=47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen(["echo $FRUIT"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + with p: + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen("echo $FRUIT", shell=1, + stdout=subprocess.PIPE, + env=newenv) + with p: + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_call_string(self): + # call() function with string argument on UNIX + fd, fname = tempfile.mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!%s\n" % support.unix_shell) + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + rc = subprocess.call(fname) + os.remove(fname) + self.assertEqual(rc, 47) + + def test_specific_shell(self): + # Issue #9265: Incorrect name passed as arg[0]. + shells = [] + for prefix in ['/bin', '/usr/bin/', '/usr/local/bin']: + for name in ['bash', 'ksh']: + sh = os.path.join(prefix, name) + if os.path.isfile(sh): + shells.append(sh) + if not shells: # Will probably work for any shell but csh. + self.skipTest("bash or ksh required for this test") + sh = '/bin/sh' + if os.path.isfile(sh) and not os.path.islink(sh): + # Test will fail if /bin/sh is a symlink to csh. + shells.append(sh) + for sh in shells: + p = subprocess.Popen("echo $0", executable=sh, shell=True, + stdout=subprocess.PIPE) + with p: + self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) + + def _kill_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + # Also set the SIGINT handler to the default to make sure it's not + # being ignored (some tests rely on that.) + old_handler = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + signal.signal(signal.SIGINT, old_handler) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + return p + + @unittest.skipIf(sys.platform.startswith(('netbsd', 'openbsd')), + "Due to known OS bug (issue #16762)") + def _kill_dead_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + p.communicate() + + def test_send_signal(self): + p = self._kill_process('send_signal', signal.SIGINT) + _, stderr = p.communicate() + self.assertIn(b'KeyboardInterrupt', stderr) + self.assertNotEqual(p.wait(), 0) + + def test_kill(self): + p = self._kill_process('kill') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGKILL) + + def test_terminate(self): + p = self._kill_process('terminate') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGTERM) + + def test_send_signal_dead(self): + # Sending a signal to a dead process + self._kill_dead_process('send_signal', signal.SIGINT) + + def test_kill_dead(self): + # Killing a dead process + self._kill_dead_process('kill') + + def test_terminate_dead(self): + # Terminating a dead process + self._kill_dead_process('terminate') + + def _save_fds(self, save_fds): + fds = [] + for fd in save_fds: + inheritable = os.get_inheritable(fd) + saved = os.dup(fd) + fds.append((fd, saved, inheritable)) + return fds + + def _restore_fds(self, fds): + for fd, saved, inheritable in fds: + os.dup2(saved, fd, inheritable=inheritable) + os.close(saved) + + def check_close_std_fds(self, fds): + # Issue #9905: test that subprocess pipes still work properly with + # some standard fds closed + stdin = 0 + saved_fds = self._save_fds(fds) + for fd, saved, inheritable in saved_fds: + if fd == 0: + stdin = saved + break + try: + for fd in fds: + os.close(fd) + out, err = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + err = support.strip_python_stderr(err) + self.assertEqual((out, err), (b'apple', b'orange')) + finally: + self._restore_fds(saved_fds) + + def test_close_fd_0(self): + self.check_close_std_fds([0]) + + def test_close_fd_1(self): + self.check_close_std_fds([1]) + + def test_close_fd_2(self): + self.check_close_std_fds([2]) + + def test_close_fds_0_1(self): + self.check_close_std_fds([0, 1]) + + def test_close_fds_0_2(self): + self.check_close_std_fds([0, 2]) + + def test_close_fds_1_2(self): + self.check_close_std_fds([1, 2]) + + def test_close_fds_0_1_2(self): + # Issue #10806: test that subprocess pipes still work properly with + # all standard fds closed. + self.check_close_std_fds([0, 1, 2]) + + def test_small_errpipe_write_fd(self): + """Issue #15798: Popen should work when stdio fds are available.""" + new_stdin = os.dup(0) + new_stdout = os.dup(1) + try: + os.close(0) + os.close(1) + + # Side test: if errpipe_write fails to have its CLOEXEC + # flag set this should cause the parent to think the exec + # failed. Extremely unlikely: everyone supports CLOEXEC. + subprocess.Popen([ + sys.executable, "-c", + "print('AssertionError:0:CLOEXEC failure.')"]).wait() + finally: + # Restore original stdin and stdout + os.dup2(new_stdin, 0) + os.dup2(new_stdout, 1) + os.close(new_stdin) + os.close(new_stdout) + + def test_remapping_std_fds(self): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + try: + temp_fds = [fd for fd, fname in temps] + + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # write some data to what will become stdin, and rewind + os.write(temp_fds[1], b"STDIN") + os.lseek(temp_fds[1], 0, 0) + + # move the standard file descriptors out of the way + saved_fds = self._save_fds(range(3)) + try: + # duplicate the file objects over the standard fd's + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # now use those files in the "wrong" order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=temp_fds[1], + stdout=temp_fds[2], + stderr=temp_fds[0]) + p.wait() + finally: + self._restore_fds(saved_fds) + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(temp_fds[2], 1024) + err = support.strip_python_stderr(os.read(temp_fds[0], 1024)) + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + def check_swap_fds(self, stdin_no, stdout_no, stderr_no): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + temp_fds = [fd for fd, fname in temps] + try: + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # save a copy of the standard file descriptors + saved_fds = self._save_fds(range(3)) + try: + # duplicate the temp files over the standard fd's 0, 1, 2 + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # write some data to what will become stdin, and rewind + os.write(stdin_no, b"STDIN") + os.lseek(stdin_no, 0, 0) + + # now use those files in the given order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=stdin_no, + stdout=stdout_no, + stderr=stderr_no) + p.wait() + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(stdout_no, 1024) + err = support.strip_python_stderr(os.read(stderr_no, 1024)) + finally: + self._restore_fds(saved_fds) + + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + # When duping fds, if there arises a situation where one of the fds is + # either 0, 1 or 2, it is possible that it is overwritten (#12607). + # This tests all combinations of this. + def test_swap_fds(self): + self.check_swap_fds(0, 1, 2) + self.check_swap_fds(0, 2, 1) + self.check_swap_fds(1, 0, 2) + self.check_swap_fds(1, 2, 0) + self.check_swap_fds(2, 0, 1) + self.check_swap_fds(2, 1, 0) + + def test_surrogates_error_message(self): + def prepare(): + raise ValueError("surrogate:\uDCff") + + try: + subprocess.call( + [sys.executable, "-c", "pass"], + preexec_fn=prepare) + except ValueError as err: + # Pure Python implementations keeps the message + self.assertIsNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "surrogate:\uDCff") + except subprocess.SubprocessError as err: + # _posixsubprocess uses a default message + self.assertIsNotNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "Exception occurred in preexec_fn.") + else: + self.fail("Expected ValueError or subprocess.SubprocessError") + + def test_undecodable_env(self): + for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): + encoded_value = value.encode("ascii", "surrogateescape") + + # test str with surrogates + script = "import os; print(ascii(os.getenv(%s)))" % repr(key) + env = os.environ.copy() + env[key] = value + # Use C locale to get ASCII for the locale encoding to force + # surrogate-escaping of \xFF in the child process; otherwise it can + # be decoded as-is if the default locale is latin-1. + env['LC_ALL'] = 'C' + if sys.platform.startswith("aix"): + # On AIX, the C locale uses the Latin1 encoding + decoded_value = encoded_value.decode("latin1", "surrogateescape") + else: + # On other UNIXes, the C locale uses the ASCII encoding + decoded_value = value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(decoded_value)) + + # test bytes + key = key.encode("ascii", "surrogateescape") + script = "import os; print(ascii(os.getenvb(%s)))" % repr(key) + env = os.environ.copy() + env[key] = encoded_value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(encoded_value)) + + def test_bytes_program(self): + abs_program = os.fsencode(sys.executable) + path, program = os.path.split(sys.executable) + program = os.fsencode(program) + + # absolute bytes path + exitcode = subprocess.call([abs_program, "-c", "pass"]) + self.assertEqual(exitcode, 0) + + # absolute bytes path as a string + cmd = b"'" + abs_program + b"' -c pass" + exitcode = subprocess.call(cmd, shell=True) + self.assertEqual(exitcode, 0) + + # bytes program, unicode PATH + env = os.environ.copy() + env["PATH"] = path + exitcode = subprocess.call([program, "-c", "pass"], env=env) + self.assertEqual(exitcode, 0) + + # bytes program, bytes PATH + envb = os.environb.copy() + envb[b"PATH"] = os.fsencode(path) + exitcode = subprocess.call([program, "-c", "pass"], env=envb) + self.assertEqual(exitcode, 0) + + def test_pipe_cloexec(self): + sleeper = support.findfile("input_reader.py", subdir="subprocessdata") + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + p1 = subprocess.Popen([sys.executable, sleeper], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=False) + + self.addCleanup(p1.communicate, b'') + + p2 = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + + output, error = p2.communicate() + result_fds = set(map(int, output.split(b','))) + unwanted_fds = set([p1.stdin.fileno(), p1.stdout.fileno(), + p1.stderr.fileno()]) + + self.assertFalse(result_fds & unwanted_fds, + "Expected no fds from %r to be open in child, " + "found %r" % + (unwanted_fds, result_fds & unwanted_fds)) + + def test_pipe_cloexec_real_tools(self): + qcat = support.findfile("qcat.py", subdir="subprocessdata") + qgrep = support.findfile("qgrep.py", subdir="subprocessdata") + + subdata = b'zxcvbn' + data = subdata * 4 + b'\n' + + p1 = subprocess.Popen([sys.executable, qcat], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + close_fds=False) + + p2 = subprocess.Popen([sys.executable, qgrep, subdata], + stdin=p1.stdout, stdout=subprocess.PIPE, + close_fds=False) + + self.addCleanup(p1.wait) + self.addCleanup(p2.wait) + def kill_p1(): + try: + p1.terminate() + except ProcessLookupError: + pass + def kill_p2(): + try: + p2.terminate() + except ProcessLookupError: + pass + self.addCleanup(kill_p1) + self.addCleanup(kill_p2) + + p1.stdin.write(data) + p1.stdin.close() + + readfiles, ignored1, ignored2 = select.select([p2.stdout], [], [], 10) + + self.assertTrue(readfiles, "The child hung") + self.assertEqual(p2.stdout.read(), data) + + p1.stdout.close() + p2.stdout.close() + + def test_close_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + + open_fds = set(fds) + # add a bunch more fds + for _ in range(9): + fd = os.open(os.devnull, os.O_RDONLY) + self.addCleanup(os.close, fd) + open_fds.add(fd) + + for fd in open_fds: + os.set_inheritable(fd, True) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertEqual(remaining_fds & open_fds, open_fds, + "Some fds were closed") + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & open_fds, + "Some fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + # Keep some of the fd's we opened open in the subprocess. + # This tests _posixsubprocess.c's proper handling of fds_to_keep. + fds_to_keep = set(open_fds.pop() for _ in range(8)) + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=()) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & fds_to_keep & open_fds, + "Some fds not in pass_fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD.") + def test_close_fds_when_max_fd_is_lowered(self): + """Confirm that issue21618 is fixed (may fail under valgrind).""" + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # This launches the meat of the test in a child process to + # avoid messing with the larger unittest processes maximum + # number of file descriptors. + # This process launches: + # +--> Process that lowers its RLIMIT_NOFILE aftr setting up + # a bunch of high open fds above the new lower rlimit. + # Those are reported via stdout before launching a new + # process with close_fds=False to run the actual test: + # +--> The TEST: This one launches a fd_status.py + # subprocess with close_fds=True so we can find out if + # any of the fds above the lowered rlimit are still open. + p = subprocess.Popen([sys.executable, '-c', textwrap.dedent( + ''' + import os, resource, subprocess, sys, textwrap + open_fds = set() + # Add a bunch more fds to pass down. + for _ in range(40): + fd = os.open(os.devnull, os.O_RDONLY) + open_fds.add(fd) + + # Leave a two pairs of low ones available for use by the + # internal child error pipe and the stdout pipe. + # We also leave 10 more open as some Python buildbots run into + # "too many open files" errors during the test if we do not. + for fd in sorted(open_fds)[:14]: + os.close(fd) + open_fds.remove(fd) + + for fd in open_fds: + #self.addCleanup(os.close, fd) + os.set_inheritable(fd, True) + + max_fd_open = max(open_fds) + + # Communicate the open_fds to the parent unittest.TestCase process. + print(','.join(map(str, sorted(open_fds)))) + sys.stdout.flush() + + rlim_cur, rlim_max = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + # 29 is lower than the highest fds we are leaving open. + resource.setrlimit(resource.RLIMIT_NOFILE, (29, rlim_max)) + # Launch a new Python interpreter with our low fd rlim_cur that + # inherits open fds above that limit. It then uses subprocess + # with close_fds=True to get a report of open fds in the child. + # An explicit list of fds to check is passed to fd_status.py as + # letting fd_status rely on its default logic would miss the + # fds above rlim_cur as it normally only checks up to that limit. + subprocess.Popen( + [sys.executable, '-c', + textwrap.dedent(""" + import subprocess, sys + subprocess.Popen([sys.executable, %r] + + [str(x) for x in range({max_fd})], + close_fds=True).wait() + """.format(max_fd=max_fd_open+1))], + close_fds=False).wait() + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, (rlim_cur, rlim_max)) + ''' % fd_status)], stdout=subprocess.PIPE) + + output, unused_stderr = p.communicate() + output_lines = output.splitlines() + self.assertEqual(len(output_lines), 2, + msg="expected exactly two lines of output:\n%r" % output) + opened_fds = set(map(int, output_lines[0].strip().split(b','))) + remaining_fds = set(map(int, output_lines[1].strip().split(b','))) + + self.assertFalse(remaining_fds & opened_fds, + msg="Some fds were left open.") + + + # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file + # descriptor of a pipe closed in the parent process is valid in the + # child process according to fstat(), but the mode of the file + # descriptor is invalid, and read or write raise an error. + @support.requires_mac_ver(10, 5) + def test_pass_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + open_fds = set() + + for x in range(5): + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + os.set_inheritable(fds[0], True) + os.set_inheritable(fds[1], True) + open_fds.update(fds) + + for fd in open_fds: + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=(fd, )) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + to_be_closed = open_fds - {fd} + + self.assertIn(fd, remaining_fds, "fd to be passed not passed") + self.assertFalse(remaining_fds & to_be_closed, + "fd to be closed passed") + + # pass_fds overrides close_fds with a warning. + with self.assertWarns(RuntimeWarning) as context: + self.assertFalse(subprocess.call( + [sys.executable, "-c", "import sys; sys.exit(0)"], + close_fds=False, pass_fds=(fd, ))) + self.assertIn('overriding close_fds', str(context.warning)) + + def test_pass_fds_inheritable(self): + script = support.findfile("fd_status.py", subdir="subprocessdata") + + inheritable, non_inheritable = os.pipe() + self.addCleanup(os.close, inheritable) + self.addCleanup(os.close, non_inheritable) + os.set_inheritable(inheritable, True) + os.set_inheritable(non_inheritable, False) + pass_fds = (inheritable, non_inheritable) + args = [sys.executable, script] + args += list(map(str, pass_fds)) + + p = subprocess.Popen(args, + stdout=subprocess.PIPE, close_fds=True, + pass_fds=pass_fds) + output, ignored = p.communicate() + fds = set(map(int, output.split(b','))) + + # the inheritable file descriptor must be inherited, so its inheritable + # flag must be set in the child process after fork() and before exec() + self.assertEqual(fds, set(pass_fds), "output=%a" % output) + + # inheritable flag must not be changed in the parent process + self.assertEqual(os.get_inheritable(inheritable), True) + self.assertEqual(os.get_inheritable(non_inheritable), False) + + def test_stdout_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stdin=inout) + p.wait() + + def test_stdout_stderr_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stderr=inout) + p.wait() + + def test_stderr_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stderr=inout, stdin=inout) + p.wait() + + def test_wait_when_sigchild_ignored(self): + # NOTE: sigchild_ignore.py may not be an effective test on all OSes. + sigchild_ignore = support.findfile("sigchild_ignore.py", + subdir="subprocessdata") + p = subprocess.Popen([sys.executable, sigchild_ignore], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" + " non-zero with this error:\n%s" % + stderr.decode('utf-8')) + + def test_select_unbuffered(self): + # Issue #11459: bufsize=0 should really set the pipes as + # unbuffered (and therefore let select() work properly). + select = support.import_module("select") + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple")'], + stdout=subprocess.PIPE, + bufsize=0) + f = p.stdout + self.addCleanup(f.close) + try: + self.assertEqual(f.read(4), b"appl") + self.assertIn(f, select.select([f], [], [], 0.0)[0]) + finally: + p.wait() + + def test_zombie_fast_process_del(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, it wouldn't be added to subprocess._active, and would + # remain a zombie. + # spawn a Popen, and delete its reference before it exits + p = subprocess.Popen([sys.executable, "-c", + 'import sys, time;' + 'time.sleep(0.2)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + with support.check_warnings(('', ResourceWarning)): + p = None + + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + def test_leak_fast_process_del_killed(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, and the process got killed by a signal, it would never + # be removed from subprocess._active, which triggered a FD and memory + # leak. + # spawn a Popen, delete its reference and kill it + p = subprocess.Popen([sys.executable, "-c", + 'import time;' + 'time.sleep(3)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + with support.check_warnings(('', ResourceWarning)): + p = None + + os.kill(pid, signal.SIGKILL) + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + # let some time for the process to exit, and create a new Popen: this + # should trigger the wait() of p + time.sleep(0.2) + with self.assertRaises(OSError) as c: + with subprocess.Popen(NONEXISTING_CMD, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + # p should have been wait()ed on, and removed from the _active list + self.assertRaises(OSError, os.waitpid, pid, 0) + self.assertNotIn(ident, [id(o) for o in subprocess._active]) + + def test_close_fds_after_preexec(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # this FD is used as dup2() target by preexec_fn, and should be closed + # in the child process + fd = os.dup(1) + self.addCleanup(os.close, fd) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + preexec_fn=lambda: os.dup2(1, fd)) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + + self.assertNotIn(fd, remaining_fds) + + @support.cpython_only + def test_fork_exec(self): + # Issue #22290: fork_exec() must not crash on memory allocation failure + # or other errors + import _posixsubprocess + gc_enabled = gc.isenabled() + try: + # Use a preexec function and enable the garbage collector + # to force fork_exec() to re-enable the garbage collector + # on error. + func = lambda: None + gc.enable() + + for args, exe_list, cwd, env_list in ( + (123, [b"exe"], None, [b"env"]), + ([b"arg"], 123, None, [b"env"]), + ([b"arg"], [b"exe"], 123, [b"env"]), + ([b"arg"], [b"exe"], None, 123), + ): + with self.assertRaises(TypeError): + _posixsubprocess.fork_exec( + args, exe_list, + True, (), cwd, env_list, + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, func) + finally: + if not gc_enabled: + gc.disable() + + @support.cpython_only + def test_fork_exec_sorted_fd_sanity_check(self): + # Issue #23564: sanity check the fork_exec() fds_to_keep sanity check. + import _posixsubprocess + class BadInt: + first = True + def __init__(self, value): + self.value = value + def __int__(self): + if self.first: + self.first = False + return self.value + raise ValueError + + gc_enabled = gc.isenabled() + try: + gc.enable() + + for fds_to_keep in ( + (-1, 2, 3, 4, 5), # Negative number. + ('str', 4), # Not an int. + (18, 23, 42, 2**63), # Out of range. + (5, 4), # Not sorted. + (6, 7, 7, 8), # Duplicate. + (BadInt(1), BadInt(2)), + ): + with self.assertRaises( + ValueError, + msg='fds_to_keep={}'.format(fds_to_keep)) as c: + _posixsubprocess.fork_exec( + [b"false"], [b"false"], + True, fds_to_keep, None, [b"env"], + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, None) + self.assertIn('fds_to_keep', str(c.exception)) + finally: + if not gc_enabled: + gc.disable() + + def test_communicate_BrokenPipeError_stdin_close(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + proc.communicate() # Should swallow BrokenPipeError from close. + mock_proc_stdin.close.assert_called_with() + + def test_communicate_BrokenPipeError_stdin_write(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.write.side_effect = BrokenPipeError + proc.communicate(b'stuff') # Should swallow the BrokenPipeError. + mock_proc_stdin.write.assert_called_once_with(b'stuff') + mock_proc_stdin.close.assert_called_once_with() + + def test_communicate_BrokenPipeError_stdin_flush(self): + # Setting stdin and stdout forces the ._communicate() code path. + # python -h exits faster than python -c pass (but spams stdout). + proc = subprocess.Popen([sys.executable, '-h'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin, \ + open(os.devnull, 'wb') as dev_null: + mock_proc_stdin.flush.side_effect = BrokenPipeError + # because _communicate registers a selector using proc.stdin... + mock_proc_stdin.fileno.return_value = dev_null.fileno() + # _communicate() should swallow BrokenPipeError from flush. + proc.communicate(b'stuff') + mock_proc_stdin.flush.assert_called_once_with() + + def test_communicate_BrokenPipeError_stdin_close_with_timeout(self): + # Setting stdin and stdout forces the ._communicate() code path. + # python -h exits faster than python -c pass (but spams stdout). + proc = subprocess.Popen([sys.executable, '-h'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + # _communicate() should swallow BrokenPipeError from close. + proc.communicate(timeout=999) + mock_proc_stdin.close.assert_called_once_with() + + @unittest.skipUnless(_testcapi is not None + and hasattr(_testcapi, 'W_STOPCODE'), + 'need _testcapi.W_STOPCODE') + def test_stopped(self): + """Test wait() behavior when waitpid returns WIFSTOPPED; issue29335.""" + args = [sys.executable, '-c', 'pass'] + proc = subprocess.Popen(args) + + # Wait until the real process completes to avoid zombie process + pid = proc.pid + pid, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + status = _testcapi.W_STOPCODE(3) + with mock.patch('subprocess.os.waitpid', return_value=(pid, status)): + returncode = proc.wait() + + self.assertEqual(returncode, -3) + + +@unittest.skipUnless(mswindows, "Windows specific tests") +class Win32ProcessTestCase(BaseTestCase): + + def test_startupinfo(self): + # startupinfo argument + # We uses hardcoded constants, because we do not want to + # depend on win32all. + STARTF_USESHOWWINDOW = 1 + SW_MAXIMIZE = 3 + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = STARTF_USESHOWWINDOW + startupinfo.wShowWindow = SW_MAXIMIZE + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_creationflags(self): + # creationflags argument + CREATE_NEW_CONSOLE = 16 + sys.stderr.write(" a DOS box should flash briefly ...\n") + subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + preexec_fn=lambda: 1) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + stdout=subprocess.PIPE, + close_fds=True) + + @support.cpython_only + def test_issue31471(self): + # There shouldn't be an assertion failure in Popen() in case the env + # argument has a bad keys() method. + class BadEnv(dict): + keys = None + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], env=BadEnv()) + + def test_close_fds(self): + # close file descriptors + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"], + close_fds=True) + self.assertEqual(rc, 47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen(["set"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + with p: + self.assertIn(b"physalis", p.stdout.read()) + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv) + with p: + self.assertIn(b"physalis", p.stdout.read()) + + def test_shell_encodings(self): + # Run command through the shell (string) + for enc in ['ansi', 'oem']: + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv, + encoding=enc) + with p: + self.assertIn("physalis", p.stdout.read(), enc) + + def test_call_string(self): + # call() function with string argument on Windows + rc = subprocess.call(sys.executable + + ' -c "import sys; sys.exit(47)"') + self.assertEqual(rc, 47) + + def _kill_process(self, method, *args): + # Some win32 buildbot raises EOFError if stdin is inherited + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + with p: + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + returncode = p.wait() + self.assertNotEqual(returncode, 0) + + def _kill_dead_process(self, method, *args): + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + sys.exit(42) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + with p: + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() + self.assertEqual(rc, 42) + + def test_send_signal(self): + self._kill_process('send_signal', signal.SIGTERM) + + def test_kill(self): + self._kill_process('kill') + + def test_terminate(self): + self._kill_process('terminate') + + def test_send_signal_dead(self): + self._kill_dead_process('send_signal', signal.SIGTERM) + + def test_kill_dead(self): + self._kill_dead_process('kill') + + def test_terminate_dead(self): + self._kill_dead_process('terminate') + +class MiscTests(unittest.TestCase): + def test_getoutput(self): + self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') + self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), + (0, 'xyzzy')) + + # we use mkdtemp in the next line to create an empty directory + # under our exclusive control; from that, we can invent a pathname + # that we _know_ won't exist. This is guaranteed to fail. + dir = None + try: + dir = tempfile.mkdtemp() + name = os.path.join(dir, "foo") + status, output = subprocess.getstatusoutput( + ("type " if mswindows else "cat ") + name) + self.assertNotEqual(status, 0) + finally: + if dir is not None: + os.rmdir(dir) + + def test__all__(self): + """Ensure that __all__ is populated properly.""" + intentionally_excluded = {"list2cmdline", "Handle"} + exported = set(subprocess.__all__) + possible_exports = set() + import types + for name, value in subprocess.__dict__.items(): + if name.startswith('_'): + continue + if isinstance(value, (types.ModuleType,)): + continue + possible_exports.add(name) + self.assertEqual(exported, possible_exports - intentionally_excluded) + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class ProcessTestCaseNoPoll(ProcessTestCase): + def setUp(self): + self.orig_selector = subprocess._PopenSelector + subprocess._PopenSelector = selectors.SelectSelector + ProcessTestCase.setUp(self) + + def tearDown(self): + subprocess._PopenSelector = self.orig_selector + ProcessTestCase.tearDown(self) + + +@unittest.skipUnless(mswindows, "Windows-specific tests") +class CommandsWithSpaces (BaseTestCase): + + def setUp(self): + super().setUp() + f, fname = tempfile.mkstemp(".py", "te st") + self.fname = fname.lower () + os.write(f, b"import sys;" + b"sys.stdout.write('%d %s' % (len(sys.argv), [a.lower () for a in sys.argv]))" + ) + os.close(f) + + def tearDown(self): + os.remove(self.fname) + super().tearDown() + + def with_spaces(self, *args, **kwargs): + kwargs['stdout'] = subprocess.PIPE + p = subprocess.Popen(*args, **kwargs) + with p: + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) + + def test_shell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd"), shell=1) + + def test_shell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) + + def test_noshell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd")) + + def test_noshell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"]) + + +class ContextManagerTests(BaseTestCase): + + def test_pipe(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write('stdout');" + "sys.stderr.write('stderr');"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + self.assertEqual(proc.stdout.read(), b"stdout") + self.assertStderrEqual(proc.stderr.read(), b"stderr") + + self.assertTrue(proc.stdout.closed) + self.assertTrue(proc.stderr.closed) + + def test_returncode(self): + with subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(100)"]) as proc: + pass + # __exit__ calls wait(), so the returncode should be set + self.assertEqual(proc.returncode, 100) + + def test_communicate_stdin(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.exit(sys.stdin.read() == 'context')"], + stdin=subprocess.PIPE) as proc: + proc.communicate(b"context") + self.assertEqual(proc.returncode, 1) + + def test_invalid_args(self): + with self.assertRaises((FileNotFoundError, PermissionError)) as c: + with subprocess.Popen(NONEXISTING_CMD, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + + def test_broken_pipe_cleanup(self): + """Broken pipe error should not prevent wait() (Issue 21619)""" + proc = subprocess.Popen([sys.executable, '-c', 'pass'], + stdin=subprocess.PIPE, + bufsize=support.PIPE_MAX_SIZE*2) + proc = proc.__enter__() + # Prepare to send enough data to overflow any OS pipe buffering and + # guarantee a broken pipe error. Data is held in BufferedWriter + # buffer until closed. + proc.stdin.write(b'x' * support.PIPE_MAX_SIZE) + self.assertIsNone(proc.returncode) + # EPIPE expected under POSIX; EINVAL under Windows + self.assertRaises(OSError, proc.__exit__, None, None, None) + self.assertEqual(proc.returncode, 0) + self.assertTrue(proc.stdin.closed) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.6/test_threading.py b/src/greentest/3.6/test_threading.py new file mode 100644 index 0000000..7ad35ec --- /dev/null +++ b/src/greentest/3.6/test_threading.py @@ -0,0 +1,1154 @@ +""" +Tests for the threading module. +""" + +import test.support +from test.support import (verbose, import_module, cpython_only, + requires_type_collecting) +from test.support.script_helper import assert_python_ok, assert_python_failure + +import random +import sys +_thread = import_module('_thread') +threading = import_module('threading') +import time +import unittest +import weakref +import os +import subprocess + +from test import lock_tests +from test import support + + +# Between fork() and exec(), only async-safe functions are allowed (issues +# #12316 and #11870), and fork() from a worker thread is known to trigger +# problems with some operating systems (issue #3863): skip problematic tests +# on platforms known to behave badly. +platforms_to_skip = ('freebsd4', 'freebsd5', 'freebsd6', 'netbsd5', + 'hp-ux11') + + +# A trivial mutable counter. +class Counter(object): + def __init__(self): + self.value = 0 + def inc(self): + self.value += 1 + def dec(self): + self.value -= 1 + def get(self): + return self.value + +class TestThread(threading.Thread): + def __init__(self, name, testcase, sema, mutex, nrunning): + threading.Thread.__init__(self, name=name) + self.testcase = testcase + self.sema = sema + self.mutex = mutex + self.nrunning = nrunning + + def run(self): + delay = random.random() / 10000.0 + if verbose: + print('task %s will run for %.1f usec' % + (self.name, delay * 1e6)) + + with self.sema: + with self.mutex: + self.nrunning.inc() + if verbose: + print(self.nrunning.get(), 'tasks are running') + self.testcase.assertLessEqual(self.nrunning.get(), 3) + + time.sleep(delay) + if verbose: + print('task', self.name, 'done') + + with self.mutex: + self.nrunning.dec() + self.testcase.assertGreaterEqual(self.nrunning.get(), 0) + if verbose: + print('%s is finished. %d tasks are running' % + (self.name, self.nrunning.get())) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test.support.threading_setup() + + def tearDown(self): + test.support.threading_cleanup(*self._threads) + test.support.reap_children() + + +class ThreadTests(BaseTestCase): + + # Create a bunch of threads, let each do some work, wait until all are + # done. + def test_various_ops(self): + # This takes about n/3 seconds to run (about n/3 clumps of tasks, + # times about 1 second per clump). + NUMTASKS = 10 + + # no more than 3 of the 10 can run at once + sema = threading.BoundedSemaphore(value=3) + mutex = threading.RLock() + numrunning = Counter() + + threads = [] + + for i in range(NUMTASKS): + t = TestThread(""%i, self, sema, mutex, numrunning) + threads.append(t) + self.assertIsNone(t.ident) + self.assertRegex(repr(t), r'^$') + t.start() + + if verbose: + print('waiting for all tasks to complete') + for t in threads: + t.join() + self.assertFalse(t.is_alive()) + self.assertNotEqual(t.ident, 0) + self.assertIsNotNone(t.ident) + self.assertRegex(repr(t), r'^$') + if verbose: + print('all tasks done') + self.assertEqual(numrunning.get(), 0) + + def test_ident_of_no_threading_threads(self): + # The ident still must work for the main thread and dummy threads. + self.assertIsNotNone(threading.currentThread().ident) + def f(): + ident.append(threading.currentThread().ident) + done.set() + done = threading.Event() + ident = [] + _thread.start_new_thread(f, ()) + done.wait() + self.assertIsNotNone(ident[0]) + # Kill the "immortal" _DummyThread + del threading._active[ident[0]] + + # run with a small(ish) thread stack size (256kB) + def test_various_ops_small_stack(self): + if verbose: + print('with 256kB thread stack size...') + try: + threading.stack_size(262144) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1MB) + def test_various_ops_large_stack(self): + if verbose: + print('with 1MB thread stack size...') + try: + threading.stack_size(0x100000) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + def test_foreign_thread(self): + # Check that a "foreign" thread can use the threading module. + def f(mutex): + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + threading.current_thread() + mutex.release() + + mutex = threading.Lock() + mutex.acquire() + tid = _thread.start_new_thread(f, (mutex,)) + # Wait for the thread to finish. + mutex.acquire() + self.assertIn(tid, threading._active) + self.assertIsInstance(threading._active[tid], threading._DummyThread) + #Issue 29376 + self.assertTrue(threading._active[tid].is_alive()) + self.assertRegex(repr(threading._active[tid]), '_DummyThread') + del threading._active[tid] + + # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) + # exposed at the Python level. This test relies on ctypes to get at it. + def test_PyThreadState_SetAsyncExc(self): + ctypes = import_module("ctypes") + + set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc + + class AsyncExc(Exception): + pass + + exception = ctypes.py_object(AsyncExc) + + # First check it works when setting the exception from the same thread. + tid = threading.get_ident() + + try: + result = set_async_exc(ctypes.c_long(tid), exception) + # The exception is async, so we might have to keep the VM busy until + # it notices. + while True: + pass + except AsyncExc: + pass + else: + # This code is unreachable but it reflects the intent. If we wanted + # to be smarter the above loop wouldn't be infinite. + self.fail("AsyncExc not raised") + try: + self.assertEqual(result, 1) # one thread state modified + except UnboundLocalError: + # The exception was raised too quickly for us to get the result. + pass + + # `worker_started` is set by the thread when it's inside a try/except + # block waiting to catch the asynchronously set AsyncExc exception. + # `worker_saw_exception` is set by the thread upon catching that + # exception. + worker_started = threading.Event() + worker_saw_exception = threading.Event() + + class Worker(threading.Thread): + def run(self): + self.id = threading.get_ident() + self.finished = False + + try: + while True: + worker_started.set() + time.sleep(0.1) + except AsyncExc: + self.finished = True + worker_saw_exception.set() + + t = Worker() + t.daemon = True # so if this fails, we don't hang Python at shutdown + t.start() + if verbose: + print(" started worker thread") + + # Try a thread id that doesn't make sense. + if verbose: + print(" trying nonsensical thread id") + result = set_async_exc(ctypes.c_long(-1), exception) + self.assertEqual(result, 0) # no thread states modified + + # Now raise an exception in the worker thread. + if verbose: + print(" waiting for worker thread to get started") + ret = worker_started.wait() + self.assertTrue(ret) + if verbose: + print(" verifying worker hasn't exited") + self.assertFalse(t.finished) + if verbose: + print(" attempting to raise asynch exception in worker") + result = set_async_exc(ctypes.c_long(t.id), exception) + self.assertEqual(result, 1) # one thread state modified + if verbose: + print(" waiting for worker to say it caught the exception") + worker_saw_exception.wait(timeout=10) + self.assertTrue(t.finished) + if verbose: + print(" all OK -- joining worker") + if t.finished: + t.join() + # else the thread is still running, and we have no way to kill it + + def test_limbo_cleanup(self): + # Issue 7481: Failure to start thread should cleanup the limbo map. + def fail_new_thread(*args): + raise threading.ThreadError() + _start_new_thread = threading._start_new_thread + threading._start_new_thread = fail_new_thread + try: + t = threading.Thread(target=lambda: None) + self.assertRaises(threading.ThreadError, t.start) + self.assertFalse( + t in threading._limbo, + "Failed to cleanup _limbo map on failure of Thread.start().") + finally: + threading._start_new_thread = _start_new_thread + + def test_finalize_runnning_thread(self): + # Issue 1402: the PyGILState_Ensure / _Release functions may be called + # very late on python exit: on deallocation of a running thread for + # example. + import_module("ctypes") + + rc, out, err = assert_python_failure("-c", """if 1: + import ctypes, sys, time, _thread + + # This lock is used as a simple event variable. + ready = _thread.allocate_lock() + ready.acquire() + + # Module globals are cleared before __del__ is run + # So we save the functions in class dict + class C: + ensure = ctypes.pythonapi.PyGILState_Ensure + release = ctypes.pythonapi.PyGILState_Release + def __del__(self): + state = self.ensure() + self.release(state) + + def waitingThread(): + x = C() + ready.release() + time.sleep(100) + + _thread.start_new_thread(waitingThread, ()) + ready.acquire() # Be sure the other thread is waiting. + sys.exit(42) + """) + self.assertEqual(rc, 42) + + def test_finalize_with_trace(self): + # Issue1733757 + # Avoid a deadlock when sys.settrace steps into threading._shutdown + assert_python_ok("-c", """if 1: + import sys, threading + + # A deadlock-killer, to prevent the + # testsuite to hang forever + def killer(): + import os, time + time.sleep(2) + print('program blocked; aborting') + os._exit(2) + t = threading.Thread(target=killer) + t.daemon = True + t.start() + + # This is the trace function + def func(frame, event, arg): + threading.current_thread() + return func + + sys.settrace(func) + """) + + def test_join_nondaemon_on_shutdown(self): + # Issue 1722344 + # Raising SystemExit skipped threading._shutdown + rc, out, err = assert_python_ok("-c", """if 1: + import threading + from time import sleep + + def child(): + sleep(1) + # As a non-daemon thread we SHOULD wake up and nothing + # should be torn down yet + print("Woke up, sleep function is:", sleep) + + threading.Thread(target=child).start() + raise SystemExit + """) + self.assertEqual(out.strip(), + b"Woke up, sleep function is: ") + self.assertEqual(err, b"") + + def test_enumerate_after_join(self): + # Try hard to trigger #1703448: a thread is still returned in + # threading.enumerate() after it has been join()ed. + enum = threading.enumerate + old_interval = sys.getswitchinterval() + try: + for i in range(1, 100): + sys.setswitchinterval(i * 0.0002) + t = threading.Thread(target=lambda: None) + t.start() + t.join() + l = enum() + self.assertNotIn(t, l, + "#1703448 triggered after %d trials: %s" % (i, l)) + finally: + sys.setswitchinterval(old_interval) + + def test_no_refcycle_through_target(self): + class RunSelfFunction(object): + def __init__(self, should_raise): + # The links in this refcycle from Thread back to self + # should be cleaned up when the thread completes. + self.should_raise = should_raise + self.thread = threading.Thread(target=self._run, + args=(self,), + kwargs={'yet_another':self}) + self.thread.start() + + def _run(self, other_ref, yet_another): + if self.should_raise: + raise SystemExit + + cyclic_object = RunSelfFunction(should_raise=False) + weak_cyclic_object = weakref.ref(cyclic_object) + cyclic_object.thread.join() + del cyclic_object + self.assertIsNone(weak_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_cyclic_object()))) + + raising_cyclic_object = RunSelfFunction(should_raise=True) + weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) + raising_cyclic_object.thread.join() + del raising_cyclic_object + self.assertIsNone(weak_raising_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_raising_cyclic_object()))) + + def test_old_threading_api(self): + # Just a quick sanity check to make sure the old method names are + # still present + t = threading.Thread() + t.isDaemon() + t.setDaemon(True) + t.getName() + t.setName("name") + t.isAlive() + e = threading.Event() + e.isSet() + threading.activeCount() + + def test_repr_daemon(self): + t = threading.Thread() + self.assertNotIn('daemon', repr(t)) + t.daemon = True + self.assertIn('daemon', repr(t)) + + def test_deamon_param(self): + t = threading.Thread() + self.assertFalse(t.daemon) + t = threading.Thread(daemon=False) + self.assertFalse(t.daemon) + t = threading.Thread(daemon=True) + self.assertTrue(t.daemon) + + @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + def test_dummy_thread_after_fork(self): + # Issue #14308: a dummy thread in the active list doesn't mess up + # the after-fork mechanism. + code = """if 1: + import _thread, threading, os, time + + def background_thread(evt): + # Creates and registers the _DummyThread instance + threading.current_thread() + evt.set() + time.sleep(10) + + evt = threading.Event() + _thread.start_new_thread(background_thread, (evt,)) + evt.wait() + assert threading.active_count() == 2, threading.active_count() + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + os.wait() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + old_interval = sys.getswitchinterval() + self.addCleanup(sys.setswitchinterval, old_interval) + + # Make the bug more likely to manifest. + test.support.setswitchinterval(1e-6) + + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + pid = os.fork() + if pid == 0: + os._exit(11 if t.is_alive() else 10) + else: + t.join() + + pid, status = os.waitpid(pid, 0) + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(10, os.WEXITSTATUS(status)) + + def test_main_thread(self): + main = threading.main_thread() + self.assertEqual(main.name, 'MainThread') + self.assertEqual(main.ident, threading.current_thread().ident) + self.assertEqual(main.ident, threading.get_ident()) + + def f(): + self.assertNotEqual(threading.main_thread().ident, + threading.current_thread().ident) + th = threading.Thread(target=f) + th.start() + th.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork(self): + code = """if 1: + import os, threading + + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + else: + os.waitpid(pid, 0) + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "MainThread\nTrue\nTrue\n") + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_nonmain_thread(self): + code = """if 1: + import os, threading, sys + + def f(): + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + os.waitpid(pid, 0) + + th = threading.Thread(target=f) + th.start() + th.join() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertIsNone(t._tstate_lock) + t.join() + + def test_repr_stopped(self): + # Verify that "stopped" shows up in repr(Thread) appropriately. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + t = threading.Thread(target=f) + t.start() + started.acquire() + self.assertIn("started", repr(t)) + finish.release() + # "stopped" should appear in the repr in a reasonable amount of time. + # Implementation detail: as of this writing, that's trivially true + # if .join() is called, and almost trivially true if .is_alive() is + # called. The detail we're testing here is that "stopped" shows up + # "all on its own". + LOOKING_FOR = "stopped" + for i in range(500): + if LOOKING_FOR in repr(t): + break + time.sleep(0.01) + self.assertIn(LOOKING_FOR, repr(t)) # we waited at least 5 seconds + t.join() + + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) + + @cpython_only + def test_frame_tstate_tracing(self): + # Issue #14432: Crash when a generator is created in a C thread that is + # destroyed while the generator is still used. The issue was that a + # generator contains a frame, and the frame kept a reference to the + # Python state of the destroyed C thread. The crash occurs when a trace + # function is setup. + + def noop_trace(frame, event, arg): + # no operation + return noop_trace + + def generator(): + while 1: + yield "generator" + + def callback(): + if callback.gen is None: + callback.gen = generator() + return next(callback.gen) + callback.gen = None + + old_trace = sys.gettrace() + sys.settrace(noop_trace) + try: + # Install a trace function + threading.settrace(noop_trace) + + # Create a generator in a C thread which exits after the call + import _testcapi + _testcapi.call_in_temporary_c_thread(callback) + + # Call the generator in a different Python thread, check that the + # generator didn't keep a reference to the destroyed thread state + for test in range(3): + # The trace function is still called here + callback() + finally: + sys.settrace(old_trace) + + +class ThreadJoinOnShutdown(BaseTestCase): + + def _run_and_join(self, script): + script = """if 1: + import sys, os, time, threading + + # a thread, which waits for the main program to terminate + def joiningfunc(mainthread): + mainthread.join() + print('end of thread') + # stdout is fully buffered because not a tty, we have to flush + # before exit. + sys.stdout.flush() + \n""" + script + + rc, out, err = assert_python_ok("-c", script) + data = out.decode().replace('\r', '') + self.assertEqual(data, "end of main\nend of thread\n") + + def test_1_join_on_shutdown(self): + # The usual case: on exit, wait for a non-daemon thread + script = """if 1: + import os + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + time.sleep(0.1) + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_2_join_in_forked_process(self): + # Like the test above, but from a forked interpreter + script = """if 1: + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_3_join_in_forked_from_thread(self): + # Like the test above, but fork() was called from a worker thread + # In the forked process, the main Thread object must be marked as stopped. + + script = """if 1: + main_thread = threading.current_thread() + def worker(): + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(main_thread,)) + print('end of main') + t.start() + t.join() # Should not block: main_thread is already stopped + + w = threading.Thread(target=worker) + w.start() + """ + self._run_and_join(script) + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_4_daemon_threads(self): + # Check that a daemon thread cannot crash the interpreter on shutdown + # by manipulating internal structures that are being disposed of in + # the main thread. + script = """if True: + import os + import random + import sys + import time + import threading + + thread_has_run = set() + + def random_io(): + '''Loop for a while sleeping random tiny amounts and doing some I/O.''' + while True: + in_f = open(os.__file__, 'rb') + stuff = in_f.read(200) + null_f = open(os.devnull, 'wb') + null_f.write(stuff) + time.sleep(random.random() / 1995) + null_f.close() + in_f.close() + thread_has_run.add(threading.current_thread()) + + def main(): + count = 0 + for _ in range(40): + new_thread = threading.Thread(target=random_io) + new_thread.daemon = True + new_thread.start() + count += 1 + while len(thread_has_run) < count: + time.sleep(0.001) + # Trigger process shutdown + sys.exit(0) + + main() + """ + rc, out, err = assert_python_ok('-c', script) + self.assertFalse(err) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_reinit_tls_after_fork(self): + # Issue #13817: fork() would deadlock in a multithreaded program with + # the ad-hoc TLS implementation. + + def do_fork_and_wait(): + # just fork a child process and wait it + pid = os.fork() + if pid > 0: + os.waitpid(pid, 0) + else: + os._exit(0) + + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() + + for t in threads: + t.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_clear_threads_states_after_fork(self): + # Issue #17094: check that threads states are cleared after fork() + + # start a bunch of threads + threads = [] + for i in range(16): + t = threading.Thread(target=lambda : time.sleep(0.3)) + threads.append(t) + t.start() + + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(0) + else: + os._exit(1) + else: + _, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + for t in threads: + t.join() + + +class SubinterpThreadingTests(BaseTestCase): + + def test_threads_join(self): + # Non-daemon threads should be joined at subinterpreter shutdown + # (issue #18808) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + @cpython_only + def test_daemon_threads_fatal_error(self): + subinterp_code = r"""if 1: + import os + import threading + import time + + def f(): + # Make sure the daemon thread is still running when + # Py_EndInterpreter is called. + time.sleep(10) + threading.Thread(target=f, daemon=True).start() + """ + script = r"""if 1: + import _testcapi + + _testcapi.run_in_subinterp(%r) + """ % (subinterp_code,) + with test.support.SuppressCrashReport(): + rc, out, err = assert_python_failure("-c", script) + self.assertIn("Fatal Python error: Py_EndInterpreter: " + "not the last thread", err.decode()) + + +class ThreadingExceptionTests(BaseTestCase): + # A RuntimeError should be raised if Thread.start() is called + # multiple times. + def test_start_thread_again(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, thread.start) + thread.join() + + def test_joining_current_thread(self): + current_thread = threading.current_thread() + self.assertRaises(RuntimeError, current_thread.join); + + def test_joining_inactive_thread(self): + thread = threading.Thread() + self.assertRaises(RuntimeError, thread.join) + + def test_daemonize_active_thread(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + thread.join() + + def test_releasing_unacquired_lock(self): + lock = threading.Lock() + self.assertRaises(RuntimeError, lock.release) + + @unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(), + 'test macosx problem') + def test_recursion_limit(self): + # Issue 9670 + # test that excessive recursion within a non-main thread causes + # an exception rather than crashing the interpreter on platforms + # like Mac OS X or FreeBSD which have small default stack sizes + # for threads + script = """if True: + import threading + + def recurse(): + return recurse() + + def outer(): + try: + recurse() + except RecursionError: + pass + + w = threading.Thread(target=outer) + w.start() + w.join() + print('end of main thread') + """ + expected_output = "end of main thread\n" + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + data = stdout.decode().replace('\r', '') + self.assertEqual(p.returncode, 0, "Unexpected error: " + stderr.decode()) + self.assertEqual(data, expected_output) + + def test_print_exception(self): + script = r"""if True: + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + @requires_type_collecting + def test_print_exception_stderr_is_none_1(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + sys.stderr = None + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_2(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + sys.stderr = None + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + self.assertNotIn("Unhandled exception", err.decode()) + + def test_bare_raise_in_brand_new_thread(self): + def bare_raise(): + raise + + class Issue27558(threading.Thread): + exc = None + + def run(self): + try: + bare_raise() + except Exception as exc: + self.exc = exc + + thread = Issue27558() + thread.start() + thread.join() + self.assertIsNotNone(thread.exc) + self.assertIsInstance(thread.exc, RuntimeError) + # explicitly break the reference cycle to not leak a dangling thread + thread.exc = None + +class TimerTests(BaseTestCase): + + def setUp(self): + BaseTestCase.setUp(self) + self.callback_args = [] + self.callback_event = threading.Event() + + def test_init_immutable_default_args(self): + # Issue 17435: constructor defaults were mutable objects, they could be + # mutated via the object attributes and affect other Timer objects. + timer1 = threading.Timer(0.01, self._callback_spy) + timer1.start() + self.callback_event.wait() + timer1.args.append("blah") + timer1.kwargs["foo"] = "bar" + self.callback_event.clear() + timer2 = threading.Timer(0.01, self._callback_spy) + timer2.start() + self.callback_event.wait() + self.assertEqual(len(self.callback_args), 2) + self.assertEqual(self.callback_args, [((), {}), ((), {})]) + timer1.join() + timer2.join() + + def _callback_spy(self, *args, **kwargs): + self.callback_args.append((args[:], kwargs.copy())) + self.callback_event.set() + +class LockTests(lock_tests.LockTests): + locktype = staticmethod(threading.Lock) + + @unittest.skip("not on gevent") + def test_locked_repr(self): + pass + + @unittest.skip("not on gevent") + def test_repr(self): + pass + +class PyRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._PyRLock) + +@unittest.skipIf(threading._CRLock is None, 'RLock not implemented in C') +class CRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._CRLock) + +class EventTests(lock_tests.EventTests): + eventtype = staticmethod(threading.Event) + + @unittest.skip("not on gevent") + def test_reset_internal_locks(self): + # XXX: gevent: this appears to have gone away by 3.6.3 + pass + +class ConditionAsRLockTests(lock_tests.RLockTests): + # Condition uses an RLock by default and exports its API. + locktype = staticmethod(threading.Condition) + +class ConditionTests(lock_tests.ConditionTests): + condtype = staticmethod(threading.Condition) + +class SemaphoreTests(lock_tests.SemaphoreTests): + semtype = staticmethod(threading.Semaphore) + +class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): + semtype = staticmethod(threading.BoundedSemaphore) + +class BarrierTests(lock_tests.BarrierTests): + barriertype = staticmethod(threading.Barrier) + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {"ThreadError"} + blacklist = {'currentThread', 'activeCount'} + support.check__all__(self, threading, ('threading', '_thread'), + extra=extra, blacklist=blacklist) + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.6/test_wsgiref.py b/src/greentest/3.6/test_wsgiref.py new file mode 100644 index 0000000..7708e20 --- /dev/null +++ b/src/greentest/3.6/test_wsgiref.py @@ -0,0 +1,785 @@ +from unittest import mock +from test import support +from test.test_httpservers import NoLogRequestHandler +from unittest import TestCase +from wsgiref.util import setup_testing_defaults +from wsgiref.headers import Headers +from wsgiref.handlers import BaseHandler, BaseCGIHandler, SimpleHandler +from wsgiref import util +from wsgiref.validate import validator +from wsgiref.simple_server import WSGIServer, WSGIRequestHandler +from wsgiref.simple_server import make_server +from http.client import HTTPConnection +from io import StringIO, BytesIO, BufferedReader +from socketserver import BaseServer +from platform import python_implementation + +import os +import re +import signal +import sys +import unittest + + +class MockServer(WSGIServer): + """Non-socket HTTP server""" + + def __init__(self, server_address, RequestHandlerClass): + BaseServer.__init__(self, server_address, RequestHandlerClass) + self.server_bind() + + def server_bind(self): + host, port = self.server_address + self.server_name = host + self.server_port = port + self.setup_environ() + + +class MockHandler(WSGIRequestHandler): + """Non-socket HTTP handler""" + def setup(self): + self.connection = self.request + self.rfile, self.wfile = self.connection + + def finish(self): + pass + + +def hello_app(environ,start_response): + start_response("200 OK", [ + ('Content-Type','text/plain'), + ('Date','Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return [b"Hello, world!"] + + +def header_app(environ, start_response): + start_response("200 OK", [ + ('Content-Type', 'text/plain'), + ('Date', 'Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return [';'.join([ + environ['HTTP_X_TEST_HEADER'], environ['QUERY_STRING'], + environ['PATH_INFO'] + ]).encode('iso-8859-1')] + + +def run_amock(app=hello_app, data=b"GET / HTTP/1.0\n\n"): + server = make_server("", 80, app, MockServer, MockHandler) + inp = BufferedReader(BytesIO(data)) + out = BytesIO() + olderr = sys.stderr + err = sys.stderr = StringIO() + + try: + server.finish_request((inp, out), ("127.0.0.1",8888)) + finally: + sys.stderr = olderr + + return out.getvalue(), err.getvalue() + +def compare_generic_iter(make_it,match): + """Utility to compare a generic 2.1/2.2+ iterator with an iterable + + If running under Python 2.2+, this tests the iterator using iter()/next(), + as well as __getitem__. 'make_it' must be a function returning a fresh + iterator to be tested (since this may test the iterator twice).""" + + it = make_it() + n = 0 + for item in match: + if not it[n]==item: raise AssertionError + n+=1 + try: + it[n] + except IndexError: + pass + else: + raise AssertionError("Too many items from __getitem__",it) + + try: + iter, StopIteration + except NameError: + pass + else: + # Only test iter mode under 2.2+ + it = make_it() + if not iter(it) is it: raise AssertionError + for item in match: + if not next(it) == item: raise AssertionError + try: + next(it) + except StopIteration: + pass + else: + raise AssertionError("Too many items from .__next__()", it) + + +class IntegrationTests(TestCase): + + def check_hello(self, out, has_length=True): + pyver = (python_implementation() + "/" + + sys.version.split()[0]) + self.assertEqual(out, + ("HTTP/1.0 200 OK\r\n" + "Server: WSGIServer/0.2 " + pyver +"\r\n" + "Content-Type: text/plain\r\n" + "Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + + (has_length and "Content-Length: 13\r\n" or "") + + "\r\n" + "Hello, world!").encode("iso-8859-1") + ) + + def test_plain_hello(self): + out, err = run_amock() + self.check_hello(out) + + def test_environ(self): + request = ( + b"GET /p%61th/?query=test HTTP/1.0\n" + b"X-Test-Header: Python test \n" + b"X-Test-Header: Python test 2\n" + b"Content-Length: 0\n\n" + ) + out, err = run_amock(header_app, request) + self.assertEqual( + out.splitlines()[-1], + b"Python test,Python test 2;query=test;/path/" + ) + + def test_request_length(self): + out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") + self.assertEqual(out.splitlines()[0], + b"HTTP/1.0 414 Request-URI Too Long") + + def test_validated_hello(self): + out, err = run_amock(validator(hello_app)) + # the middleware doesn't support len(), so content-length isn't there + self.check_hello(out, has_length=False) + + def test_simple_validation_error(self): + def bad_app(environ,start_response): + start_response("200 OK", ('Content-Type','text/plain')) + return ["Hello, world!"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], + "AssertionError: Headers (('Content-Type', 'text/plain')) must" + " be of type list: " + ) + + def test_status_validation_errors(self): + def create_bad_app(status): + def bad_app(environ, start_response): + start_response(status, [("Content-Type", "text/plain; charset=utf-8")]) + return [b"Hello, world!"] + return bad_app + + tests = [ + ('200', 'AssertionError: Status must be at least 4 characters'), + ('20X OK', 'AssertionError: Status message must begin w/3-digit code'), + ('200OK', 'AssertionError: Status message must have a space after code'), + ] + + for status, exc_message in tests: + with self.subTest(status=status): + out, err = run_amock(create_bad_app(status)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual(err.splitlines()[-2], exc_message) + + def test_wsgi_input(self): + def bad_app(e,s): + e["wsgi.input"].read() + s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) + return [b"data"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], "AssertionError" + ) + + def test_bytes_validation(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain; charset=utf-8"), + ("Date", "Wed, 24 Dec 2008 13:29:32 GMT"), + ]) + return [b"data"] + out, err = run_amock(validator(app)) + self.assertTrue(err.endswith('"GET / HTTP/1.0" 200 4\n')) + ver = sys.version.split()[0].encode('ascii') + py = python_implementation().encode('ascii') + pyver = py + b"/" + ver + self.assertEqual( + b"HTTP/1.0 200 OK\r\n" + b"Server: WSGIServer/0.2 "+ pyver + b"\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"Date: Wed, 24 Dec 2008 13:29:32 GMT\r\n" + b"\r\n" + b"data", + out) + + def test_cp1252_url(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain"), + ("Date", "Wed, 24 Dec 2008 13:29:32 GMT"), + ]) + # PEP3333 says environ variables are decoded as latin1. + # Encode as latin1 to get original bytes + return [e["PATH_INFO"].encode("latin1")] + + out, err = run_amock( + validator(app), data=b"GET /\x80%80 HTTP/1.0") + self.assertEqual( + [ + b"HTTP/1.0 200 OK", + mock.ANY, + b"Content-Type: text/plain", + b"Date: Wed, 24 Dec 2008 13:29:32 GMT", + b"", + b"/\x80\x80", + ], + out.splitlines()) + + def test_interrupted_write(self): + # BaseHandler._write() and _flush() have to write all data, even if + # it takes multiple send() calls. Test this by interrupting a send() + # call with a Unix signal. + threading = support.import_module("threading") + pthread_kill = support.get_attribute(signal, "pthread_kill") + + def app(environ, start_response): + start_response("200 OK", []) + return [b'\0' * support.SOCK_MAX_SIZE] + + class WsgiHandler(NoLogRequestHandler, WSGIRequestHandler): + pass + + server = make_server(support.HOST, 0, app, handler_class=WsgiHandler) + self.addCleanup(server.server_close) + interrupted = threading.Event() + + def signal_handler(signum, frame): + interrupted.set() + + original = signal.signal(signal.SIGUSR1, signal_handler) + self.addCleanup(signal.signal, signal.SIGUSR1, original) + received = None + main_thread = threading.get_ident() + + def run_client(): + http = HTTPConnection(*server.server_address) + http.request("GET", "/") + with http.getresponse() as response: + response.read(100) + # The main thread should now be blocking in a send() system + # call. But in theory, it could get interrupted by other + # signals, and then retried. So keep sending the signal in a + # loop, in case an earlier signal happens to be delivered at + # an inconvenient moment. + while True: + pthread_kill(main_thread, signal.SIGUSR1) + if interrupted.wait(timeout=float(1)): + break + nonlocal received + received = len(response.read()) + http.close() + + background = threading.Thread(target=run_client) + background.start() + server.handle_request() + background.join() + self.assertEqual(received, support.SOCK_MAX_SIZE - 100) + + +class UtilityTests(TestCase): + + def checkShift(self,sn_in,pi_in,part,sn_out,pi_out): + env = {'SCRIPT_NAME':sn_in,'PATH_INFO':pi_in} + util.setup_testing_defaults(env) + self.assertEqual(util.shift_path_info(env),part) + self.assertEqual(env['PATH_INFO'],pi_out) + self.assertEqual(env['SCRIPT_NAME'],sn_out) + return env + + def checkDefault(self, key, value, alt=None): + # Check defaulting when empty + env = {} + util.setup_testing_defaults(env) + if isinstance(value, StringIO): + self.assertIsInstance(env[key], StringIO) + elif isinstance(value,BytesIO): + self.assertIsInstance(env[key],BytesIO) + else: + self.assertEqual(env[key], value) + + # Check existing value + env = {key:alt} + util.setup_testing_defaults(env) + self.assertIs(env[key], alt) + + def checkCrossDefault(self,key,value,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(kw[key],value) + + def checkAppURI(self,uri,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.application_uri(kw),uri) + + def checkReqURI(self,uri,query=1,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.request_uri(kw,query),uri) + + def checkFW(self,text,size,match): + + def make_it(text=text,size=size): + return util.FileWrapper(StringIO(text),size) + + compare_generic_iter(make_it,match) + + it = make_it() + self.assertFalse(it.filelike.closed) + + for item in it: + pass + + self.assertFalse(it.filelike.closed) + + it.close() + self.assertTrue(it.filelike.closed) + + def testSimpleShifts(self): + self.checkShift('','/', '', '/', '') + self.checkShift('','/x', 'x', '/x', '') + self.checkShift('/','', None, '/', '') + self.checkShift('/a','/x/y', 'x', '/a/x', '/y') + self.checkShift('/a','/x/', 'x', '/a/x', '/') + + def testNormalizedShifts(self): + self.checkShift('/a/b', '/../y', '..', '/a', '/y') + self.checkShift('', '/../y', '..', '', '/y') + self.checkShift('/a/b', '//y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '//y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '/./y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '/./y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '///./..//y/.//', '..', '/a', '/y/') + self.checkShift('/a/b', '///', '', '/a/b/', '') + self.checkShift('/a/b', '/.//', '', '/a/b/', '') + self.checkShift('/a/b', '/x//', 'x', '/a/b/x', '/') + self.checkShift('/a/b', '/.', None, '/a/b', '') + + def testDefaults(self): + for key, value in [ + ('SERVER_NAME','127.0.0.1'), + ('SERVER_PORT', '80'), + ('SERVER_PROTOCOL','HTTP/1.0'), + ('HTTP_HOST','127.0.0.1'), + ('REQUEST_METHOD','GET'), + ('SCRIPT_NAME',''), + ('PATH_INFO','/'), + ('wsgi.version', (1,0)), + ('wsgi.run_once', 0), + ('wsgi.multithread', 0), + ('wsgi.multiprocess', 0), + ('wsgi.input', BytesIO()), + ('wsgi.errors', StringIO()), + ('wsgi.url_scheme','http'), + ]: + self.checkDefault(key,value) + + def testCrossDefaults(self): + self.checkCrossDefault('HTTP_HOST',"foo.bar",SERVER_NAME="foo.bar") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="on") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="1") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="yes") + self.checkCrossDefault('wsgi.url_scheme',"http",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"80",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"443",HTTPS="on") + + def testGuessScheme(self): + self.assertEqual(util.guess_scheme({}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"foo"}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"on"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"yes"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"1"}), "https") + + def testAppURIs(self): + self.checkAppURI("http://127.0.0.1/") + self.checkAppURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkAppURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkAppURI("http://spam.example.com:2071/", + HTTP_HOST="spam.example.com:2071", SERVER_PORT="2071") + self.checkAppURI("http://spam.example.com/", + SERVER_NAME="spam.example.com") + self.checkAppURI("http://127.0.0.1/", + HTTP_HOST="127.0.0.1", SERVER_NAME="spam.example.com") + self.checkAppURI("https://127.0.0.1/", HTTPS="on") + self.checkAppURI("http://127.0.0.1:8000/", SERVER_PORT="8000", + HTTP_HOST=None) + + def testReqURIs(self): + self.checkReqURI("http://127.0.0.1/") + self.checkReqURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkReqURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam", + SCRIPT_NAME="/spammity", PATH_INFO="/spam") + self.checkReqURI("http://127.0.0.1/spammity/sp%E4m", + SCRIPT_NAME="/spammity", PATH_INFO="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam;ham", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;ham") + self.checkReqURI("http://127.0.0.1/spammity/spam;cookie=1234,5678", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;cookie=1234,5678") + self.checkReqURI("http://127.0.0.1/spammity/spam?say=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam?s%E4y=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="s%E4y=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam", 0, + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + + def testFileWrapper(self): + self.checkFW("xyz"*50, 120, ["xyz"*40,"xyz"*10]) + + def testHopByHop(self): + for hop in ( + "Connection Keep-Alive Proxy-Authenticate Proxy-Authorization " + "TE Trailers Transfer-Encoding Upgrade" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertTrue(util.is_hop_by_hop(alt)) + + # Not comprehensive, just a few random header names + for hop in ( + "Accept Cache-Control Date Pragma Trailer Via Warning" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertFalse(util.is_hop_by_hop(alt)) + +class HeaderTests(TestCase): + + def testMappingInterface(self): + test = [('x','y')] + self.assertEqual(len(Headers()), 0) + self.assertEqual(len(Headers([])),0) + self.assertEqual(len(Headers(test[:])),1) + self.assertEqual(Headers(test[:]).keys(), ['x']) + self.assertEqual(Headers(test[:]).values(), ['y']) + self.assertEqual(Headers(test[:]).items(), test) + self.assertIsNot(Headers(test).items(), test) # must be copy! + + h = Headers() + del h['foo'] # should not raise an error + + h['Foo'] = 'bar' + for m in h.__contains__, h.get, h.get_all, h.__getitem__: + self.assertTrue(m('foo')) + self.assertTrue(m('Foo')) + self.assertTrue(m('FOO')) + self.assertFalse(m('bar')) + + self.assertEqual(h['foo'],'bar') + h['foo'] = 'baz' + self.assertEqual(h['FOO'],'baz') + self.assertEqual(h.get_all('foo'),['baz']) + + self.assertEqual(h.get("foo","whee"), "baz") + self.assertEqual(h.get("zoo","whee"), "whee") + self.assertEqual(h.setdefault("foo","whee"), "baz") + self.assertEqual(h.setdefault("zoo","whee"), "whee") + self.assertEqual(h["foo"],"baz") + self.assertEqual(h["zoo"],"whee") + + def testRequireList(self): + self.assertRaises(TypeError, Headers, "foo") + + def testExtras(self): + h = Headers() + self.assertEqual(str(h),'\r\n') + + h.add_header('foo','bar',baz="spam") + self.assertEqual(h['foo'], 'bar; baz="spam"') + self.assertEqual(str(h),'foo: bar; baz="spam"\r\n\r\n') + + h.add_header('Foo','bar',cheese=None) + self.assertEqual(h.get_all('foo'), + ['bar; baz="spam"', 'bar; cheese']) + + self.assertEqual(str(h), + 'foo: bar; baz="spam"\r\n' + 'Foo: bar; cheese\r\n' + '\r\n' + ) + +class ErrorHandler(BaseCGIHandler): + """Simple handler subclass for testing BaseHandler""" + + # BaseHandler records the OS environment at import time, but envvars + # might have been changed later by other tests, which trips up + # HandlerTests.testEnviron(). + os_environ = dict(os.environ.items()) + + def __init__(self,**kw): + setup_testing_defaults(kw) + BaseCGIHandler.__init__( + self, BytesIO(), BytesIO(), StringIO(), kw, + multithread=True, multiprocess=True + ) + +class TestHandler(ErrorHandler): + """Simple handler subclass for testing BaseHandler, w/error passthru""" + + def handle_error(self): + raise # for testing, we want to see what's happening + + +class HandlerTests(TestCase): + + def checkEnvironAttrs(self, handler): + env = handler.environ + for attr in [ + 'version','multithread','multiprocess','run_once','file_wrapper' + ]: + if attr=='file_wrapper' and handler.wsgi_file_wrapper is None: + continue + self.assertEqual(getattr(handler,'wsgi_'+attr),env['wsgi.'+attr]) + + def checkOSEnviron(self,handler): + empty = {}; setup_testing_defaults(empty) + env = handler.environ + from os import environ + for k,v in environ.items(): + if k not in empty: + self.assertEqual(env[k],v) + for k,v in empty.items(): + self.assertIn(k, env) + + def testEnviron(self): + h = TestHandler(X="Y") + h.setup_environ() + self.checkEnvironAttrs(h) + self.checkOSEnviron(h) + self.assertEqual(h.environ["X"],"Y") + + def testCGIEnviron(self): + h = BaseCGIHandler(None,None,None,{}) + h.setup_environ() + for key in 'wsgi.url_scheme', 'wsgi.input', 'wsgi.errors': + self.assertIn(key, h.environ) + + def testScheme(self): + h=TestHandler(HTTPS="on"); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'https') + h=TestHandler(); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'http') + + def testAbstractMethods(self): + h = BaseHandler() + for name in [ + '_flush','get_stdin','get_stderr','add_cgi_vars' + ]: + self.assertRaises(NotImplementedError, getattr(h,name)) + self.assertRaises(NotImplementedError, h._write, "test") + + def testContentLength(self): + # Demo one reason iteration is better than write()... ;) + + def trivial_app1(e,s): + s('200 OK',[]) + return [e['wsgi.url_scheme'].encode('iso-8859-1')] + + def trivial_app2(e,s): + s('200 OK',[])(e['wsgi.url_scheme'].encode('iso-8859-1')) + return [] + + def trivial_app3(e,s): + s('200 OK',[]) + return ['\u0442\u0435\u0441\u0442'.encode("utf-8")] + + def trivial_app4(e,s): + # Simulate a response to a HEAD request + s('200 OK',[('Content-Length', '12345')]) + return [] + + h = TestHandler() + h.run(trivial_app1) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "Content-Length: 4\r\n" + "\r\n" + "http").encode("iso-8859-1")) + + h = TestHandler() + h.run(trivial_app2) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "\r\n" + "http").encode("iso-8859-1")) + + h = TestHandler() + h.run(trivial_app3) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 8\r\n' + b'\r\n' + b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82') + + h = TestHandler() + h.run(trivial_app4) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 12345\r\n' + b'\r\n') + + def testBasicErrorOutput(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + def error_app(e,s): + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(non_error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n").encode("iso-8859-1")) + self.assertEqual(h.stderr.getvalue(),"") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: %s\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: %d\r\n" + "\r\n" % (h.error_status,len(h.error_body))).encode('iso-8859-1') + + h.error_body) + + self.assertIn("AssertionError", h.stderr.getvalue()) + + def testErrorAfterOutput(self): + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "\r\n".encode("iso-8859-1")+MSG)) + self.assertIn("AssertionError", h.stderr.getvalue()) + + def testHeaderFormats(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + stdpat = ( + r"HTTP/%s 200 OK\r\n" + r"Date: \w{3}, [ 0123]\d \w{3} \d{4} \d\d:\d\d:\d\d GMT\r\n" + r"%s" r"Content-Length: 0\r\n" r"\r\n" + ) + shortpat = ( + "Status: 200 OK\r\n" "Content-Length: 0\r\n" "\r\n" + ).encode("iso-8859-1") + + for ssw in "FooBar/1.0", None: + sw = ssw and "Server: %s\r\n" % ssw or "" + + for version in "1.0", "1.1": + for proto in "HTTP/0.9", "HTTP/1.0", "HTTP/1.1": + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = False + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + self.assertEqual(shortpat,h.stdout.getvalue()) + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = True + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + if proto=="HTTP/0.9": + self.assertEqual(h.stdout.getvalue(),b"") + else: + self.assertTrue( + re.match((stdpat%(version,sw)).encode("iso-8859-1"), + h.stdout.getvalue()), + ((stdpat%(version,sw)).encode("iso-8859-1"), + h.stdout.getvalue()) + ) + + def testBytesData(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain; charset=utf-8"), + ]) + return [b"data"] + + h = TestHandler() + h.run(app) + self.assertEqual(b"Status: 200 OK\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"Content-Length: 4\r\n" + b"\r\n" + b"data", + h.stdout.getvalue()) + + def testCloseOnError(self): + side_effects = {'close_called': False} + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + class CrashyIterable(object): + def __iter__(self): + while True: + yield b'blah' + raise AssertionError("This should be caught by handler") + def close(self): + side_effects['close_called'] = True + return CrashyIterable() + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(side_effects['close_called'], True) + + def testPartialWrite(self): + written = bytearray() + + class PartialWriter: + def write(self, b): + partial = b[:7] + written.extend(partial) + return len(partial) + + def flush(self): + pass + + environ = {"SERVER_PROTOCOL": "HTTP/1.0"} + h = SimpleHandler(BytesIO(), PartialWriter(), sys.stderr, environ) + msg = "should not do partial writes" + with self.assertWarnsRegex(DeprecationWarning, msg): + h.run(hello_app) + self.assertEqual(b"HTTP/1.0 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + b"Content-Length: 13\r\n" + b"\r\n" + b"Hello, world!", + written) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.6/version b/src/greentest/3.6/version new file mode 100644 index 0000000..4a788a0 --- /dev/null +++ b/src/greentest/3.6/version @@ -0,0 +1 @@ +3.6.3 diff --git a/src/greentest/3.6/wrongcert.pem b/src/greentest/3.6/wrongcert.pem new file mode 100644 index 0000000..5f92f9b --- /dev/null +++ b/src/greentest/3.6/wrongcert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH +FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T +f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB +AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq +1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW +7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg +SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe +19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg +ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666 +lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs +ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv +frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk +2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6 ++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK +24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G +A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9 +OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ +fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E +usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+ +43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw +eSHj5jpC8iZKjCHBn+mAi4cQ514= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/allsans.pem b/src/greentest/3.7/allsans.pem new file mode 100644 index 0000000..bf59f30 --- /dev/null +++ b/src/greentest/3.7/allsans.pem @@ -0,0 +1,64 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQD59JyhPgYe7nhZ +Z2IGhaklNgtRkD+5BVs7lEWovYRBlXpPA6PuaHat25rI8EGYHmlufPherg2Qu6sC +GmZZKo7TgjlmDcwVS4hkebtFH7OZy5Il7Y2ZIdiK7Xp9Z0EPoqwacYowB0a8WhZY +I2Vm4EzCNKl6/htkwjgn2JXGizxvGt/1kNqP/GBAX+vjgeahOsn8jVh96KpFHJbS +g83cX4t8M7FJv7yNoDLvORHnvKCOXbQmr6ZMGcZN8PwS8awQ31khZTpEx+hCe+Pi +GzeOlxpZimXWDAGWA4tZ58Ka/QvO7VQbD5Ci166ODvvs+tEXfBUExtPcS+02IBJV +tzhBna9VAgMBAAECggEAPar9DccIqY76QEyCYcuOPLEFv9zP6+0HYj6lpQkE3U1s +vJvQURyS0zgQCy1Dca1nI6xPdsSIckHq4fzzbWJTlJlXYfdbd5GIGAn0iwxUOkiA +ST0/px0zmKsYgmH8KkhfH7MNfeX9rLCpPJuXA/eo2G03tzGEPqqwQhxsb2ygv2Qs +M7OqJz6RJu87K1Y+psWIv9+VhNVja0kvsg52QMK9mtp8layb54qLI5R5e09sIudq +RHegtnSOBo9kt32H9vWUFaF5PpYt4yks4KYI4ulKGWJGXHMDW4uHUaE/tjNQuYAX +DuDvjN+ECSJvigiUbu2k0xB2KYIb1fpcxlz/YBdADQKBgQD/Z2VtBUjOFnJKz00f +xN0akp7XPgd1yCb1/wZq9PQiGvzIAMDIplioTvjOjhOzPJaWD0GICNeypzQ48+0P +UsPIKbazpIZN6bZncr65plSpg0KANq46hbkPHOo8PHDa7yoxBUSPr8F7P1OCRkn6 ++QdgcnrAly7yfqO2ahAWOX7iCwKBgQD6ifXSCKfRF1GUb3Ws7S1rLxeBasWq+EmC +sUnck0S+AyaMkN+kZ5zejbN+NDuUMQ7+3wUIheTclUhzR0LP3+r5jjHsimJuvOml +wuV37F+Om5lD/Xx27NfbtRKn/bK6o0zDL8JB2eFB0N7Fh7hRYoUMdrpQs5sU91IC +pNYlAcLwHwKBgGvLK9eTf2LbvmksjRR3dgodD8UwfN2NGESC2iaSM+ehFEclajhF +XO3MRt6GwHHJhJTY44OSl9bjEvtmmAr7l34HfQDc04JWvZFzsGOSe/D/YTXT3jz8 +61ohjgrWR5tfjaMa4hDy0Oo/k/NLzzWJnT9rkbtvE3VtVZNLuHZo1dB5AoGBAMHO +wStV6MO1nzUNN+Gqo8zbY/qIJxsH8I26KaIJBk9azpJEa8yZHl+HDEffjgsoHCqL +STB7qzv7+0y53nRCClo8ZmBN+LEjUDcbWjl3z7/YnCpdR9ATjTP3kdQETCNWucXw +Bvy72CX6tqnlQG8soDGxEpXlKl2AqJ9E9icwgqUPAoGAL6xTDdgcYTbk9wxCd41l +NhHTSvLrGXLAzv61PCnlOJEJbuuezb2VW0ibsud5CA4Mi0tf9ET790XSOFd5nCjQ +6rr06AkjQsoFvjL1dO9EzVFPW0JrZ3C9y8ZOjdeAfPEmFL2T6VqmQ+IcCUNhSr39 +NBdKrboEFfnKanfbstekhAs= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIGMDCCBRigAwIBAgIJAJYf8T95ptq5MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTgwMTE5 +MTkwOTA3WhcNMjgwMTE3MTkwOTA3WjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO +Q2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0 +aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA+fScoT4GHu54WWdiBoWpJTYLUZA/uQVbO5RFqL2EQZV6TwOj7mh2rdua +yPBBmB5pbnz4Xq4NkLurAhpmWSqO04I5Zg3MFUuIZHm7RR+zmcuSJe2NmSHYiu16 +fWdBD6KsGnGKMAdGvFoWWCNlZuBMwjSpev4bZMI4J9iVxos8bxrf9ZDaj/xgQF/r +44HmoTrJ/I1YfeiqRRyW0oPN3F+LfDOxSb+8jaAy7zkR57ygjl20Jq+mTBnGTfD8 +EvGsEN9ZIWU6RMfoQnvj4hs3jpcaWYpl1gwBlgOLWefCmv0Lzu1UGw+Qoteujg77 +7PrRF3wVBMbT3EvtNiASVbc4QZ2vVQIDAQABo4IC8TCCAu0wggEwBgNVHREEggEn +MIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBpZGVudGlmaWVyoDUG +BisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMCAQGhDDAKGwh1c2Vy +bmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUub3JnpGcwZTELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGlybmFtZSBleGFtcGxl +hhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAAAAAAAAAAAAAAAAAA +AYgEKgMEBTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG +AQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFH9ye3+WhBnHqNhtFu059bzY +SWM8MIGPBgNVHSMEgYcwgYSAFH9ye3+WhBnHqNhtFu059bzYSWM8oWGkXzBdMQsw +CQYDVQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5 +dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRAwDgYDVQQDDAdhbGxzYW5zggkAlh/x +P3mm2rkwgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3Rj +YS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzAB +hilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNV +HR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3Rj +YS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAQEAYwYJcerUPvnsP7e2 +HGp/It0OZ8Cvpt8Qf7A+NSPvJqkyKakl8zK/50iq/qQKH09CnfEae4rfXLdlYsvV +2PZYK0LDWnyTcHSJWAVJjlSFIFt3ig9FdHv9GYtSWWod66cZ0sEZOoF2IHZUGby+ +Qa+JQpmv5jEuGIZzjcsh6hSOou8ph7LsCsRdVlQqk8rM97vB7DAgh01vedlbolsq +JxsuPRydNFV/eWq3AgAWgZL3LdYYIAgaVOTnnd3xARw8DlT1q6+Lzc71GBXrRZYh +qgd+xC/K1812gMPImTX02bxpkhCuIdVd7cztWi8sdQmSgDEFdYMXo4NzlFTK8dlC +Y4wa3Q== +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/badcert.pem b/src/greentest/3.7/badcert.pem new file mode 100644 index 0000000..c419146 --- /dev/null +++ b/src/greentest/3.7/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/badkey.pem b/src/greentest/3.7/badkey.pem new file mode 100644 index 0000000..1c8a955 --- /dev/null +++ b/src/greentest/3.7/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/capath/4e1295a3.0 b/src/greentest/3.7/capath/4e1295a3.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.7/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/capath/5ed36f99.0 b/src/greentest/3.7/capath/5ed36f99.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.7/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/capath/6e88d7b8.0 b/src/greentest/3.7/capath/6e88d7b8.0 new file mode 100644 index 0000000..9d7ac23 --- /dev/null +++ b/src/greentest/3.7/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/capath/99d0fa06.0 b/src/greentest/3.7/capath/99d0fa06.0 new file mode 100644 index 0000000..e7dfc82 --- /dev/null +++ b/src/greentest/3.7/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/capath/b1930218.0 b/src/greentest/3.7/capath/b1930218.0 new file mode 100644 index 0000000..07556ff --- /dev/null +++ b/src/greentest/3.7/capath/b1930218.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJAILtv0HIgJGbMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yODAxMTcx +OTA5MDZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMMYaWvJRymYjrFWwi76Dl68I4CzB2Ik0kJb +8Uq/qcghdcjj5iwfhzxufBvtOTKVt0CyYEjDmhYI/m1niDQ7d3dwHHBa0R9fBCFU +uQzjQYUdWO4v7fMO79gjofpz+0wo4OXmTQsCUkmGx75+veZWdotwjgqPBjMgHXtb +qtDFG6ubzFQJPL/kQGbx+9b3Fp3EGdTD8v8HvG9aniUbAkql7EKWOnDSbJkrzr7o +0gHv1bqwz5Q+gtAB1ktxgAMKEkWGeYHYS9LotbcsbJpMihAQ5OT1386EkcrRRuCE +cxdm22lDeICDvhRN8T4a1mz13kXzOa+R1T1URL9BzHNoGvwk25ECAwEAAaNQME4w +HQYDVR0OBBYEFJrPz27rcT3bPPGuiGtWcgPLCKdIMB8GA1UdIwQYMBaAFJrPz27r +cT3bPPGuiGtWcgPLCKdIMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +ABAlyNwMVVzLg25573fsDY4MBsFLDNb3dVIhuBdKOIids3jEQvu4fBQ4EPus2hEA +W0KHXkWfbU5CpJoYBjkPRaaWiTLWWbPTjuOVtsSiS3QvZ8H7u/lybzdK5/RIM3Hf +uPXmQT/V1S8mCfgOkv9w6varWPuQBNZDLo+x+waradDcqPhbB/LUZh9j+F3BnkFE +u8nofeBG5KfIMl8xYuUcXIndt6JPng0TuF+xhFNMH84Z4QEAXr9BVZSppRPb8lnz +1k65nZ25CtmyGG18sfeWqr32+ZUPSm48fEZb39R47Jrc4uMB5oh3OZOcuipj+SVL +T6wIeTnGe98HNbrAwlC/WoE= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/capath/ceff1710.0 b/src/greentest/3.7/capath/ceff1710.0 new file mode 100644 index 0000000..07556ff --- /dev/null +++ b/src/greentest/3.7/capath/ceff1710.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJAILtv0HIgJGbMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yODAxMTcx +OTA5MDZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMMYaWvJRymYjrFWwi76Dl68I4CzB2Ik0kJb +8Uq/qcghdcjj5iwfhzxufBvtOTKVt0CyYEjDmhYI/m1niDQ7d3dwHHBa0R9fBCFU +uQzjQYUdWO4v7fMO79gjofpz+0wo4OXmTQsCUkmGx75+veZWdotwjgqPBjMgHXtb +qtDFG6ubzFQJPL/kQGbx+9b3Fp3EGdTD8v8HvG9aniUbAkql7EKWOnDSbJkrzr7o +0gHv1bqwz5Q+gtAB1ktxgAMKEkWGeYHYS9LotbcsbJpMihAQ5OT1386EkcrRRuCE +cxdm22lDeICDvhRN8T4a1mz13kXzOa+R1T1URL9BzHNoGvwk25ECAwEAAaNQME4w +HQYDVR0OBBYEFJrPz27rcT3bPPGuiGtWcgPLCKdIMB8GA1UdIwQYMBaAFJrPz27r +cT3bPPGuiGtWcgPLCKdIMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +ABAlyNwMVVzLg25573fsDY4MBsFLDNb3dVIhuBdKOIids3jEQvu4fBQ4EPus2hEA +W0KHXkWfbU5CpJoYBjkPRaaWiTLWWbPTjuOVtsSiS3QvZ8H7u/lybzdK5/RIM3Hf +uPXmQT/V1S8mCfgOkv9w6varWPuQBNZDLo+x+waradDcqPhbB/LUZh9j+F3BnkFE +u8nofeBG5KfIMl8xYuUcXIndt6JPng0TuF+xhFNMH84Z4QEAXr9BVZSppRPb8lnz +1k65nZ25CtmyGG18sfeWqr32+ZUPSm48fEZb39R47Jrc4uMB5oh3OZOcuipj+SVL +T6wIeTnGe98HNbrAwlC/WoE= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/dh1024.pem b/src/greentest/3.7/dh1024.pem new file mode 100644 index 0000000..a391176 --- /dev/null +++ b/src/greentest/3.7/dh1024.pem @@ -0,0 +1,7 @@ +-----BEGIN DH PARAMETERS----- +MIGHAoGBAIbzw1s9CT8SV5yv6L7esdAdZYZjPi3qWFs61CYTFFQnf2s/d09NYaJt +rrvJhIzWavqnue71qXCf83/J3nz3FEwUU/L0mGyheVbsSHiI64wUo3u50wK5Igo0 +RNs/LD0irs7m0icZ//hijafTU+JOBiuA8zMI+oZfU7BGuc9XrUprAgEC +-----END DH PARAMETERS----- + +Generated with: openssl dhparam -out dh1024.pem 1024 diff --git a/src/greentest/3.7/idnsans.pem b/src/greentest/3.7/idnsans.pem new file mode 100644 index 0000000..b4a771c --- /dev/null +++ b/src/greentest/3.7/idnsans.pem @@ -0,0 +1,136 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDeU8YQtyeEjPLA +SdMBPTS9QcuAZIJjbJRgr8nsRb767pbmWR9C1JuDy/Bz/AprFC6Om950fLn3pOqR +zDUWZX/qTe+o27i8u0Qzk06bhRkxAdTEoTfRcH/FkJaimJqeTt9rZqc+AGSNKM8o +4GyPW4IELnavmMB30+7rKIJMIpIn1a1k6MybJYdWNSuVqwArAVvRlj5qOiqX7KAS +otFRP8pz+Lgw3qREQzgnZz/bcScKd+5Uy4qMFPNOMjgW6nDV60ekNx0GT+59E/+8 +64GRq34rNVu2SN0XXcQh33R3LwwrvAdymaLyr1YyIRM5gLPxugxCIA0SYjG0YoGB +uUSwtNa7AgMBAAECggEBAJjxUGPXW1wYCja1km5byJgZVwEwI3J6E2igBWyAXm0J +DM3RqWu0DneQKA3h6NjYvV5lY5cG5nex/5vkuvB5SpHIo4GqBV/wA27ne0AJQ9cu +x0utDFUL6xnh6X5ZNKSK5a9gotRIOOPSmxAnswa7kKmHvSX3ExBbvxQOffQaJCk5 +0GHl6I/HltqVzMu4ICAo0NY0gw1n+hVKTo28KkJ9PL7X6v6H5yvZ3L6TkMytSvqf +9iVlYuIN66ToBtxaI4g2RiUJtA2hdT9IP7Wg4YD6Ptyih90zXz2wTzWppFem6UA9 +dePig94R9moj9ucuK0tx3kSATNo0op/XEx1e3OOtcQECgYEA/w7pNOPYgj7VMyYx +p4Lx4BOllzQts8mIBtUVZVQSJ2miun6DTalZVT2V3ayTuE0qhUHd1SHu9F77a9fQ +qaSUUY9elwXyfvcNCfhYVRJxyxirI4Z6ZCBwjpWOGSBB59NTeDhVnbkTlfE6guqS +3KRS1pfIQ6FCvGIrhjRZgHo1TGECgYEA3yXsospbOS7VeBj0UPSB87fp1QM+r48o +RflIsRzdsN9Ka2j6EiYpgKdbgXr80vkctYTK0dT8jrFSk81Y932CZezH2IWo8Meo +40qaFWMboNFBIC4yv6RSRxJMQfYsKnXC2trSnXH+qf55Trey4uZNMX7VJ+RFKExS +ieSWSbTWmJsCgYEAzo3yyoRiiEf+PKgHulLPMtp2VddJ07m30WCrLR5CfWyM/l8K +UtB8qg1v2s+x6aWEc9p9necXLwvkrNdgAqJoAw0KW1/TnILSKmrWjj6brRBTODfl +0kR7It128F4xQV7g0BE/NLX3aIytB+yT9t+Uvni5FBv6gbk26j5m5ScTFsECgYEA +hzrQYQcIqWq8av+Ub8r9Rdlal4BT6Mh0u5MKfmrj3mAzFUyU35LI6/J//cOum5vj +zg0fbHIKa98CEBgNpk4lS+dmZMz7SI92xedb4UIiaB7nvLzCfGj0g6WPGRo6QbED +2OVrZYbDsflJQm8ItYCjny8htf8b+gPmsTIZ8ajps6kCgYBnES8waDDAkL98lK28 +dcgnJXN+1UzeI6//If2uvDZEQ9tG/yMk2JYc84qZJLU5bRplMAjIQUVUcFWa+ZzV +ylnDhagAtiWkHPcElWHym9dH8CRuYM3OTDsApZ7yMB/ArCcZMIA35OvNf6uc4lNV +VD9VkaygPIg6ilv4npeTceqp8A== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9f + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 19 19:09:07 2018 GMT + Not After : Nov 28 19:09:07 2027 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:de:53:c6:10:b7:27:84:8c:f2:c0:49:d3:01:3d: + 34:bd:41:cb:80:64:82:63:6c:94:60:af:c9:ec:45: + be:fa:ee:96:e6:59:1f:42:d4:9b:83:cb:f0:73:fc: + 0a:6b:14:2e:8e:9b:de:74:7c:b9:f7:a4:ea:91:cc: + 35:16:65:7f:ea:4d:ef:a8:db:b8:bc:bb:44:33:93: + 4e:9b:85:19:31:01:d4:c4:a1:37:d1:70:7f:c5:90: + 96:a2:98:9a:9e:4e:df:6b:66:a7:3e:00:64:8d:28: + cf:28:e0:6c:8f:5b:82:04:2e:76:af:98:c0:77:d3: + ee:eb:28:82:4c:22:92:27:d5:ad:64:e8:cc:9b:25: + 87:56:35:2b:95:ab:00:2b:01:5b:d1:96:3e:6a:3a: + 2a:97:ec:a0:12:a2:d1:51:3f:ca:73:f8:b8:30:de: + a4:44:43:38:27:67:3f:db:71:27:0a:77:ee:54:cb: + 8a:8c:14:f3:4e:32:38:16:ea:70:d5:eb:47:a4:37: + 1d:06:4f:ee:7d:13:ff:bc:eb:81:91:ab:7e:2b:35: + 5b:b6:48:dd:17:5d:c4:21:df:74:77:2f:0c:2b:bc: + 07:72:99:a2:f2:af:56:32:21:13:39:80:b3:f1:ba: + 0c:42:20:0d:12:62:31:b4:62:81:81:b9:44:b0:b4: + d6:bb + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:idnsans, DNS:xn--knig-5qa.idn.pythontest.net, DNS:xn--knigsgsschen-lcb0w.idna2003.pythontest.net, DNS:xn--knigsgchen-b4a3dun.idna2008.pythontest.net, DNS:xn--nxasmq6b.idna2003.pythontest.net, DNS:xn--nxasmm1c.idna2008.pythontest.net + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 3B:F0:22:A0:1E:9B:CE:2A:7C:AE:B1:32:1B:B0:8E:3E:33:40:E3:FA + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:82:ED:BF:41:C8:80:91:9B + + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + + X509v3 CRL Distribution Points: + + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + + Signature Algorithm: sha1WithRSAEncryption + 8b:1f:d7:e4:0d:15:76:b4:f5:87:33:de:b9:84:9b:f2:c1:9b: + c9:97:50:f7:18:33:ed:b7:60:83:be:bb:94:1c:49:39:ae:54: + 24:43:f7:85:d8:2a:8c:26:17:56:1e:a6:b7:63:c5:05:f1:6e: + f4:79:eb:fd:af:12:84:3c:28:4a:8f:b1:01:97:91:ba:18:2b: + ba:54:25:49:1b:5b:2e:1e:6b:33:2d:f5:07:2e:76:04:e0:a8: + 95:25:3f:cc:c8:26:c0:30:b6:90:d2:2b:e1:e2:13:b0:a8:76: + f0:06:90:b9:d5:28:6b:8a:e9:72:1a:ed:4f:7e:3c:37:2e:00: + aa:9b:f1:29:44:94:f2:dc:c8:31:5f:4c:2d:00:d3:5e:78:6c: + 68:fc:0e:1e:46:be:d8:2e:29:88:78:8e:7e:f5:50:c8:5c:5d: + 5f:4c:09:d5:51:07:40:be:9b:30:ed:a3:29:68:25:6b:88:69: + c7:43:35:54:2f:6e:9a:30:f1:d6:87:54:84:20:ef:a5:aa:33: + df:00:6a:87:a9:b4:d7:89:1f:e7:60:0d:01:60:66:11:61:3f: + d0:9f:86:37:cc:b3:b8:48:7e:1f:d2:7a:0f:02:e7:11:1d:dd: + 34:c4:0b:45:47:2b:05:37:dd:ee:6e:0e:1c:bd:de:24:42:50: + a4:07:af:e5 +-----BEGIN CERTIFICATE----- +MIIFvTCCBKWgAwIBAgIJAILtv0HIgJGfMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDdaFw0yNzExMjgx +OTA5MDdaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2lk +bnNhbnMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeU8YQtyeEjPLA +SdMBPTS9QcuAZIJjbJRgr8nsRb767pbmWR9C1JuDy/Bz/AprFC6Om950fLn3pOqR +zDUWZX/qTe+o27i8u0Qzk06bhRkxAdTEoTfRcH/FkJaimJqeTt9rZqc+AGSNKM8o +4GyPW4IELnavmMB30+7rKIJMIpIn1a1k6MybJYdWNSuVqwArAVvRlj5qOiqX7KAS +otFRP8pz+Lgw3qREQzgnZz/bcScKd+5Uy4qMFPNOMjgW6nDV60ekNx0GT+59E/+8 +64GRq34rNVu2SN0XXcQh33R3LwwrvAdymaLyr1YyIRM5gLPxugxCIA0SYjG0YoGB +uUSwtNa7AgMBAAGjggKOMIICijCB4QYDVR0RBIHZMIHWggdpZG5zYW5zgh94bi0t +a25pZy01cWEuaWRuLnB5dGhvbnRlc3QubmV0gi54bi0ta25pZ3Nnc3NjaGVuLWxj +YjB3LmlkbmEyMDAzLnB5dGhvbnRlc3QubmV0gi54bi0ta25pZ3NnY2hlbi1iNGEz +ZHVuLmlkbmEyMDA4LnB5dGhvbnRlc3QubmV0giR4bi0tbnhhc21xNmIuaWRuYTIw +MDMucHl0aG9udGVzdC5uZXSCJHhuLS1ueGFzbW0xYy5pZG5hMjAwOC5weXRob250 +ZXN0Lm5ldDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG +AQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFDvwIqAem84qfK6xMhuwjj4z +QOP6MH0GA1UdIwR2MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYD +VQQGEwJYWTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0Ex +FjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEE +dzB1MDwGCCsGAQUFBzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9weWNhY2VydC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0 +aG9udGVzdC5uZXQvdGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6 +Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0G +CSqGSIb3DQEBBQUAA4IBAQCLH9fkDRV2tPWHM965hJvywZvJl1D3GDPtt2CDvruU +HEk5rlQkQ/eF2CqMJhdWHqa3Y8UF8W70eev9rxKEPChKj7EBl5G6GCu6VCVJG1su +HmszLfUHLnYE4KiVJT/MyCbAMLaQ0ivh4hOwqHbwBpC51ShriulyGu1Pfjw3LgCq +m/EpRJTy3MgxX0wtANNeeGxo/A4eRr7YLimIeI5+9VDIXF1fTAnVUQdAvpsw7aMp +aCVriGnHQzVUL26aMPHWh1SEIO+lqjPfAGqHqbTXiR/nYA0BYGYRYT/Qn4Y3zLO4 +SH4f0noPAucRHd00xAtFRysFN93ubg4cvd4kQlCkB6/l +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/keycert.passwd.pem b/src/greentest/3.7/keycert.passwd.pem new file mode 100644 index 0000000..0ad6960 --- /dev/null +++ b/src/greentest/3.7/keycert.passwd.pem @@ -0,0 +1,50 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,E74528136B90D2DD + +WRHVD2PJXPqjFSHg92HURIsUzvsTE4a9oi0SC5yMBFKNWA5Z933gK3XTifp6jul5 +zpNYi8jBXZ2EqJJBxCuVcefmXSxL0q7CMej25TdIC4BVAFJVveeprHPUFkNB0IM1 +go5Lg4YofYqTCg3OE3k7WvfR3Zg1cRYxksDKO+WNZgWyKBex5X4vjOiyUqDl3GKt +kQXnkg1VgPV2Vrx93S9XNdELNRTguwf+XG0fkhtYhp/zCto8uKTgy5elK2P/ulGp +7fe6uj7h/uN9L7EOC6CjRkitywfeBUER739mOcGT4imSFJ9G27TCqPzj2ea3uuaf +/v1xhkQ4M6lNY/gcRfgVpCXhW43aAQV8XXQRMJTqLmz5Y5hYTKn+Ugq5vJ/ngyRM +lu1gUJnYYaemBTb4hbm6aBvnYK9mORa891Pmf+vxU9rYuQIdVAhvvXh4KBreSEBI +1AFy6dFKXl8ZKs6Wrq5wPefmFFkRmZ8OBiiq0fp2ApCRGZw6LsjCgwrRM38JiY7d +3OdsJpKvRYufgUyuuzUE0xA+E4yMvD48M9pPq2fC8O5giuGL1uEekQWXJuq+6ZRI +XYKIeSkuQALbX3RAzCPXTUEMtCYXKm/gxrrwJ+Bet4ob2amf3MX0uvWwOuAq++Fk +J0HFSBxrwvIWOhyQXOPuJdAN8PXA7dWOXfOgOMF0hQYqZCl3T4TiVZJbwVQtg1sN +dO7oAD5ZPHiKzvveZuB6k1FlBG8j0TyAC+44ChxkPDD3jF4dd6zGe62sDf85p4/d +W80gxJeD3xnDxG0ePPns+GuKUpUaWS7WvHtDpeFW1JEhvOqf8p1Li9a7RzWVo8ML +mGTdQgBIYIf6/fk69pFKl0nKtBU75KaunZz4nAmd9bNED4naDurMBg44u5TvODbJ +vgYIYXIYjNvONbskJatVrrTS8zch2NwVIjCi8L/hecwBXbIXzo1pECpc6BU7sQT8 ++i9sDKBeJcRipzfKZNHvnO19mUZaPCY8+a/f9c21DgKXz+bgLcJbohpSaeGM8Gfc +aZd3Vp9n3OJ3g2zQR1++HO9v1vR/wLELu6MeydkvMduHLmOPCn54gZ9z51ZNPAwa +qfFIsH+mLh9ks0H74ssF59uIlstkgB9zmZHv/Q0dK9ZfG/VEH6rSgdETWhZxhoMQ +Z92jXBEFT0zhI3rrIPNY+XS7eJCQIc1wc84Ea3cRk7SP+S1og3JtAxX56ykUwtkM +LQ/Dwwa6h1aqD0l2d5x1/BSdavtTuSegISRWQ4iOmSvEdlFP7H4g6RZk/okbLzMD +Evq5gNc7vlXhVawoQU8JCanJ5ZbbWnIRZfiXxBQS4lpYPKvJt4ML9z/x+82XxcXv +Z93N2Wep7wWW5OwS2LcQcOgZRDSIPompwo/0pMFGOS+5oort0ZDRHdmmGLjvBcCb +1KQmKQ4+8brI/3rjRzts6uDLjTGNxSCieNsnqhwHUv9Mg9WDSWupcGa+x27L89x3 +rObf6+3umcFLSjIzU8wuv1hx/e/y98Kv7BDBNYpAr6kVMrLnzYjAfJbBmqlxkzkQ +IgQzgrk2QZoTdgwR+S374NAMO0AE5IlO+/qa6qp2SORGTDX64I3UNw== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx +MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 +RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj +9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj +SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN +sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn +96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ +bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK +Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk +ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 +qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu +4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS +dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/keycert.pem b/src/greentest/3.7/keycert.pem new file mode 100644 index 0000000..9545dcf --- /dev/null +++ b/src/greentest/3.7/keycert.pem @@ -0,0 +1,48 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCr77F9oBLYuLIb +3t4mDHPoxOEnu1h7NQaJdn9tR/KjW6AzhXXm9USh45qt3sR6Wo8sDlBIJ3vZyhyW +P939qYUeGOZJahupYi4IkqVckZXARm3k9qhAI/aC4ixVFyxnxg3bNpk8Ir0AyyQV +oudY+33+ZNj6+fHzeboGDJ9uE/RTcp9JqE/qo0haATZufJfY63ZCUpYFn6j5W4jG +vpqg5/0hba8Cxdk62387uNknfVHSMzGkkq82zbBpud6TYQofCp3VlEPBjz9iLCz6 +FFOKgLZmbk8QsdktXF6zfRJJk+vbZTh/OGH0p/eiIfW1kXOzOcuW31XRFTPnykJd +4QUX9OajAgMBAAECggEAHppmXDbuw9Z0FVPg9KLIysioTtsgz6VLiZIm8juZK4x2 +glUh/D7xvWL2uDXrgN+3lh7iGUW13LkFx5SMncbbo9TIwI57Z/XKvcnkVwquve+L +RfLFVc1Q5lD9lROv2rS86KTaN4LzYz3FKXi6dvMkpPAsUtfEQhMLkmISypQQq/1z +EJaqo7r85OjN7e0wKazlKZpOzJEa5FQLMVRjTRFhLFNbHXX/tAet2jw+umATKbw8 +hYgiuZ44TwSEd9JeIV/oSYWfI/3HetuYW0ru3caiztRF2NySNu8lcsWgNC7fIku9 +mcHjtSNzs91QN1Qlu7GQvvhpt6OWDirNDCW+49WGaQKBgQDg9SDhfF0jRYslgYbH +cqO4ggaFdHjrAAYpwnAgvanhFZL/zEqm5G1E7l/e2fCkJ9VOSFO0A208chvwMcr+ +dCjHE2tVdE81aQ2v/Eo83VdS1RcOV4Y75yPH48rMhxPaHvxWD/FFDbf0/P2mtPB7 +SZ3kIeZMkE1wxdaO3AKUbQoozwKBgQDDqYgg7kVtygyICE1mB8Hwp6nUxFTczG7y +4XcsDqMIrKmw+PbQluvkoHoStxeVrsTloDhkTjIrpmYLyAiazg+PUJdkd6xrfLSj +VV6X93W0S/1egEb1F1CGFxtk8v/PWH4K76EPL2vxXdxjywz3GWlrL9yDYaB2szzS +DqgwVMqx7QKBgDCD7UF0Bsoyl13RX3XoPXLvZ+SkR+e2q52Z94C4JskKVBeiwX7Y +yNAS8M4pBoMArDoj0xmBm69rlKbqtjLGbnzwrTdSzDpim7cWnBQgUFLm7gAD1Elb +AhZ8BCK0Bw4FnLoa2hfga4oEfdfUMgEE0W5/+SEOBgWKRUmuHUhRc911AoGAY2EN +YmSDYSM5wDIvVb5k9B3EtevOiqNPSw/XnsoEZtiEC/44JnQxdltIBY93bDBrk5IQ +cmoBM4h91kgQjshQwOMXMhFSwvmBKmCm/hrTbvMVytTutXfVD3ZXFKwT4DW7N0TF +ElhsxBh/YzRz7mG62JVjtFt2zDN3ld2Z8YpvtXUCgYEA4EJ4ObS5YyvcXAKHJFo6 +Fxmavyrf8LSm3MFA65uSnFvWukMVqqRMReQc5jvpxHKCis+XvnHzyOfL0gW9ZTi7 +tWGGbBi0TRJCa8BkvgngUZxOxUlMfg/7cVxOIB0TPoUSgxFd/+qVz4GZMvr0dPu7 +eAF7J/8ECVvb0wSPTUI1N3c= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx +MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 +RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj +9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj +SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN +sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn +96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ +bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK +Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk +ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 +qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu +4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS +dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/keycert2.pem b/src/greentest/3.7/keycert2.pem new file mode 100644 index 0000000..bb5fa65 --- /dev/null +++ b/src/greentest/3.7/keycert2.pem @@ -0,0 +1,49 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC3ulRNfhbOAey/ +B+wIVYx+d5az7EV4riR6yi/qE6G+bxbTvay2pqySHtDweuaYSh2cVmcasBKKIFJm +rCD1zR8UmLb5i2XFIina1t3eePCuBZMrvZZwkzlQUSM1AZtjGOO/W0I3FwO6y645 +9xA5PduKI7SMYkH/VL3zE5W1JwMovv6bvNiT+GU5l6mB9ylCTgLpmUqoQhRqz/35 +zCzVyoh+ppDvVcpWYfvXywsXsgQwbAF0QJm8SSFi0TZm5ykv4WE16afQp08yuZS0 +3U4K3MJCa4rxO58edcxBopWYfQ29K3iINM8enRfr5q+u5mAAbALAEEvyFjgLWl/u +7arxn7bJAgMBAAECggEBAJfMt8KfHzBunrDnVrk8FayYGkfmOzAOkc1yKEx6k/TH +zFB+Mqlm5MaF95P5t3S0J+r36JBAUdEWC38RUNpF9BwMYYGlDxzlsTdCuGYL/q+J +o6NMLXQt7/jQUQqGnWAvPFzqhbcGqOo5R2ZVH25sEWv9PDuRI35XAepIkDTwWsfa +P6UcJJoP+4v9B++fb3sSL4zNwp1BqS4wxR8YTR0t1zQqOxJ5BGPw1J8aBMs1sq5t +qyosAQAT63kLrdqWotHaM26QxjqEQUMlh12XMWb5GdBXUxbvyGtEabsqskGa/f8B +RdHE437J8D8l+jxb2mZLzrlaH3dq2tbFGCe1rT8qLRECgYEA5CWIvoD/YnQydLGA +OlEhCSocqURuqcotg9Ev0nt/C60jkr/NHFLGppz9lhqjIDjixt3sIMGZMFzxRtwM +pSYal3XiR7rZuHau9iM35yDhpuytEiGbYy1ADakJRzY5jq/Qa8RfPP9Atua5xAeP +q6DiSnq9vhHv9G+O4MxzHBmrw9sCgYEAziiJWFthcwvuXn3Jv9xFYKEb/06puZAx +EgQCz/3rPzv5fmGD/sKVo1U/K4z/eA82DNeKG8QRTFJCxT8TCNRxOmGV7HdCYo/B +4BTNNvbKcdi3l0j75kKoADg+nt5CD5lz6gLG0GrUEnVO1y5HVfCTb3BEAfa36C85 +9i0sfQGiwysCgYEAuus9k8cgdct5oz3iLuVVSark/JGCkT2B+OOkaLChsDFUWeEm +7TOsaclpwldkmvvAYOplkZjMJ2GelE2pVo1XcAw3LkmaI5WpVyQXoxe/iQGT8qzy +IFlsh0Scw2lb0tmcyw6CcPk4TiHOxRrkzNrtS9QwLM+JZx0XVHptPPKTVc0CgYAu +j/VFYY5G/8Dc0qhIjyWUR48dQNUQtkJ/ASzpcT46z/7vznKTjbtiYpSb74KbyUO5 +7sygrM4DYOj3x+Eys1jHiNbly6HQxQtS4x/edCsRP5NntfI+9XsgYZOzKhvdjhki +F3J0DEzNxnUCIM+311hVaRPTJbgv1srOkTFlIoNydQKBgQC6/OHGaC/OewQqRlRK +Mg5KZm01/pk4iKrpA5nG7OTAeoa70NzXNtG8J3WnaJ4mWanNwNUOyRMAMrsUAy9q +EeGqHM5mMFpY4TeVuNLL21lu/x3KYw6mKL3Ctinn+JLAoYoqEy8deZnEA5/tjYlz +YhFBchnUicjoUN1chdpM6SpV2Q== +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDYjCCAkqgAwIBAgIJALJXRr8qF6oIMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTAeFw0x +ODAxMTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMGIxCzAJBgNVBAYTAlhZMRcwFQYD +VQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZv +dW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALe6VE1+Fs4B7L8H7AhVjH53lrPsRXiuJHrKL+oTob5v +FtO9rLamrJIe0PB65phKHZxWZxqwEoogUmasIPXNHxSYtvmLZcUiKdrW3d548K4F +kyu9lnCTOVBRIzUBm2MY479bQjcXA7rLrjn3EDk924ojtIxiQf9UvfMTlbUnAyi+ +/pu82JP4ZTmXqYH3KUJOAumZSqhCFGrP/fnMLNXKiH6mkO9VylZh+9fLCxeyBDBs +AXRAmbxJIWLRNmbnKS/hYTXpp9CnTzK5lLTdTgrcwkJrivE7nx51zEGilZh9Db0r +eIg0zx6dF+vmr67mYABsAsAQS/IWOAtaX+7tqvGftskCAwEAAaMbMBkwFwYDVR0R +BBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBCwUAA4IBAQCZhHhGItpkqhEq +ntMRd6Hv0GoOJixNvgeMwK4NJSRT/no3OirtUTzccn46h+SWibSa2eVssAV+pAVJ +HbzkN/DH27A1mMx1zJL1ekcOKA1AF6MXhUnrUGXMqW36YNtzHfXJLrwvpLJ13OQg +/Kxo4Nw68bGzM+PyRtKU/mpgYyfcvwR+ZSeIDh1fvUZK/IEVCf8ub42GPVs5wPfv +M+k5aHxWTxeif3K1byTRzxHupYNG2yWO4XEdnBGOuOwzzN4/iQyNcsuQKeuKHGrt +YvIlG/ri04CQ7xISZCj74yjTZ+/A2bXre2mQXAHqKPumHL7cl34+erzbUaxYxbTE +u5FcOmLQ +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/keycert3.pem b/src/greentest/3.7/keycert3.pem new file mode 100644 index 0000000..621eb08 --- /dev/null +++ b/src/greentest/3.7/keycert3.pem @@ -0,0 +1,132 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDgV4G+Zzf2DT5n +oAisIGFhn/bz7Vn5WiXUqbDsxROJOh/7BtOlduZka0pPhFylGbnxS8l1kEWHRI2T +6hOoWzumB6ItKiN+T5J30lAvSyo7iwdFoAQ/S5nPXQfhNARQe/NEOhRtpcuNdyx4 +BWdPdPuJQA1ASNJCLwcLOoRxaLbKLvb2V5T5FCAkeNPtRvPuT4gKQItMmiHfAhoV +C8MZWF/GC0RukHINys5MwqeFexam8CznmQPMYrLdhmKTj3DTivCPoh97EDIFGlgZ +SCaaYDVQA+aqlo/q2pi52PFwC1KzhNEA7EeOqSwC1NQjwjHuhcnf9WbxrgTq2zh3 +rv5YEW2ZAgMBAAECggEAPfSMtTumPcJskIumuXp7yk02EyliZrWZqwBuBwVqHsS5 +nkbFXnXWrLbgn9MrDsFrE5NdgKUmPnQVMVs8sIr5jyGejSCNCs4I4iRn1pfIgwcj +K/xEEALd6GGF0pDd/CgvB5GOoLVf4KKf2kmLvWrOKJpSzoUN5A8+v8AaYYOMr4sC +czbvfGomzEIewEG+Rw9zOVUDlmwyEKPQZ47E7PQ+EEA7oeFdR+1Zj6eT9ndegf8B +54frySYCLRUCk/sHCpWhaJBtBrcpht7Y8CfY7hiH/7x866fvuLnYPz4YALtUb0wN +7zUCNS9ol3n4LbjFFKfZtiRjKaCBRzMjK0rz6ydFcQKBgQDyLI3oGbnW73vqcDe/ +6eR0w++fiCAVhfMs3AO/gOaJi2la2JHlJ5u+cIHQIOFwEhn6Zq0AtdmnFx1TS5IQ +C0MdXI0XoQQw7rEF8EJcvfe85Z0QxENVhzydtdb8QpJfnQGfBfLyQlaaRYzRRHB6 +VdYUHF3EIPVIhbjbghal+Qep/QKBgQDtJlRPHkWwTMevu0J0fYbWN1ywtVTFUR// +k7VyORSf8yuuSnaQRop4cbcqONxmDKH6Or1fl3NYBsAxtXkkOK1E2OZNo2sfQdRa +wpA7o7mPHRhztQFpT5vflp+8P6+PEFat8D04eBOhNwrwwfhiPjD4gv5KvN4XutRW +VWv/2pnmzQKBgHPvHGg2mJ7quvm6ixXW1MWJX1eSBToIjCe3lBvDi5nhIaiZ8Q4w +7gA3QA3xD7tlDwauzLeAVxgEmsdbcCs6GQEfY3QiYy1Bt4FOSZa4YrcNfSmfq1Rw +j3Y4rRjKjeQz96i3YlzToT3tecJc7zPBj+DEy6au2H3Fdn+vQURneWHJAoGBANG7 +XES8mRVaUh/wlM1BVsaNH8SIGfiHzqzRjV7/bGYpQTBbWpAuUrhCmaMVtpXqBjav +TFwGLVRkZAWSYRjPpy2ERenT5SE3rv61o6mbGrifGsj6A82HQmtzYsGx8SmtYXtj +REF0sKebbmmOooUAS379GrguYJzL9o6D7YfRZNrhAoGAVfb/tiFU4S67DSpYpQey +ULhgfsFpDByICY6Potsg67gVFf9jIaB83NPTx3u/r6sHFgxFw7lQsuZcgSuWMu7t +glzOXVIP11Y5sl5CJ5OsfeK1/0umMZF5MWPyAQCx/qrPlZL86vXjt24Y/VaOxsAi +CZYdyJsjgOrJrWoMbo5ta54= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9c + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 19 19:09:06 2018 GMT + Not After : Nov 28 19:09:06 2027 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:e0:57:81:be:67:37:f6:0d:3e:67:a0:08:ac:20: + 61:61:9f:f6:f3:ed:59:f9:5a:25:d4:a9:b0:ec:c5: + 13:89:3a:1f:fb:06:d3:a5:76:e6:64:6b:4a:4f:84: + 5c:a5:19:b9:f1:4b:c9:75:90:45:87:44:8d:93:ea: + 13:a8:5b:3b:a6:07:a2:2d:2a:23:7e:4f:92:77:d2: + 50:2f:4b:2a:3b:8b:07:45:a0:04:3f:4b:99:cf:5d: + 07:e1:34:04:50:7b:f3:44:3a:14:6d:a5:cb:8d:77: + 2c:78:05:67:4f:74:fb:89:40:0d:40:48:d2:42:2f: + 07:0b:3a:84:71:68:b6:ca:2e:f6:f6:57:94:f9:14: + 20:24:78:d3:ed:46:f3:ee:4f:88:0a:40:8b:4c:9a: + 21:df:02:1a:15:0b:c3:19:58:5f:c6:0b:44:6e:90: + 72:0d:ca:ce:4c:c2:a7:85:7b:16:a6:f0:2c:e7:99: + 03:cc:62:b2:dd:86:62:93:8f:70:d3:8a:f0:8f:a2: + 1f:7b:10:32:05:1a:58:19:48:26:9a:60:35:50:03: + e6:aa:96:8f:ea:da:98:b9:d8:f1:70:0b:52:b3:84: + d1:00:ec:47:8e:a9:2c:02:d4:d4:23:c2:31:ee:85: + c9:df:f5:66:f1:ae:04:ea:db:38:77:ae:fe:58:11: + 6d:99 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 85:11:BE:16:47:04:D1:30:EE:86:8A:18:70:BE:A8:28:6F:82:3D:CE + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:82:ED:BF:41:C8:80:91:9B + + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + + X509v3 CRL Distribution Points: + + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + + Signature Algorithm: sha1WithRSAEncryption + 7f:a1:7e:3e:68:01:b0:32:b8:57:b8:03:68:13:13:b3:e3:f4: + 70:2f:15:e5:0f:87:b9:fd:e0:12:e3:16:f2:91:53:c7:4e:25: + af:ca:cb:a7:d9:9d:57:4d:bf:a2:80:d4:78:aa:04:31:fd:6d: + cc:6d:82:43:e9:62:16:0d:0e:26:8b:e7:f1:3d:57:5c:68:02: + 9c:2b:b6:c9:fd:62:2f:10:85:88:cc:44:a5:e7:a2:3e:89:f2: + 1f:02:6a:3f:d0:3c:6c:24:2d:bc:51:62:7a:ec:25:c5:86:87: + 77:35:8f:f9:7e:d0:17:3d:77:56:bf:1a:0c:be:09:78:ee:ea: + 73:97:65:60:94:91:35:b3:5c:46:8a:5e:6d:94:52:de:48:b7: + 1f:6c:28:79:7f:ff:08:8d:e4:7d:d0:b9:0b:7c:ae:c4:1d:2a: + a1:b3:50:11:82:03:5e:6c:e7:26:fa:05:32:39:07:83:49:b9: + a2:fa:04:da:0d:e5:ff:4c:db:97:d0:c3:a7:43:37:4c:16:de: + 3c:b5:e9:7e:82:d4:b3:10:df:d1:c1:66:72:9c:15:67:19:3b: + 7b:91:0a:82:07:67:c5:06:03:5f:80:54:08:81:8a:b1:5c:7c: + 4c:d2:07:38:92:eb:12:f5:71:ae:de:05:15:c8:e1:33:f0:e4: + 96:0f:0f:1e +-----BEGIN CERTIFICATE----- +MIIE8TCCA9mgAwIBAgIJAILtv0HIgJGcMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx +OTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOBXgb5nN/YN +PmegCKwgYWGf9vPtWflaJdSpsOzFE4k6H/sG06V25mRrSk+EXKUZufFLyXWQRYdE +jZPqE6hbO6YHoi0qI35PknfSUC9LKjuLB0WgBD9Lmc9dB+E0BFB780Q6FG2ly413 +LHgFZ090+4lADUBI0kIvBws6hHFotsou9vZXlPkUICR40+1G8+5PiApAi0yaId8C +GhULwxlYX8YLRG6Qcg3KzkzCp4V7FqbwLOeZA8xist2GYpOPcNOK8I+iH3sQMgUa +WBlIJppgNVAD5qqWj+ramLnY8XALUrOE0QDsR46pLALU1CPCMe6Fyd/1ZvGuBOrb +OHeu/lgRbZkCAwEAAaOCAcAwggG8MBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAOBgNV +HQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1Ud +EwEB/wQCMAAwHQYDVR0OBBYEFIURvhZHBNEw7oaKGHC+qChvgj3OMH0GA1UdIwR2 +MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYDVQQGEwJYWTEmMCQG +A1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91 +ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUF +BzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2Vy +dC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQv +dGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0 +aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBBQUA +A4IBAQB/oX4+aAGwMrhXuANoExOz4/RwLxXlD4e5/eAS4xbykVPHTiWvysun2Z1X +Tb+igNR4qgQx/W3MbYJD6WIWDQ4mi+fxPVdcaAKcK7bJ/WIvEIWIzESl56I+ifIf +Amo/0DxsJC28UWJ67CXFhod3NY/5ftAXPXdWvxoMvgl47upzl2VglJE1s1xGil5t +lFLeSLcfbCh5f/8IjeR90LkLfK7EHSqhs1ARggNebOcm+gUyOQeDSbmi+gTaDeX/ +TNuX0MOnQzdMFt48tel+gtSzEN/RwWZynBVnGTt7kQqCB2fFBgNfgFQIgYqxXHxM +0gc4kusS9XGu3gUVyOEz8OSWDw8e +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/keycert4.pem b/src/greentest/3.7/keycert4.pem new file mode 100644 index 0000000..b7df7f3 --- /dev/null +++ b/src/greentest/3.7/keycert4.pem @@ -0,0 +1,132 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDH/76hZAZH4cSV +CmVZa5HEqKCjCKrcPwBECs9BS+3ibwN4x9NnFNP+tCeFGgJXl7WGFoeXgg3oK+1p +FsOWpsRHuF3BdqkCnShSydmT8bLaGHwKeL0cPxJP5T/uW7ezPKW2VWXGMwmwRaRJ +9dj2VCUu20vDZWSGFr9zjnjoJczBtH3RsVUgpK7euEHuQ5pIM9QSOaCo+5FPR7s7 +1nU7YqbFWtd+NhC8Og1G497B31DQlHciF6BRm6/cNGAmHaAErKUGBFdkGtFPHBn4 +vktoEg9fwxJAZLvGpoTZWrB4HRsRwVTmFdGvK+JXK225xF23AXRXp/snhSuSFeLj +E5cpyJJ7AgMBAAECggEAQOv527X2e/sDr0XSpHZQuT/r9UBpBlnFIlFH+fBF5k0X +GWv0ae/O6U1dzs0kmX57xG0n0ry6+vTXeleTYiH8cTOd66EzN9AAOO+hG29IGZf9 +HAEZkkO/FARc/mjzdtFnEYsjIHWM3ZWdwQx3Q28JKu6w51rQiN51g3NqOCGdF/uF +rE5XPKsKndn+nLHvsNuApFgUYZEwdrozgUueEgRaPTUCNhzotcA9eWoBdA24XNhk +x8Cm/bZWabXm7gBO75zl3Cu2F21ay+EuwyOZTsx6lZi6YX9/zo1mkO81Zi3tQk50 +NMEI0feLNwsdxTbmOcVJadjOgd+QVghlFyr5HGBWMQKBgQD3AH3rhnAo6tOyNkGN ++IzIU1MhUS452O7IavykUYO9sM24BVChpRtlI9Dpev4yE/q3BAO3+oWT3cJrN7/3 +iyo1dzAkpGvI65XWfElXFM4nLjEiZzx4W9fiPN91Oucpr0ED6+BZXTtz4gVm0TP/ +TUc2xvTB6EKvIyWmKOYEi0snxQKBgQDPSOjbz9jWOrC9XY7PmtLB6QJDDz7XSGVK +wzD+gDAPpAwhk58BEokdOhBx2Lwl8zMJi0CRHgH2vNvkRyhvUQ4UFzisrqann/Tw +klp5sw3iWC6ERC8z9zL7GfHs7sK3mOVeAdK6ffowPM3JrZ2vPusVBdr0MN3oZwki +CtNXqbY1PwKBgGheQNbAW6wubX0kB9chavtKmhm937Z5v4vYCSC1gOEqUAKt3EAx +L74wwBmn6rjmUE382EVpCgBM99WuHONQXmlxD1qsTw763LlgkuzE0cckcYaD8L06 +saHa7uDuHrcyYlpx1L5t8q0ol/e19i6uTKUMtGcq6OJwC3yGU4sgAIWxAoGBAMVq +qiQXm2vFL+jafxYoXUvDMJ1PmskMsTP4HOR2j8+FrOwZnVk3HxGP6HOVOPRn4JbZ +YiAT1Uj6a+7I+rCyINdvmlGUcTK6fFzW9oZryvBkjcD483/pkktmVWwTpa2YV/Ml +h16IdsyUTGYlDUYHhXtbPUJOfDpIT4F1j/0wrFGfAoGAO82BcUsehEUQE0xvQLIn +7QaFtUI5z19WW730jVuEobiYlh9Ka4DPbKMvka8MwyOxEwhk39gZQavmfG6+wZm+ +kjERU23LhHziJGWS2Um4yIhC7myKbWaLzjHEq72dszLpQku4BzE5fT60fxI7cURD +WGm/Z3Q2weS3ZGIoMj1RNPI= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9d + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 19 19:09:06 2018 GMT + Not After : Nov 28 19:09:06 2027 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:c7:ff:be:a1:64:06:47:e1:c4:95:0a:65:59:6b: + 91:c4:a8:a0:a3:08:aa:dc:3f:00:44:0a:cf:41:4b: + ed:e2:6f:03:78:c7:d3:67:14:d3:fe:b4:27:85:1a: + 02:57:97:b5:86:16:87:97:82:0d:e8:2b:ed:69:16: + c3:96:a6:c4:47:b8:5d:c1:76:a9:02:9d:28:52:c9: + d9:93:f1:b2:da:18:7c:0a:78:bd:1c:3f:12:4f:e5: + 3f:ee:5b:b7:b3:3c:a5:b6:55:65:c6:33:09:b0:45: + a4:49:f5:d8:f6:54:25:2e:db:4b:c3:65:64:86:16: + bf:73:8e:78:e8:25:cc:c1:b4:7d:d1:b1:55:20:a4: + ae:de:b8:41:ee:43:9a:48:33:d4:12:39:a0:a8:fb: + 91:4f:47:bb:3b:d6:75:3b:62:a6:c5:5a:d7:7e:36: + 10:bc:3a:0d:46:e3:de:c1:df:50:d0:94:77:22:17: + a0:51:9b:af:dc:34:60:26:1d:a0:04:ac:a5:06:04: + 57:64:1a:d1:4f:1c:19:f8:be:4b:68:12:0f:5f:c3: + 12:40:64:bb:c6:a6:84:d9:5a:b0:78:1d:1b:11:c1: + 54:e6:15:d1:af:2b:e2:57:2b:6d:b9:c4:5d:b7:01: + 74:57:a7:fb:27:85:2b:92:15:e2:e3:13:97:29:c8: + 92:7b + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:fakehostname + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + F8:76:79:CB:11:85:F0:46:E5:95:E6:7E:69:CB:12:5E:4E:AA:EC:4D + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:82:ED:BF:41:C8:80:91:9B + + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + + X509v3 CRL Distribution Points: + + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + + Signature Algorithm: sha1WithRSAEncryption + 6d:50:8d:fb:ee:4e:93:8b:eb:47:56:ba:38:cc:80:e1:9d:c7: + e1:9e:1f:9c:22:0c:d2:08:9b:ed:bf:31:d9:00:ee:af:8c:56: + 78:92:d1:7c:ba:4e:81:7f:82:1f:f4:68:99:86:91:c6:cb:57: + d3:b9:41:12:fa:75:53:fd:22:32:21:50:af:6b:4c:b1:34:36: + d1:a8:25:0a:d0:f0:f8:81:7d:69:58:6e:af:e3:d2:c4:32:87: + 79:d7:cd:ad:0c:56:f3:15:27:10:0c:f9:57:59:53:00:ed:af: + 5d:4d:07:86:7a:e5:f3:97:88:bc:86:b4:f1:17:46:33:55:28: + 66:7b:70:d3:a5:12:b9:4f:c7:ed:e6:13:20:2d:f0:9e:ec:17: + 64:cf:fd:13:14:1b:76:ba:64:ac:c5:51:b6:cd:13:0a:93:b1: + fd:43:09:a0:0b:44:6c:77:45:43:0b:e5:ed:70:b2:76:dc:08: + 4a:5b:73:5f:c1:fc:7f:63:70:f8:b9:ca:3c:98:06:5f:fd:98: + d1:e4:e6:61:5f:09:8f:6c:18:86:98:9c:cb:3f:73:7b:3f:38: + f5:a7:09:20:ee:a5:63:1c:ff:8b:a6:d1:8c:e8:f4:84:3d:99: + 38:0f:cc:e0:52:03:f9:18:05:23:76:39:de:52:ce:8e:fb:a6: + 6e:f5:4f:c3 +-----BEGIN CERTIFICATE----- +MIIE9zCCA9+gAwIBAgIJAILtv0HIgJGdMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx +OTA5MDZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh +a2Vob3N0bmFtZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMf/vqFk +BkfhxJUKZVlrkcSooKMIqtw/AEQKz0FL7eJvA3jH02cU0/60J4UaAleXtYYWh5eC +Degr7WkWw5amxEe4XcF2qQKdKFLJ2ZPxstoYfAp4vRw/Ek/lP+5bt7M8pbZVZcYz +CbBFpEn12PZUJS7bS8NlZIYWv3OOeOglzMG0fdGxVSCkrt64Qe5Dmkgz1BI5oKj7 +kU9HuzvWdTtipsVa1342ELw6DUbj3sHfUNCUdyIXoFGbr9w0YCYdoASspQYEV2Qa +0U8cGfi+S2gSD1/DEkBku8amhNlasHgdGxHBVOYV0a8r4lcrbbnEXbcBdFen+yeF +K5IV4uMTlynIknsCAwEAAaOCAcMwggG/MBcGA1UdEQQQMA6CDGZha2Vob3N0bmFt +ZTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC +MAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPh2ecsRhfBG5ZXmfmnLEl5OquxNMH0G +A1UdIwR2MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEEdzB1MDwG +CCsGAQUFBzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9w +eWNhY2VydC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVz +dC5uZXQvdGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0 +Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3 +DQEBBQUAA4IBAQBtUI377k6Ti+tHVro4zIDhncfhnh+cIgzSCJvtvzHZAO6vjFZ4 +ktF8uk6Bf4If9GiZhpHGy1fTuUES+nVT/SIyIVCva0yxNDbRqCUK0PD4gX1pWG6v +49LEMod5182tDFbzFScQDPlXWVMA7a9dTQeGeuXzl4i8hrTxF0YzVShme3DTpRK5 +T8ft5hMgLfCe7Bdkz/0TFBt2umSsxVG2zRMKk7H9QwmgC0Rsd0VDC+XtcLJ23AhK +W3Nfwfx/Y3D4uco8mAZf/ZjR5OZhXwmPbBiGmJzLP3N7Pzj1pwkg7qVjHP+LptGM +6PSEPZk4D8zgUgP5GAUjdjneUs6O+6Zu9U/D +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/keycertecc.pem b/src/greentest/3.7/keycertecc.pem new file mode 100644 index 0000000..deb484f --- /dev/null +++ b/src/greentest/3.7/keycertecc.pem @@ -0,0 +1,96 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDe3QWmhZX07HZbntz4 +CFqAOaoYMdYwD7Z3WPNIc2zR7p4D6BMOa7NAWjLV5A7CUw6hZANiAAQ5IVKzLLz4 +LCfcpy6fMOp+jk5KwywsU3upPtjA6E3UetxPcfnnv+gghRyDAYLN2OVqZgLMEmUo +F1j1SM1QrbhHIuNcVxI9gPPMdumcNFSz/hqxrBRtA/8Z2gywczdNLjc= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9e + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 19 19:09:06 2018 GMT + Not After : Nov 28 19:09:06 2027 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc + Subject Public Key Info: + Public Key Algorithm: id-ecPublicKey + Public-Key: (384 bit) + pub: + 04:39:21:52:b3:2c:bc:f8:2c:27:dc:a7:2e:9f:30: + ea:7e:8e:4e:4a:c3:2c:2c:53:7b:a9:3e:d8:c0:e8: + 4d:d4:7a:dc:4f:71:f9:e7:bf:e8:20:85:1c:83:01: + 82:cd:d8:e5:6a:66:02:cc:12:65:28:17:58:f5:48: + cd:50:ad:b8:47:22:e3:5c:57:12:3d:80:f3:cc:76: + e9:9c:34:54:b3:fe:1a:b1:ac:14:6d:03:ff:19:da: + 0c:b0:73:37:4d:2e:37 + ASN1 OID: secp384r1 + NIST CURVE: P-384 + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost-ecc + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 33:23:0E:15:04:83:2E:3D:BF:DA:81:6D:10:38:80:C3:C2:B0:A4:74 + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:82:ED:BF:41:C8:80:91:9B + + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + + X509v3 CRL Distribution Points: + + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + + Signature Algorithm: sha1WithRSAEncryption + 3b:6f:97:af:7e:5f:e0:14:34:ed:57:7e:de:ce:c4:85:1e:aa: + 84:52:94:7c:e5:ce:e9:9c:88:8b:ad:b5:4d:16:ac:af:81:ea: + b8:a2:e2:50:2e:cb:e9:11:bd:1b:a6:3f:0c:a2:d7:7b:67:72: + b3:43:16:ad:c6:87:ac:6e:ac:47:78:ef:2f:8c:86:e8:9b:d1: + 43:8c:c1:7a:91:30:e9:14:d6:9f:41:8b:9b:0b:24:9b:78:86: + 11:8a:fc:2b:cd:c9:13:ee:90:4f:14:33:51:a3:c4:9e:d6:06: + 48:f5:41:12:af:f0:f2:71:40:78:f5:96:c2:5d:cf:e1:38:ff: + bf:10:eb:74:2f:c2:23:21:3e:27:f5:f1:f2:af:2c:62:82:31: + 00:c8:96:1b:c3:7e:8d:71:89:e7:40:b5:67:1a:33:fb:c0:8b: + 96:0c:36:78:25:27:82:d8:27:27:52:0f:f7:69:cd:ff:2b:92: + 10:d3:d2:0a:db:65:ed:af:90:eb:db:76:f3:8a:7a:13:9e:c6: + 33:57:15:42:06:13:d6:54:49:fa:84:a7:0e:1d:14:72:ca:19: + 8e:2b:aa:a4:02:54:3c:f6:1c:23:81:7a:59:54:b0:92:65:72: + c8:e5:ba:9f:03:4e:30:f2:4d:45:85:e3:35:a8:b1:68:58:b9: + 3b:20:a3:eb +-----BEGIN CERTIFICATE----- +MIIESzCCAzOgAwIBAgIJAILtv0HIgJGeMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx +OTA5MDZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv +Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQ5IVKzLLz4LCfcpy6f +MOp+jk5KwywsU3upPtjA6E3UetxPcfnnv+gghRyDAYLN2OVqZgLMEmUoF1j1SM1Q +rbhHIuNcVxI9gPPMdumcNFSz/hqxrBRtA/8Z2gywczdNLjejggHEMIIBwDAYBgNV +HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU +BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUMyMO +FQSDLj2/2oFtEDiAw8KwpHQwfQYDVR0jBHYwdIAUms/PbutxPds88a6Ia1ZyA8sI +p0ihUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAILtv0HIgJGb +MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 +aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 +cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww +OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 +b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQEFBQADggEBADtvl69+X+AUNO1Xft7OxIUe +qoRSlHzlzumciIuttU0WrK+B6rii4lAuy+kRvRumPwyi13tncrNDFq3Gh6xurEd4 +7y+Mhuib0UOMwXqRMOkU1p9Bi5sLJJt4hhGK/CvNyRPukE8UM1GjxJ7WBkj1QRKv +8PJxQHj1lsJdz+E4/78Q63QvwiMhPif18fKvLGKCMQDIlhvDfo1xiedAtWcaM/vA +i5YMNnglJ4LYJydSD/dpzf8rkhDT0grbZe2vkOvbdvOKehOexjNXFUIGE9ZUSfqE +pw4dFHLKGY4rqqQCVDz2HCOBellUsJJlcsjlup8DTjDyTUWF4zWosWhYuTsgo+s= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/nokia.pem b/src/greentest/3.7/nokia.pem new file mode 100644 index 0000000..0d044df --- /dev/null +++ b/src/greentest/3.7/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/nullbytecert.pem b/src/greentest/3.7/nullbytecert.pem new file mode 100644 index 0000000..447186c --- /dev/null +++ b/src/greentest/3.7/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/nullcert.pem b/src/greentest/3.7/nullcert.pem new file mode 100644 index 0000000..e69de29 diff --git a/src/greentest/3.7/pycacert.pem b/src/greentest/3.7/pycacert.pem new file mode 100644 index 0000000..850fa32 --- /dev/null +++ b/src/greentest/3.7/pycacert.pem @@ -0,0 +1,79 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9b + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 19 19:09:06 2018 GMT + Not After : Jan 17 19:09:06 2028 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:c3:18:69:6b:c9:47:29:98:8e:b1:56:c2:2e:fa: + 0e:5e:bc:23:80:b3:07:62:24:d2:42:5b:f1:4a:bf: + a9:c8:21:75:c8:e3:e6:2c:1f:87:3c:6e:7c:1b:ed: + 39:32:95:b7:40:b2:60:48:c3:9a:16:08:fe:6d:67: + 88:34:3b:77:77:70:1c:70:5a:d1:1f:5f:04:21:54: + b9:0c:e3:41:85:1d:58:ee:2f:ed:f3:0e:ef:d8:23: + a1:fa:73:fb:4c:28:e0:e5:e6:4d:0b:02:52:49:86: + c7:be:7e:bd:e6:56:76:8b:70:8e:0a:8f:06:33:20: + 1d:7b:5b:aa:d0:c5:1b:ab:9b:cc:54:09:3c:bf:e4: + 40:66:f1:fb:d6:f7:16:9d:c4:19:d4:c3:f2:ff:07: + bc:6f:5a:9e:25:1b:02:4a:a5:ec:42:96:3a:70:d2: + 6c:99:2b:ce:be:e8:d2:01:ef:d5:ba:b0:cf:94:3e: + 82:d0:01:d6:4b:71:80:03:0a:12:45:86:79:81:d8: + 4b:d2:e8:b5:b7:2c:6c:9a:4c:8a:10:10:e4:e4:f5: + df:ce:84:91:ca:d1:46:e0:84:73:17:66:db:69:43: + 78:80:83:be:14:4d:f1:3e:1a:d6:6c:f5:de:45:f3: + 39:af:91:d5:3d:54:44:bf:41:cc:73:68:1a:fc:24: + db:91 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 10:25:c8:dc:0c:55:5c:cb:83:6e:79:ef:77:ec:0d:8e:0c:06: + c1:4b:0c:d6:f7:75:52:21:b8:17:4a:38:88:9d:b3:78:c4:42: + fb:b8:7c:14:38:10:fb:ac:da:11:00:5b:42:87:5e:45:9f:6d: + 4e:42:a4:9a:18:06:39:0f:45:a6:96:89:32:d6:59:b3:d3:8e: + e3:95:b6:c4:a2:4b:74:2f:67:c1:fb:bb:f9:72:6f:37:4a:e7: + f4:48:33:71:df:b8:f5:e6:41:3f:d5:d5:2f:26:09:f8:0e:92: + ff:70:ea:f6:ab:58:fb:90:04:d6:43:2e:8f:b1:fb:06:ab:69: + d0:dc:a8:f8:5b:07:f2:d4:66:1f:63:f8:5d:c1:9e:41:44:bb: + c9:e8:7d:e0:46:e4:a7:c8:32:5f:31:62:e5:1c:5c:89:dd:b7: + a2:4f:9e:0d:13:b8:5f:b1:84:53:4c:1f:ce:19:e1:01:00:5e: + bf:41:55:94:a9:a5:13:db:f2:59:f3:d6:4e:b9:9d:9d:b9:0a: + d9:b2:18:6d:7c:b1:f7:96:aa:bd:f6:f9:95:0f:4a:6e:3c:7c: + 46:5b:df:d4:78:ec:9a:dc:e2:e3:01:e6:88:77:39:93:9c:ba: + 2a:63:f9:25:4b:4f:ac:08:79:39:c6:7b:df:07:35:ba:c0:c2: + 50:bf:5a:81 +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJAILtv0HIgJGbMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yODAxMTcx +OTA5MDZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAMMYaWvJRymYjrFWwi76Dl68I4CzB2Ik0kJb +8Uq/qcghdcjj5iwfhzxufBvtOTKVt0CyYEjDmhYI/m1niDQ7d3dwHHBa0R9fBCFU +uQzjQYUdWO4v7fMO79gjofpz+0wo4OXmTQsCUkmGx75+veZWdotwjgqPBjMgHXtb +qtDFG6ubzFQJPL/kQGbx+9b3Fp3EGdTD8v8HvG9aniUbAkql7EKWOnDSbJkrzr7o +0gHv1bqwz5Q+gtAB1ktxgAMKEkWGeYHYS9LotbcsbJpMihAQ5OT1386EkcrRRuCE +cxdm22lDeICDvhRN8T4a1mz13kXzOa+R1T1URL9BzHNoGvwk25ECAwEAAaNQME4w +HQYDVR0OBBYEFJrPz27rcT3bPPGuiGtWcgPLCKdIMB8GA1UdIwQYMBaAFJrPz27r +cT3bPPGuiGtWcgPLCKdIMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +ABAlyNwMVVzLg25573fsDY4MBsFLDNb3dVIhuBdKOIids3jEQvu4fBQ4EPus2hEA +W0KHXkWfbU5CpJoYBjkPRaaWiTLWWbPTjuOVtsSiS3QvZ8H7u/lybzdK5/RIM3Hf +uPXmQT/V1S8mCfgOkv9w6varWPuQBNZDLo+x+waradDcqPhbB/LUZh9j+F3BnkFE +u8nofeBG5KfIMl8xYuUcXIndt6JPng0TuF+xhFNMH84Z4QEAXr9BVZSppRPb8lnz +1k65nZ25CtmyGG18sfeWqr32+ZUPSm48fEZb39R47Jrc4uMB5oh3OZOcuipj+SVL +T6wIeTnGe98HNbrAwlC/WoE= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/pycakey.pem b/src/greentest/3.7/pycakey.pem new file mode 100644 index 0000000..16b7587 --- /dev/null +++ b/src/greentest/3.7/pycakey.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDDGGlryUcpmI6x +VsIu+g5evCOAswdiJNJCW/FKv6nIIXXI4+YsH4c8bnwb7TkylbdAsmBIw5oWCP5t +Z4g0O3d3cBxwWtEfXwQhVLkM40GFHVjuL+3zDu/YI6H6c/tMKODl5k0LAlJJhse+ +fr3mVnaLcI4KjwYzIB17W6rQxRurm8xUCTy/5EBm8fvW9xadxBnUw/L/B7xvWp4l +GwJKpexCljpw0myZK86+6NIB79W6sM+UPoLQAdZLcYADChJFhnmB2EvS6LW3LGya +TIoQEOTk9d/OhJHK0UbghHMXZttpQ3iAg74UTfE+GtZs9d5F8zmvkdU9VES/Qcxz +aBr8JNuRAgMBAAECggEAZHgv4hg3k45C/cSmH7caq2LMDb0kskAwH4hlzI7DipLg +q2Hh6Rsbc92aAG+8IvbC9ohl2VMSCQL8s667j9qH/XQ40QuT4kn2QIv2+FIYLcsd +Pxxjt+YbUf2XrvkHkwMCPqLJTkAVzFOijdGLThF83vZJz9oz4SRKyno8j2LSix68 +WEfnjdyWqYb0eS0luKrLHw+IL7bD5vfc/P0q6u31zJ9h8zEyN5EBCj5OxM/hD0VO +nObrp6r9Bs+xx+yRx+8J5Db6LPXggl5nBqsqrDKVDe6uTysYVgstqkfaDv1L78Vu +3BNdKPAdJ+ucPJrQufzFHBDIIN+Xwckf/09gdQagGQKBgQDnvFaOjZfqc6wL/kNK +tszQtedbdwP20L+EWdNEVsVWK1TOw36Pmkrp2AYLXMd7W1QQu0KukM89EFb84wKo +s4C9V/ch162mUhEAveaLioi7bMwMPIib2V6pHmYGG8nQVRvgkZVYx6ZtPEvWye1v +wmCzzxxK0gC6PQGxp8MSv9yXDwKBgQDXhe57ufc52pgJ+Agyl4PLkllIbG2DKQHG +LwY06v73jllirTpWBOBvN0NvEsI2Pj4aK/BXRNYN1PS7xi/3C6MVWxnOpBtbq3H5 +DwFb5mpfgJmhV6DZ6jMw7h3Yvy35ViKoiI9UK3eTmhkerH3DsILEje7jE9dGmIOJ +4oLa50JjXwKBgQDdTfyveMNasIrejTzATmC89Or0a22KuQIdKBddjSw5xXnhV8s2 +4temCJqFIV6UDLz0mZDt2vc+zqr0KOtyJrLMoAQv+qQoUPlR5wkTvAImU5luGiUw +CN+gzJoMPV93KMBNr1qcBVaHvWyDvCWXdF8beLABOBpfwUEr4xWlgzrruwKBgCvf +cr2zDJW1Xu/gkuKhn02ofA5XLC/gACF03yGUmNSSILYKp25tTba2HD8XJXvfTcsM +GL/bHmvwZuV2obr7nnYxdl5vX7ZYfzoBCPjJPew1BJEognD50PPr9R1zRYuVMjb2 +nZ63vn7IhsaMvIlCfExAzFljZ5ZSY6yE9LhVDVmnAoGBALOwMwpkm1drx5UNSJO7 +70Q8kYzg0oQhCo/7b6DWbAglDPSWQS5IA4rHYOwL3sE+69G2Exe+1454rVDisojW +XdSyA3svI/YQeom8R2LIM/ayCPxCc3/Dxy9+aQQT4lW3F0XQIxod/QsQJxpZIOnF +jOSPclypgV2X6dDOwDkd2Tgh +-----END PRIVATE KEY----- diff --git a/src/greentest/3.7/revocation.crl b/src/greentest/3.7/revocation.crl new file mode 100644 index 0000000..53cb4b3 --- /dev/null +++ b/src/greentest/3.7/revocation.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTE4MDExOTE5MDkwNloXDTI3MTEyODE5MDkwNlqgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQBYVzH8n2LdyJJ/t8CpWjz652hZJ0sY +DeNYcwuTPzR9CbSwEwNbf0kY9bgWXGfoRD2SnnCnuNNJXO2MuXtxf6qYx2ZjhJm8 +qgxXs0Bz4agRswNMbumjHCmqIv1t88vbrO0+ItEZDK7RJVIMBtVJ0XYOHvD/IG/i +zqa1Fl3uCTvQbTJ2TrqzJeP/Vl40hOD+VdBBZK3j0r4AkCKU3tAiHYTGmHKhPxy1 +f8Yet+4SRMGp1BdDezTI1bICpSZhRJ4geW0UzuCZnXPW8IZzioUmdUBAmAMHPWFr +B0sTTc/ntD4jHG1/T5b0oiDMbXIbh5Iz9iQNcY0IbotkCw39h+K90wY6 +-----END X509 CRL----- diff --git a/src/greentest/3.7/selfsigned_pythontestdotnet.pem b/src/greentest/3.7/selfsigned_pythontestdotnet.pem new file mode 100644 index 0000000..b6d259b --- /dev/null +++ b/src/greentest/3.7/selfsigned_pythontestdotnet.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/ssl_cert.pem b/src/greentest/3.7/ssl_cert.pem new file mode 100644 index 0000000..b1dd3f3 --- /dev/null +++ b/src/greentest/3.7/ssl_cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx +MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 +RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj +9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj +SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN +sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn +96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ +bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK +Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk +ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 +qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu +4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS +dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF +-----END CERTIFICATE----- diff --git a/src/greentest/3.7/ssl_key.passwd.pem b/src/greentest/3.7/ssl_key.passwd.pem new file mode 100644 index 0000000..669c7ce --- /dev/null +++ b/src/greentest/3.7/ssl_key.passwd.pem @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,2D5DD30B9D440DBB + +01gIwpy3XxPGsY0PoK59vAxdLhkVj3odO0Z1ULamUzIte6ThKL1HqnZiUlXpYKfK +XqHVVeQ1xouxiDRNFLJ4CqBG4HbRtqTkl+sfaNTVveL18lOOMAZy6W3dCGAnWOTZ +Z0RJyZlQxxjNQLuko4tIvFkrShXIgdiVFjwAhRU0KTUb7UQ2xfFA9R0Kfde30pzz +zSjb/OmYqAIhkdvafGXvJxzZAorQkU9akDh+uJ6cht5B/RGZsbKACYDSv2WSV5yW +r+fKVYcTup33r0Jj8hAD6fVY15K8BJknpkF9HfSlZnmmr2WDaffLokOOnCV/I1ie +WD7ENA7K//48km5D3Ogh2b2/0Iwuzjq8Mvd8aR39N9nINbGR+HNT85pawoo1S0W9 +pQTU4XTmxfXjtR2287C6XZyQ/tBwvNDMFPVhlxsGOdLYwoV5e/L1t1qIfkTlbuvd +JaMzOhSSLjiC156IFoH7PTPe+g75hw2b32XJURFGlaYknHF7P4BmCiwXOQYo5CCo +MQGGlw5qBCqODrIsc03wpL2jUzgvyPqLyaw395ITuSoGX+WO7vUQaGW0Tz/sOoTs +3pK+bTi2QmqZMe7xBOj07CYMMOo4QPrM6NpbObt+Jja2UXaxvKa9BwqCEQzA4pQZ +8ZHHfEWIaDffKTGkAlqm+S8qCtsrEZJhIn3aI/ikzK8v+YkWw6w+8t/tR1V8ET/s +CoYGIR7I8WhdfKAwgx2QT5bt1jkYKJyKPm4Iacp2mNh9gNFVq+JSKF318e7BrR3+ +wyqMkDxRYnov3ybtf6kiICxPREDqa6UG1xRq3SbWz6NnIF/1hoHs79YlSYbMfXNU +ffIxBaXNCcH6jM9duP2YRnO29jLwfLM/mmokTBBjyOBaKZia9GPa4naXoATXW3z+ +Xx4EKIUkKdb53kiV6NtEKMPialAnkeoHTEjyLPgaV8mCHLvGQbnxbYwvpPJH0e2f +CWgiw6ci4ROOzcZ7HJHIDUprwK0xRKn43hoI44fivlSHOFX6da6o3wIqhEUqMKwL +JQDS1GORRk1ndRXP+7Ub1dO+Vo/DqO1VcTr2o5RwZ1LWPnzLqbCG50mvTLH4djB+ ++hf6vlmnFC30N3yUFXWE5vS10nJHYP88dD9CB2RsaWzpxD9Zxl+PKcRsppen6HyO +u3b71a/TBOkJcI+lkOatEFvbuqzBAqhMceMctO+Dl55RFsbxfIw/IXZjdP0PYZ0C +t20DrIdBsvl9F/mfYpmkV4DF7yci78DqnRBcxylVNF2vwX7o+2fq/TsEwsHn3KnT +kvcF5Cq8Vr5C8ugWX8JfveNym0BjLu6Lr58qS4a6qCNGEGPFKyB+xkm4KEScbarQ +aLbEbfulMM7q9//sEOOLexIx7mNoLd29Xzn5hsLCAZLWX6wMq6JVJ/zbBOAHDbBT +yhi03yd5Kvw3swSt4QZj+uR3qTFwxkXUFiVvrSfxRZoyKsxsLr9Z7D8aoH9Rkb2L +6KjZ31nt9Drh7NJfh6ReANBW6INdDW0Y2mbzoDozLszAYjVfuUUEE76iJqXY0N4W +kNr0OQQTUtDpVk0AZZZvy17xV+rkqGgwlOqTvHbwFYEQvgwVz4EKUw== +-----END RSA PRIVATE KEY----- diff --git a/src/greentest/3.7/ssl_key.pem b/src/greentest/3.7/ssl_key.pem new file mode 100644 index 0000000..b63f38b --- /dev/null +++ b/src/greentest/3.7/ssl_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCr77F9oBLYuLIb +3t4mDHPoxOEnu1h7NQaJdn9tR/KjW6AzhXXm9USh45qt3sR6Wo8sDlBIJ3vZyhyW +P939qYUeGOZJahupYi4IkqVckZXARm3k9qhAI/aC4ixVFyxnxg3bNpk8Ir0AyyQV +oudY+33+ZNj6+fHzeboGDJ9uE/RTcp9JqE/qo0haATZufJfY63ZCUpYFn6j5W4jG +vpqg5/0hba8Cxdk62387uNknfVHSMzGkkq82zbBpud6TYQofCp3VlEPBjz9iLCz6 +FFOKgLZmbk8QsdktXF6zfRJJk+vbZTh/OGH0p/eiIfW1kXOzOcuW31XRFTPnykJd +4QUX9OajAgMBAAECggEAHppmXDbuw9Z0FVPg9KLIysioTtsgz6VLiZIm8juZK4x2 +glUh/D7xvWL2uDXrgN+3lh7iGUW13LkFx5SMncbbo9TIwI57Z/XKvcnkVwquve+L +RfLFVc1Q5lD9lROv2rS86KTaN4LzYz3FKXi6dvMkpPAsUtfEQhMLkmISypQQq/1z +EJaqo7r85OjN7e0wKazlKZpOzJEa5FQLMVRjTRFhLFNbHXX/tAet2jw+umATKbw8 +hYgiuZ44TwSEd9JeIV/oSYWfI/3HetuYW0ru3caiztRF2NySNu8lcsWgNC7fIku9 +mcHjtSNzs91QN1Qlu7GQvvhpt6OWDirNDCW+49WGaQKBgQDg9SDhfF0jRYslgYbH +cqO4ggaFdHjrAAYpwnAgvanhFZL/zEqm5G1E7l/e2fCkJ9VOSFO0A208chvwMcr+ +dCjHE2tVdE81aQ2v/Eo83VdS1RcOV4Y75yPH48rMhxPaHvxWD/FFDbf0/P2mtPB7 +SZ3kIeZMkE1wxdaO3AKUbQoozwKBgQDDqYgg7kVtygyICE1mB8Hwp6nUxFTczG7y +4XcsDqMIrKmw+PbQluvkoHoStxeVrsTloDhkTjIrpmYLyAiazg+PUJdkd6xrfLSj +VV6X93W0S/1egEb1F1CGFxtk8v/PWH4K76EPL2vxXdxjywz3GWlrL9yDYaB2szzS +DqgwVMqx7QKBgDCD7UF0Bsoyl13RX3XoPXLvZ+SkR+e2q52Z94C4JskKVBeiwX7Y +yNAS8M4pBoMArDoj0xmBm69rlKbqtjLGbnzwrTdSzDpim7cWnBQgUFLm7gAD1Elb +AhZ8BCK0Bw4FnLoa2hfga4oEfdfUMgEE0W5/+SEOBgWKRUmuHUhRc911AoGAY2EN +YmSDYSM5wDIvVb5k9B3EtevOiqNPSw/XnsoEZtiEC/44JnQxdltIBY93bDBrk5IQ +cmoBM4h91kgQjshQwOMXMhFSwvmBKmCm/hrTbvMVytTutXfVD3ZXFKwT4DW7N0TF +ElhsxBh/YzRz7mG62JVjtFt2zDN3ld2Z8YpvtXUCgYEA4EJ4ObS5YyvcXAKHJFo6 +Fxmavyrf8LSm3MFA65uSnFvWukMVqqRMReQc5jvpxHKCis+XvnHzyOfL0gW9ZTi7 +tWGGbBi0TRJCa8BkvgngUZxOxUlMfg/7cVxOIB0TPoUSgxFd/+qVz4GZMvr0dPu7 +eAF7J/8ECVvb0wSPTUI1N3c= +-----END PRIVATE KEY----- diff --git a/src/greentest/3.7/test_asyncore.py b/src/greentest/3.7/test_asyncore.py new file mode 100644 index 0000000..694ddff --- /dev/null +++ b/src/greentest/3.7/test_asyncore.py @@ -0,0 +1,834 @@ +import asyncore +import unittest +import select +import os +import socket +import sys +import time +import errno +import struct +import threading + +from test import support +from io import BytesIO + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + + +TIMEOUT = 3 +HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') + +class dummysocket: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def fileno(self): + return 42 + +class dummychannel: + def __init__(self): + self.socket = dummysocket() + + def close(self): + self.socket.close() + +class exitingdummy: + def __init__(self): + pass + + def handle_read_event(self): + raise asyncore.ExitNow() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + +class crashingdummy: + def __init__(self): + self.error_handled = False + + def handle_read_event(self): + raise Exception() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + + def handle_error(self): + self.error_handled = True + +# used when testing senders; just collects what it gets until newline is sent +def capture_server(evt, buf, serv): + try: + serv.listen() + conn, addr = serv.accept() + except socket.timeout: + pass + else: + n = 200 + start = time.time() + while n > 0 and time.time() - start < 3.0: + r, w, e = select.select([conn], [], [], 0.1) + if r: + n -= 1 + data = conn.recv(10) + # keep everything except for the newline terminator + buf.write(data.replace(b'\n', b'')) + if b'\n' in data: + break + time.sleep(0.01) + + conn.close() + finally: + serv.close() + evt.set() + +def bind_af_aware(sock, addr): + """Helper function to bind a socket according to its family.""" + if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: + # Make sure the path doesn't exist. + support.unlink(addr) + support.bind_unix_socket(sock, addr) + else: + sock.bind(addr) + + +class HelperFunctionTests(unittest.TestCase): + def test_readwriteexc(self): + # Check exception handling behavior of read, write and _exception + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore read/write/_exception calls + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.read, tr1) + self.assertRaises(asyncore.ExitNow, asyncore.write, tr1) + self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + asyncore.read(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore.write(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore._exception(tr2) + self.assertEqual(tr2.error_handled, True) + + # asyncore.readwrite uses constants in the select module that + # are not present in Windows systems (see this thread: + # http://mail.python.org/pipermail/python-list/2001-October/109973.html) + # These constants should be present as long as poll is available + + @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') + def test_readwrite(self): + # Check that correct methods are called by readwrite() + + attributes = ('read', 'expt', 'write', 'closed', 'error_handled') + + expected = ( + (select.POLLIN, 'read'), + (select.POLLPRI, 'expt'), + (select.POLLOUT, 'write'), + (select.POLLERR, 'closed'), + (select.POLLHUP, 'closed'), + (select.POLLNVAL, 'closed'), + ) + + class testobj: + def __init__(self): + self.read = False + self.write = False + self.closed = False + self.expt = False + self.error_handled = False + + def handle_read_event(self): + self.read = True + + def handle_write_event(self): + self.write = True + + def handle_close(self): + self.closed = True + + def handle_expt_event(self): + self.expt = True + + def handle_error(self): + self.error_handled = True + + for flag, expectedattr in expected: + tobj = testobj() + self.assertEqual(getattr(tobj, expectedattr), False) + asyncore.readwrite(tobj, flag) + + # Only the attribute modified by the routine we expect to be + # called should be True. + for attr in attributes: + self.assertEqual(getattr(tobj, attr), attr==expectedattr) + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore readwrite call + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + self.assertEqual(tr2.error_handled, False) + asyncore.readwrite(tr2, flag) + self.assertEqual(tr2.error_handled, True) + + def test_closeall(self): + self.closeall_check(False) + + def test_closeall_default(self): + self.closeall_check(True) + + def closeall_check(self, usedefault): + # Check that close_all() closes everything in a given map + + l = [] + testmap = {} + for i in range(10): + c = dummychannel() + l.append(c) + self.assertEqual(c.socket.closed, False) + testmap[i] = c + + if usedefault: + socketmap = asyncore.socket_map + try: + asyncore.socket_map = testmap + asyncore.close_all() + finally: + testmap, asyncore.socket_map = asyncore.socket_map, socketmap + else: + asyncore.close_all(testmap) + + self.assertEqual(len(testmap), 0) + + for c in l: + self.assertEqual(c.socket.closed, True) + + def test_compact_traceback(self): + try: + raise Exception("I don't like spam!") + except: + real_t, real_v, real_tb = sys.exc_info() + r = asyncore.compact_traceback() + else: + self.fail("Expected exception") + + (f, function, line), t, v, info = r + self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py') + self.assertEqual(function, 'test_compact_traceback') + self.assertEqual(t, real_t) + self.assertEqual(v, real_v) + self.assertEqual(info, '[%s|%s|%s]' % (f, function, line)) + + +class DispatcherTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + def test_basic(self): + d = asyncore.dispatcher() + self.assertEqual(d.readable(), True) + self.assertEqual(d.writable(), True) + + def test_repr(self): + d = asyncore.dispatcher() + self.assertEqual(repr(d), '' % id(d)) + + def test_log(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log() (to stderr) + l1 = "Lovely spam! Wonderful spam!" + l2 = "I don't like spam!" + with support.captured_stderr() as stderr: + d.log(l1) + d.log(l2) + + lines = stderr.getvalue().splitlines() + self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2]) + + def test_log_info(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log_info() (to stdout via print) + l1 = "Have you got anything without spam?" + l2 = "Why can't she have egg bacon spam and sausage?" + l3 = "THAT'S got spam in it!" + with support.captured_stdout() as stdout: + d.log_info(l1, 'EGGS') + d.log_info(l2) + d.log_info(l3, 'SPAM') + + lines = stdout.getvalue().splitlines() + expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3] + self.assertEqual(lines, expected) + + def test_unhandled(self): + d = asyncore.dispatcher() + d.ignore_log_types = () + + # capture output of dispatcher.log_info() (to stdout via print) + with support.captured_stdout() as stdout: + d.handle_expt() + d.handle_read() + d.handle_write() + d.handle_connect() + + lines = stdout.getvalue().splitlines() + expected = ['warning: unhandled incoming priority event', + 'warning: unhandled read event', + 'warning: unhandled write event', + 'warning: unhandled connect event'] + self.assertEqual(lines, expected) + + def test_strerror(self): + # refers to bug #8573 + err = asyncore._strerror(errno.EPERM) + if hasattr(os, 'strerror'): + self.assertEqual(err, os.strerror(errno.EPERM)) + err = asyncore._strerror(-1) + self.assertTrue(err != "") + + +class dispatcherwithsend_noread(asyncore.dispatcher_with_send): + def readable(self): + return False + + def handle_connect(self): + pass + + +class DispatcherWithSendTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + @support.reap_threads + def test_send(self): + evt = threading.Event() + sock = socket.socket() + sock.settimeout(3) + port = support.bind_port(sock) + + cap = BytesIO() + args = (evt, cap, sock) + t = threading.Thread(target=capture_server, args=args) + t.start() + try: + # wait a little longer for the server to initialize (it sometimes + # refuses connections on slow machines without this wait) + time.sleep(0.2) + + data = b"Suppose there isn't a 16-ton weight?" + d = dispatcherwithsend_noread() + d.create_socket() + d.connect((support.HOST, port)) + + # give time for socket to connect + time.sleep(0.1) + + d.send(data) + d.send(data) + d.send(b'\n') + + n = 1000 + while d.out_buffer and n > 0: + asyncore.poll() + n -= 1 + + evt.wait() + + self.assertEqual(cap.getvalue(), data*2) + finally: + support.join_thread(t, timeout=TIMEOUT) + + +@unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), + 'asyncore.file_wrapper required') +class FileWrapperTest(unittest.TestCase): + def setUp(self): + self.d = b"It's not dead, it's sleeping!" + with open(support.TESTFN, 'wb') as file: + file.write(self.d) + + def tearDown(self): + support.unlink(support.TESTFN) + + def test_recv(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + w = asyncore.file_wrapper(fd) + os.close(fd) + + self.assertNotEqual(w.fd, fd) + self.assertNotEqual(w.fileno(), fd) + self.assertEqual(w.recv(13), b"It's not dead") + self.assertEqual(w.read(6), b", it's") + w.close() + self.assertRaises(OSError, w.read, 1) + + def test_send(self): + d1 = b"Come again?" + d2 = b"I want to buy some cheese." + fd = os.open(support.TESTFN, os.O_WRONLY | os.O_APPEND) + w = asyncore.file_wrapper(fd) + os.close(fd) + + w.write(d1) + w.send(d2) + w.close() + with open(support.TESTFN, 'rb') as file: + self.assertEqual(file.read(), self.d + d1 + d2) + + @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'), + 'asyncore.file_dispatcher required') + def test_dispatcher(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + data = [] + class FileDispatcher(asyncore.file_dispatcher): + def handle_read(self): + data.append(self.recv(29)) + s = FileDispatcher(fd) + os.close(fd) + asyncore.loop(timeout=0.01, use_poll=True, count=2) + self.assertEqual(b"".join(data), self.d) + + def test_resource_warning(self): + # Issue #11453 + fd = os.open(support.TESTFN, os.O_RDONLY) + f = asyncore.file_wrapper(fd) + + os.close(fd) + with support.check_warnings(('', ResourceWarning)): + f = None + support.gc_collect() + + def test_close_twice(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + f = asyncore.file_wrapper(fd) + os.close(fd) + + os.close(f.fd) # file_wrapper dupped fd + with self.assertRaises(OSError): + f.close() + + self.assertEqual(f.fd, -1) + # calling close twice should not fail + f.close() + + +class BaseTestHandler(asyncore.dispatcher): + + def __init__(self, sock=None): + asyncore.dispatcher.__init__(self, sock) + self.flag = False + + def handle_accept(self): + raise Exception("handle_accept not supposed to be called") + + def handle_accepted(self): + raise Exception("handle_accepted not supposed to be called") + + def handle_connect(self): + raise Exception("handle_connect not supposed to be called") + + def handle_expt(self): + raise Exception("handle_expt not supposed to be called") + + def handle_close(self): + raise Exception("handle_close not supposed to be called") + + def handle_error(self): + raise + + +class BaseServer(asyncore.dispatcher): + """A server which listens on an address and dispatches the + connection to a handler. + """ + + def __init__(self, family, addr, handler=BaseTestHandler): + asyncore.dispatcher.__init__(self) + self.create_socket(family) + self.set_reuse_addr() + bind_af_aware(self.socket, addr) + self.listen(5) + self.handler = handler + + @property + def address(self): + return self.socket.getsockname() + + def handle_accepted(self, sock, addr): + self.handler(sock) + + def handle_error(self): + raise + + +class BaseClient(BaseTestHandler): + + def __init__(self, family, address): + BaseTestHandler.__init__(self) + self.create_socket(family) + self.connect(address) + + def handle_connect(self): + pass + + +class BaseTestAPI: + + def tearDown(self): + asyncore.close_all(ignore_all=True) + + def loop_waiting_for_flag(self, instance, timeout=5): + timeout = float(timeout) / 100 + count = 100 + while asyncore.socket_map and count > 0: + asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) + if instance.flag: + return + count -= 1 + time.sleep(timeout) + self.fail("flag not set") + + def test_handle_connect(self): + # make sure handle_connect is called on connect() + + class TestClient(BaseClient): + def handle_connect(self): + self.flag = True + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_accept(self): + # make sure handle_accept() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self, family, addr): + BaseTestHandler.__init__(self) + self.create_socket(family) + bind_af_aware(self.socket, addr) + self.listen(5) + self.address = self.socket.getsockname() + + def handle_accept(self): + self.flag = True + + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) + self.loop_waiting_for_flag(server) + + def test_handle_accepted(self): + # make sure handle_accepted() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self, family, addr): + BaseTestHandler.__init__(self) + self.create_socket(family) + bind_af_aware(self.socket, addr) + self.listen(5) + self.address = self.socket.getsockname() + + def handle_accept(self): + asyncore.dispatcher.handle_accept(self) + + def handle_accepted(self, sock, addr): + sock.close() + self.flag = True + + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) + self.loop_waiting_for_flag(server) + + + def test_handle_read(self): + # make sure handle_read is called on data received + + class TestClient(BaseClient): + def handle_read(self): + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.send(b'x' * 1024) + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_write(self): + # make sure handle_write is called + + class TestClient(BaseClient): + def handle_write(self): + self.flag = True + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close(self): + # make sure handle_close is called when the other end closes + # the connection + + class TestClient(BaseClient): + + def handle_read(self): + # in order to make handle_close be called we are supposed + # to make at least one recv() call + self.recv(1024) + + def handle_close(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.close() + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close_after_conn_broken(self): + # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and + # #11265). + + data = b'\0' * 128 + + class TestClient(BaseClient): + + def handle_write(self): + self.send(data) + + def handle_close(self): + self.flag = True + self.close() + + def handle_expt(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + + def handle_read(self): + self.recv(len(data)) + self.close() + + def writable(self): + return False + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + @unittest.skipIf(sys.platform.startswith("sunos"), + "OOB support is broken on Solaris") + def test_handle_expt(self): + # Make sure handle_expt is called on OOB data received. + # Note: this might fail on some platforms as OOB data is + # tenuously supported and rarely used. + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + + if sys.platform == "darwin" and self.use_poll: + self.skipTest("poll may fail on macOS; see issue #28087") + + class TestClient(BaseClient): + def handle_expt(self): + self.socket.recv(1024, socket.MSG_OOB) + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB) + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_error(self): + + class TestClient(BaseClient): + def handle_write(self): + 1.0 / 0 + def handle_error(self): + self.flag = True + try: + raise + except ZeroDivisionError: + pass + else: + raise Exception("exception not raised") + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_connection_attributes(self): + server = BaseServer(self.family, self.addr) + client = BaseClient(self.family, server.address) + + # we start disconnected + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + # this can't be taken for granted across all platforms + #self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # execute some loops so that client connects to server + asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100) + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertTrue(client.connected) + self.assertFalse(client.accepting) + + # disconnect the client + client.close() + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # stop serving + server.close() + self.assertFalse(server.connected) + self.assertFalse(server.accepting) + + def test_create_socket(self): + s = asyncore.dispatcher() + s.create_socket(self.family) + self.assertEqual(s.socket.type, socket.SOCK_STREAM) + self.assertEqual(s.socket.family, self.family) + self.assertEqual(s.socket.gettimeout(), 0) + self.assertFalse(s.socket.get_inheritable()) + + def test_bind(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + s1 = asyncore.dispatcher() + s1.create_socket(self.family) + s1.bind(self.addr) + s1.listen(5) + port = s1.socket.getsockname()[1] + + s2 = asyncore.dispatcher() + s2.create_socket(self.family) + # EADDRINUSE indicates the socket was correctly bound + self.assertRaises(OSError, s2.bind, (self.addr[0], port)) + + def test_set_reuse_addr(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + + with socket.socket(self.family) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except OSError: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket(self.family)) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.socket.close() + s.create_socket(self.family) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + + @support.reap_threads + def test_quick_connect(self): + # see: http://bugs.python.org/issue10340 + if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): + self.skipTest("test specific to AF_INET and AF_INET6") + + server = BaseServer(self.family, self.addr) + # run the thread 500 ms: the socket should be connected in 200 ms + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, + count=5)) + t.start() + try: + with socket.socket(self.family, socket.SOCK_STREAM) as s: + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + + try: + s.connect(server.address) + except OSError: + pass + finally: + support.join_thread(t, timeout=TIMEOUT) + +class TestAPI_UseIPv4Sockets(BaseTestAPI): + family = socket.AF_INET + addr = (support.HOST, 0) + +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 support required') +class TestAPI_UseIPv6Sockets(BaseTestAPI): + family = socket.AF_INET6 + addr = (support.HOSTv6, 0) + +@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required') +class TestAPI_UseUnixSockets(BaseTestAPI): + if HAS_UNIX_SOCKETS: + family = socket.AF_UNIX + addr = support.TESTFN + + def tearDown(self): + support.unlink(self.addr) + BaseTestAPI.tearDown(self) + +class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets, unittest.TestCase): + use_poll = True + +class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets, unittest.TestCase): + use_poll = True + +class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets, unittest.TestCase): + use_poll = True + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.7/test_ftplib.py b/src/greentest/3.7/test_ftplib.py new file mode 100644 index 0000000..f9488a9 --- /dev/null +++ b/src/greentest/3.7/test_ftplib.py @@ -0,0 +1,1091 @@ +"""Test script for ftplib module.""" + +# Modified by Giampaolo Rodola' to test FTP class, IPv6 and TLS +# environment + +import ftplib +import asyncore +import asynchat +import socket +import io +import errno +import os +import threading +import time +try: + import ssl +except ImportError: + ssl = None + +from unittest import TestCase, skipUnless +from test import support +from test.support import HOST, HOSTv6 + +TIMEOUT = 3 +# the dummy data returned by server over the data channel when +# RETR, LIST, NLST, MLSD commands are issued +RETR_DATA = 'abcde12345\r\n' * 1000 +LIST_DATA = 'foo\r\nbar\r\n' +NLST_DATA = 'foo\r\nbar\r\n' +MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n" + "type=pdir;perm=e;unique==keVO1+d?3; ..\r\n" + "type=OS.unix=slink:/foobar;perm=;unique==keVO1+4G4; foobar\r\n" + "type=OS.unix=chr-13/29;perm=;unique==keVO1+5G4; device\r\n" + "type=OS.unix=blk-11/108;perm=;unique==keVO1+6G4; block\r\n" + "type=file;perm=awr;unique==keVO1+8G4; writable\r\n" + "type=dir;perm=cpmel;unique==keVO1+7G4; promiscuous\r\n" + "type=dir;perm=;unique==keVO1+1t2; no-exec\r\n" + "type=file;perm=r;unique==keVO1+EG4; two words\r\n" + "type=file;perm=r;unique==keVO1+IH4; leading space\r\n" + "type=file;perm=r;unique==keVO1+1G4; file1\r\n" + "type=dir;perm=cpmel;unique==keVO1+7G4; incoming\r\n" + "type=file;perm=r;unique==keVO1+1G4; file2\r\n" + "type=file;perm=r;unique==keVO1+1G4; file3\r\n" + "type=file;perm=r;unique==keVO1+1G4; file4\r\n") + + +class DummyDTPHandler(asynchat.async_chat): + dtp_conn_closed = False + + def __init__(self, conn, baseclass): + asynchat.async_chat.__init__(self, conn) + self.baseclass = baseclass + self.baseclass.last_received_data = '' + + def handle_read(self): + self.baseclass.last_received_data += self.recv(1024).decode('ascii') + + def handle_close(self): + # XXX: this method can be called many times in a row for a single + # connection, including in clear-text (non-TLS) mode. + # (behaviour witnessed with test_data_connection) + if not self.dtp_conn_closed: + self.baseclass.push('226 transfer complete') + self.close() + self.dtp_conn_closed = True + + def push(self, what): + if self.baseclass.next_data is not None: + what = self.baseclass.next_data + self.baseclass.next_data = None + if not what: + return self.close_when_done() + super(DummyDTPHandler, self).push(what.encode('ascii')) + + def handle_error(self): + raise Exception + + +class DummyFTPHandler(asynchat.async_chat): + + dtp_handler = DummyDTPHandler + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + # tells the socket to handle urgent data inline (ABOR command) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_OOBINLINE, 1) + self.set_terminator(b"\r\n") + self.in_buffer = [] + self.dtp = None + self.last_received_cmd = None + self.last_received_data = '' + self.next_response = '' + self.next_data = None + self.rest = None + self.next_retr_data = RETR_DATA + self.push('220 welcome') + + def collect_incoming_data(self, data): + self.in_buffer.append(data) + + def found_terminator(self): + line = b''.join(self.in_buffer).decode('ascii') + self.in_buffer = [] + if self.next_response: + self.push(self.next_response) + self.next_response = '' + cmd = line.split(' ')[0].lower() + self.last_received_cmd = cmd + space = line.find(' ') + if space != -1: + arg = line[space + 1:] + else: + arg = "" + if hasattr(self, 'cmd_' + cmd): + method = getattr(self, 'cmd_' + cmd) + method(arg) + else: + self.push('550 command "%s" not understood.' %cmd) + + def handle_error(self): + raise Exception + + def push(self, data): + asynchat.async_chat.push(self, data.encode('ascii') + b'\r\n') + + def cmd_port(self, arg): + addr = list(map(int, arg.split(','))) + ip = '%d.%d.%d.%d' %tuple(addr[:4]) + port = (addr[4] * 256) + addr[5] + s = socket.create_connection((ip, port), timeout=TIMEOUT) + self.dtp = self.dtp_handler(s, baseclass=self) + self.push('200 active data connection established') + + def cmd_pasv(self, arg): + with socket.socket() as sock: + sock.bind((self.socket.getsockname()[0], 0)) + sock.listen() + sock.settimeout(TIMEOUT) + ip, port = sock.getsockname()[:2] + ip = ip.replace('.', ','); p1 = port / 256; p2 = port % 256 + self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2)) + conn, addr = sock.accept() + self.dtp = self.dtp_handler(conn, baseclass=self) + + def cmd_eprt(self, arg): + af, ip, port = arg.split(arg[0])[1:-1] + port = int(port) + s = socket.create_connection((ip, port), timeout=TIMEOUT) + self.dtp = self.dtp_handler(s, baseclass=self) + self.push('200 active data connection established') + + def cmd_epsv(self, arg): + with socket.socket(socket.AF_INET6) as sock: + sock.bind((self.socket.getsockname()[0], 0)) + sock.listen() + sock.settimeout(TIMEOUT) + port = sock.getsockname()[1] + self.push('229 entering extended passive mode (|||%d|)' %port) + conn, addr = sock.accept() + self.dtp = self.dtp_handler(conn, baseclass=self) + + def cmd_echo(self, arg): + # sends back the received string (used by the test suite) + self.push(arg) + + def cmd_noop(self, arg): + self.push('200 noop ok') + + def cmd_user(self, arg): + self.push('331 username ok') + + def cmd_pass(self, arg): + self.push('230 password ok') + + def cmd_acct(self, arg): + self.push('230 acct ok') + + def cmd_rnfr(self, arg): + self.push('350 rnfr ok') + + def cmd_rnto(self, arg): + self.push('250 rnto ok') + + def cmd_dele(self, arg): + self.push('250 dele ok') + + def cmd_cwd(self, arg): + self.push('250 cwd ok') + + def cmd_size(self, arg): + self.push('250 1000') + + def cmd_mkd(self, arg): + self.push('257 "%s"' %arg) + + def cmd_rmd(self, arg): + self.push('250 rmd ok') + + def cmd_pwd(self, arg): + self.push('257 "pwd ok"') + + def cmd_type(self, arg): + self.push('200 type ok') + + def cmd_quit(self, arg): + self.push('221 quit ok') + self.close() + + def cmd_abor(self, arg): + self.push('226 abor ok') + + def cmd_stor(self, arg): + self.push('125 stor ok') + + def cmd_rest(self, arg): + self.rest = arg + self.push('350 rest ok') + + def cmd_retr(self, arg): + self.push('125 retr ok') + if self.rest is not None: + offset = int(self.rest) + else: + offset = 0 + self.dtp.push(self.next_retr_data[offset:]) + self.dtp.close_when_done() + self.rest = None + + def cmd_list(self, arg): + self.push('125 list ok') + self.dtp.push(LIST_DATA) + self.dtp.close_when_done() + + def cmd_nlst(self, arg): + self.push('125 nlst ok') + self.dtp.push(NLST_DATA) + self.dtp.close_when_done() + + def cmd_opts(self, arg): + self.push('200 opts ok') + + def cmd_mlsd(self, arg): + self.push('125 mlsd ok') + self.dtp.push(MLSD_DATA) + self.dtp.close_when_done() + + def cmd_setlongretr(self, arg): + # For testing. Next RETR will return long line. + self.next_retr_data = 'x' * int(arg) + self.push('125 setlongretr ok') + + +class DummyFTPServer(asyncore.dispatcher, threading.Thread): + + handler = DummyFTPHandler + + def __init__(self, address, af=socket.AF_INET): + threading.Thread.__init__(self) + asyncore.dispatcher.__init__(self) + self.daemon = True + self.create_socket(af, socket.SOCK_STREAM) + self.bind(address) + self.listen(5) + self.active = False + self.active_lock = threading.Lock() + self.host, self.port = self.socket.getsockname()[:2] + self.handler_instance = None + + def start(self): + assert not self.active + self.__flag = threading.Event() + threading.Thread.start(self) + self.__flag.wait() + + def run(self): + self.active = True + self.__flag.set() + while self.active and asyncore.socket_map: + self.active_lock.acquire() + asyncore.loop(timeout=0.1, count=1) + self.active_lock.release() + asyncore.close_all(ignore_all=True) + + def stop(self): + assert self.active + self.active = False + self.join() + + def handle_accepted(self, conn, addr): + self.handler_instance = self.handler(conn) + + def handle_connect(self): + self.close() + handle_read = handle_connect + + def writable(self): + return 0 + + def handle_error(self): + raise Exception + + +if ssl is not None: + + CERTFILE = os.path.join(os.path.dirname(__file__), "keycert3.pem") + CAFILE = os.path.join(os.path.dirname(__file__), "pycacert.pem") + + class SSLConnection(asyncore.dispatcher): + """An asyncore.dispatcher subclass supporting TLS/SSL.""" + + _ssl_accepting = False + _ssl_closing = False + + def secure_connection(self): + context = ssl.SSLContext() + context.load_cert_chain(CERTFILE) + socket = context.wrap_socket(self.socket, + suppress_ragged_eofs=False, + server_side=True, + do_handshake_on_connect=False) + self.del_channel() + self.set_socket(socket) + self._ssl_accepting = True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + elif err.args[0] == ssl.SSL_ERROR_EOF: + return self.handle_close() + # TODO: SSLError does not expose alert information + elif "SSLV3_ALERT_BAD_CERTIFICATE" in err.args[1]: + return self.handle_close() + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def _do_ssl_shutdown(self): + self._ssl_closing = True + try: + self.socket = self.socket.unwrap() + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return + except OSError as err: + # Any "socket error" corresponds to a SSL_ERROR_SYSCALL return + # from OpenSSL's SSL_shutdown(), corresponding to a + # closed socket condition. See also: + # http://www.mail-archive.com/openssl-users@openssl.org/msg60710.html + pass + self._ssl_closing = False + if getattr(self, '_ccc', False) is False: + super(SSLConnection, self).close() + else: + pass + + def handle_read_event(self): + if self._ssl_accepting: + self._do_ssl_handshake() + elif self._ssl_closing: + self._do_ssl_shutdown() + else: + super(SSLConnection, self).handle_read_event() + + def handle_write_event(self): + if self._ssl_accepting: + self._do_ssl_handshake() + elif self._ssl_closing: + self._do_ssl_shutdown() + else: + super(SSLConnection, self).handle_write_event() + + def send(self, data): + try: + return super(SSLConnection, self).send(data) + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN, + ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return 0 + raise + + def recv(self, buffer_size): + try: + return super(SSLConnection, self).recv(buffer_size) + except ssl.SSLError as err: + if err.args[0] in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + return b'' + if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN): + self.handle_close() + return b'' + raise + + def handle_error(self): + raise Exception + + def close(self): + if (isinstance(self.socket, ssl.SSLSocket) and + self.socket._sslobj is not None): + self._do_ssl_shutdown() + else: + super(SSLConnection, self).close() + + + class DummyTLS_DTPHandler(SSLConnection, DummyDTPHandler): + """A DummyDTPHandler subclass supporting TLS/SSL.""" + + def __init__(self, conn, baseclass): + DummyDTPHandler.__init__(self, conn, baseclass) + if self.baseclass.secure_data_channel: + self.secure_connection() + + + class DummyTLS_FTPHandler(SSLConnection, DummyFTPHandler): + """A DummyFTPHandler subclass supporting TLS/SSL.""" + + dtp_handler = DummyTLS_DTPHandler + + def __init__(self, conn): + DummyFTPHandler.__init__(self, conn) + self.secure_data_channel = False + self._ccc = False + + def cmd_auth(self, line): + """Set up secure control channel.""" + self.push('234 AUTH TLS successful') + self.secure_connection() + + def cmd_ccc(self, line): + self.push('220 Reverting back to clear-text') + self._ccc = True + self._do_ssl_shutdown() + + def cmd_pbsz(self, line): + """Negotiate size of buffer for secure data transfer. + For TLS/SSL the only valid value for the parameter is '0'. + Any other value is accepted but ignored. + """ + self.push('200 PBSZ=0 successful.') + + def cmd_prot(self, line): + """Setup un/secure data channel.""" + arg = line.upper() + if arg == 'C': + self.push('200 Protection set to Clear') + self.secure_data_channel = False + elif arg == 'P': + self.push('200 Protection set to Private') + self.secure_data_channel = True + else: + self.push("502 Unrecognized PROT type (use C or P).") + + + class DummyTLS_FTPServer(DummyFTPServer): + handler = DummyTLS_FTPHandler + + +class TestFTPClass(TestCase): + + def setUp(self): + self.server = DummyFTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None + asyncore.close_all(ignore_all=True) + + def check_data(self, received, expected): + self.assertEqual(len(received), len(expected)) + self.assertEqual(received, expected) + + def test_getwelcome(self): + self.assertEqual(self.client.getwelcome(), '220 welcome') + + def test_sanitize(self): + self.assertEqual(self.client.sanitize('foo'), repr('foo')) + self.assertEqual(self.client.sanitize('pass 12345'), repr('pass *****')) + self.assertEqual(self.client.sanitize('PASS 12345'), repr('PASS *****')) + + def test_exceptions(self): + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\r\n0') + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\n0') + self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\r0') + self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 400') + self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 499') + self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 500') + self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 599') + self.assertRaises(ftplib.error_proto, self.client.sendcmd, 'echo 999') + + def test_all_errors(self): + exceptions = (ftplib.error_reply, ftplib.error_temp, ftplib.error_perm, + ftplib.error_proto, ftplib.Error, OSError, + EOFError) + for x in exceptions: + try: + raise x('exception not included in all_errors set') + except ftplib.all_errors: + pass + + def test_set_pasv(self): + # passive mode is supposed to be enabled by default + self.assertTrue(self.client.passiveserver) + self.client.set_pasv(True) + self.assertTrue(self.client.passiveserver) + self.client.set_pasv(False) + self.assertFalse(self.client.passiveserver) + + def test_voidcmd(self): + self.client.voidcmd('echo 200') + self.client.voidcmd('echo 299') + self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199') + self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300') + + def test_login(self): + self.client.login() + + def test_acct(self): + self.client.acct('passwd') + + def test_rename(self): + self.client.rename('a', 'b') + self.server.handler_instance.next_response = '200' + self.assertRaises(ftplib.error_reply, self.client.rename, 'a', 'b') + + def test_delete(self): + self.client.delete('foo') + self.server.handler_instance.next_response = '199' + self.assertRaises(ftplib.error_reply, self.client.delete, 'foo') + + def test_size(self): + self.client.size('foo') + + def test_mkd(self): + dir = self.client.mkd('/foo') + self.assertEqual(dir, '/foo') + + def test_rmd(self): + self.client.rmd('foo') + + def test_cwd(self): + dir = self.client.cwd('/foo') + self.assertEqual(dir, '250 cwd ok') + + def test_pwd(self): + dir = self.client.pwd() + self.assertEqual(dir, 'pwd ok') + + def test_quit(self): + self.assertEqual(self.client.quit(), '221 quit ok') + # Ensure the connection gets closed; sock attribute should be None + self.assertEqual(self.client.sock, None) + + def test_abort(self): + self.client.abort() + + def test_retrbinary(self): + def callback(data): + received.append(data.decode('ascii')) + received = [] + self.client.retrbinary('retr', callback) + self.check_data(''.join(received), RETR_DATA) + + def test_retrbinary_rest(self): + def callback(data): + received.append(data.decode('ascii')) + for rest in (0, 10, 20): + received = [] + self.client.retrbinary('retr', callback, rest=rest) + self.check_data(''.join(received), RETR_DATA[rest:]) + + def test_retrlines(self): + received = [] + self.client.retrlines('retr', received.append) + self.check_data(''.join(received), RETR_DATA.replace('\r\n', '')) + + def test_storbinary(self): + f = io.BytesIO(RETR_DATA.encode('ascii')) + self.client.storbinary('stor', f) + self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + # test new callback arg + flag = [] + f.seek(0) + self.client.storbinary('stor', f, callback=lambda x: flag.append(None)) + self.assertTrue(flag) + + def test_storbinary_rest(self): + f = io.BytesIO(RETR_DATA.replace('\r\n', '\n').encode('ascii')) + for r in (30, '30'): + f.seek(0) + self.client.storbinary('stor', f, rest=r) + self.assertEqual(self.server.handler_instance.rest, str(r)) + + def test_storlines(self): + f = io.BytesIO(RETR_DATA.replace('\r\n', '\n').encode('ascii')) + self.client.storlines('stor', f) + self.check_data(self.server.handler_instance.last_received_data, RETR_DATA) + # test new callback arg + flag = [] + f.seek(0) + self.client.storlines('stor foo', f, callback=lambda x: flag.append(None)) + self.assertTrue(flag) + + f = io.StringIO(RETR_DATA.replace('\r\n', '\n')) + # storlines() expects a binary file, not a text file + with support.check_warnings(('', BytesWarning), quiet=True): + self.assertRaises(TypeError, self.client.storlines, 'stor foo', f) + + def test_nlst(self): + self.client.nlst() + self.assertEqual(self.client.nlst(), NLST_DATA.split('\r\n')[:-1]) + + def test_dir(self): + l = [] + self.client.dir(lambda x: l.append(x)) + self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) + + def test_mlsd(self): + list(self.client.mlsd()) + list(self.client.mlsd(path='/')) + list(self.client.mlsd(path='/', facts=['size', 'type'])) + + ls = list(self.client.mlsd()) + for name, facts in ls: + self.assertIsInstance(name, str) + self.assertIsInstance(facts, dict) + self.assertTrue(name) + self.assertIn('type', facts) + self.assertIn('perm', facts) + self.assertIn('unique', facts) + + def set_data(data): + self.server.handler_instance.next_data = data + + def test_entry(line, type=None, perm=None, unique=None, name=None): + type = 'type' if type is None else type + perm = 'perm' if perm is None else perm + unique = 'unique' if unique is None else unique + name = 'name' if name is None else name + set_data(line) + _name, facts = next(self.client.mlsd()) + self.assertEqual(_name, name) + self.assertEqual(facts['type'], type) + self.assertEqual(facts['perm'], perm) + self.assertEqual(facts['unique'], unique) + + # plain + test_entry('type=type;perm=perm;unique=unique; name\r\n') + # "=" in fact value + test_entry('type=ty=pe;perm=perm;unique=unique; name\r\n', type="ty=pe") + test_entry('type==type;perm=perm;unique=unique; name\r\n', type="=type") + test_entry('type=t=y=pe;perm=perm;unique=unique; name\r\n', type="t=y=pe") + test_entry('type=====;perm=perm;unique=unique; name\r\n', type="====") + # spaces in name + test_entry('type=type;perm=perm;unique=unique; na me\r\n', name="na me") + test_entry('type=type;perm=perm;unique=unique; name \r\n', name="name ") + test_entry('type=type;perm=perm;unique=unique; name\r\n', name=" name") + test_entry('type=type;perm=perm;unique=unique; n am e\r\n', name="n am e") + # ";" in name + test_entry('type=type;perm=perm;unique=unique; na;me\r\n', name="na;me") + test_entry('type=type;perm=perm;unique=unique; ;name\r\n', name=";name") + test_entry('type=type;perm=perm;unique=unique; ;name;\r\n', name=";name;") + test_entry('type=type;perm=perm;unique=unique; ;;;;\r\n', name=";;;;") + # case sensitiveness + set_data('Type=type;TyPe=perm;UNIQUE=unique; name\r\n') + _name, facts = next(self.client.mlsd()) + for x in facts: + self.assertTrue(x.islower()) + # no data (directory empty) + set_data('') + self.assertRaises(StopIteration, next, self.client.mlsd()) + set_data('') + for x in self.client.mlsd(): + self.fail("unexpected data %s" % x) + + def test_makeport(self): + with self.client.makeport(): + # IPv4 is in use, just make sure send_eprt has not been used + self.assertEqual(self.server.handler_instance.last_received_cmd, + 'port') + + def test_makepasv(self): + host, port = self.client.makepasv() + conn = socket.create_connection((host, port), timeout=TIMEOUT) + conn.close() + # IPv4 is in use, just make sure send_epsv has not been used + self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv') + + def test_with_statement(self): + self.client.quit() + + def is_client_connected(): + if self.client.sock is None: + return False + try: + self.client.sendcmd('noop') + except (OSError, EOFError): + return False + return True + + # base test + with ftplib.FTP(timeout=TIMEOUT) as self.client: + self.client.connect(self.server.host, self.server.port) + self.client.sendcmd('noop') + self.assertTrue(is_client_connected()) + self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit') + self.assertFalse(is_client_connected()) + + # QUIT sent inside the with block + with ftplib.FTP(timeout=TIMEOUT) as self.client: + self.client.connect(self.server.host, self.server.port) + self.client.sendcmd('noop') + self.client.quit() + self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit') + self.assertFalse(is_client_connected()) + + # force a wrong response code to be sent on QUIT: error_perm + # is expected and the connection is supposed to be closed + try: + with ftplib.FTP(timeout=TIMEOUT) as self.client: + self.client.connect(self.server.host, self.server.port) + self.client.sendcmd('noop') + self.server.handler_instance.next_response = '550 error on quit' + except ftplib.error_perm as err: + self.assertEqual(str(err), '550 error on quit') + else: + self.fail('Exception not raised') + # needed to give the threaded server some time to set the attribute + # which otherwise would still be == 'noop' + time.sleep(0.1) + self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit') + self.assertFalse(is_client_connected()) + + def test_source_address(self): + self.client.quit() + port = support.find_unused_port() + try: + self.client.connect(self.server.host, self.server.port, + source_address=(HOST, port)) + self.assertEqual(self.client.sock.getsockname()[1], port) + self.client.quit() + except OSError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + + def test_source_address_passive_connection(self): + port = support.find_unused_port() + self.client.source_address = (HOST, port) + try: + with self.client.transfercmd('list') as sock: + self.assertEqual(sock.getsockname()[1], port) + except OSError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + + def test_parse257(self): + self.assertEqual(ftplib.parse257('257 "/foo/bar"'), '/foo/bar') + self.assertEqual(ftplib.parse257('257 "/foo/bar" created'), '/foo/bar') + self.assertEqual(ftplib.parse257('257 ""'), '') + self.assertEqual(ftplib.parse257('257 "" created'), '') + self.assertRaises(ftplib.error_reply, ftplib.parse257, '250 "/foo/bar"') + # The 257 response is supposed to include the directory + # name and in case it contains embedded double-quotes + # they must be doubled (see RFC-959, chapter 7, appendix 2). + self.assertEqual(ftplib.parse257('257 "/foo/b""ar"'), '/foo/b"ar') + self.assertEqual(ftplib.parse257('257 "/foo/b""ar" created'), '/foo/b"ar') + + def test_line_too_long(self): + self.assertRaises(ftplib.Error, self.client.sendcmd, + 'x' * self.client.maxline * 2) + + def test_retrlines_too_long(self): + self.client.sendcmd('SETLONGRETR %d' % (self.client.maxline * 2)) + received = [] + self.assertRaises(ftplib.Error, + self.client.retrlines, 'retr', received.append) + + def test_storlines_too_long(self): + f = io.BytesIO(b'x' * self.client.maxline * 2) + self.assertRaises(ftplib.Error, self.client.storlines, 'stor', f) + + +@skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") +class TestIPv6Environment(TestCase): + + def setUp(self): + self.server = DummyFTPServer((HOSTv6, 0), af=socket.AF_INET6) + self.server.start() + self.client = ftplib.FTP(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None + asyncore.close_all(ignore_all=True) + + def test_af(self): + self.assertEqual(self.client.af, socket.AF_INET6) + + def test_makeport(self): + with self.client.makeport(): + self.assertEqual(self.server.handler_instance.last_received_cmd, + 'eprt') + + def test_makepasv(self): + host, port = self.client.makepasv() + conn = socket.create_connection((host, port), timeout=TIMEOUT) + conn.close() + self.assertEqual(self.server.handler_instance.last_received_cmd, 'epsv') + + def test_transfer(self): + def retr(): + def callback(data): + received.append(data.decode('ascii')) + received = [] + self.client.retrbinary('retr', callback) + self.assertEqual(len(''.join(received)), len(RETR_DATA)) + self.assertEqual(''.join(received), RETR_DATA) + self.client.set_pasv(True) + retr() + self.client.set_pasv(False) + retr() + + +@skipUnless(ssl, "SSL not available") +class TestTLS_FTPClassMixin(TestFTPClass): + """Repeat TestFTPClass tests starting the TLS layer for both control + and data connections first. + """ + + def setUp(self): + self.server = DummyTLS_FTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP_TLS(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + # enable TLS + self.client.auth() + self.client.prot_p() + + +@skipUnless(ssl, "SSL not available") +class TestTLS_FTPClass(TestCase): + """Specific TLS_FTP class tests.""" + + def setUp(self): + self.server = DummyTLS_FTPServer((HOST, 0)) + self.server.start() + self.client = ftplib.FTP_TLS(timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + + def tearDown(self): + self.client.close() + self.server.stop() + # Explicitly clear the attribute to prevent dangling thread + self.server = None + asyncore.close_all(ignore_all=True) + + def test_control_connection(self): + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.auth() + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + + def test_data_connection(self): + # clear text + with self.client.transfercmd('list') as sock: + self.assertNotIsInstance(sock, ssl.SSLSocket) + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + # secured, after PROT P + self.client.prot_p() + with self.client.transfercmd('list') as sock: + self.assertIsInstance(sock, ssl.SSLSocket) + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + # PROT C is issued, the connection must be in cleartext again + self.client.prot_c() + with self.client.transfercmd('list') as sock: + self.assertNotIsInstance(sock, ssl.SSLSocket) + self.assertEqual(self.client.voidresp(), "226 transfer complete") + + def test_login(self): + # login() is supposed to implicitly secure the control connection + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.login() + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + # make sure that AUTH TLS doesn't get issued again + self.client.login() + + def test_auth_issued_twice(self): + self.client.auth() + self.assertRaises(ValueError, self.client.auth) + + def test_context(self): + self.client.quit() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE, + context=ctx) + self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, + context=ctx) + self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, + keyfile=CERTFILE, context=ctx) + + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) + self.client.connect(self.server.host, self.server.port) + self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) + self.client.auth() + self.assertIs(self.client.sock.context, ctx) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + + self.client.prot_p() + with self.client.transfercmd('list') as sock: + self.assertIs(sock.context, ctx) + self.assertIsInstance(sock, ssl.SSLSocket) + + def test_ccc(self): + self.assertRaises(ValueError, self.client.ccc) + self.client.login(secure=True) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + self.client.ccc() + self.assertRaises(ValueError, self.client.sock.unwrap) + + @skipUnless(False, "FIXME: bpo-32706") + def test_check_hostname(self): + self.client.quit() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.check_hostname, True) + ctx.load_verify_locations(CAFILE) + self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT) + + # 127.0.0.1 doesn't match SAN + self.client.connect(self.server.host, self.server.port) + with self.assertRaises(ssl.CertificateError): + self.client.auth() + # exception quits connection + + self.client.connect(self.server.host, self.server.port) + self.client.prot_p() + with self.assertRaises(ssl.CertificateError): + with self.client.transfercmd("list") as sock: + pass + self.client.quit() + + self.client.connect("localhost", self.server.port) + self.client.auth() + self.client.quit() + + self.client.connect("localhost", self.server.port) + self.client.prot_p() + with self.client.transfercmd("list") as sock: + pass + + +class TestTimeouts(TestCase): + + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(20) + self.port = support.bind_port(self.sock) + self.server_thread = threading.Thread(target=self.server) + self.server_thread.daemon = True + self.server_thread.start() + # Wait for the server to be ready. + self.evt.wait() + self.evt.clear() + self.old_port = ftplib.FTP.port + ftplib.FTP.port = self.port + + def tearDown(self): + ftplib.FTP.port = self.old_port + self.server_thread.join() + # Explicitly clear the attribute to prevent dangling thread + self.server_thread = None + + def server(self): + # This method sets the evt 3 times: + # 1) when the connection is ready to be accepted. + # 2) when it is safe for the caller to close the connection + # 3) when we have closed the socket + self.sock.listen() + # (1) Signal the caller that we are ready to accept the connection. + self.evt.set() + try: + conn, addr = self.sock.accept() + except socket.timeout: + pass + else: + conn.sendall(b"1 Hola mundo\n") + conn.shutdown(socket.SHUT_WR) + # (2) Signal the caller that it is safe to close the socket. + self.evt.set() + conn.close() + finally: + self.sock.close() + + def testTimeoutDefault(self): + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + ftp = ftplib.FTP(HOST) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutNone(self): + # no timeout -- do not use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + ftp = ftplib.FTP(HOST, timeout=None) + finally: + socket.setdefaulttimeout(None) + self.assertIsNone(ftp.sock.gettimeout()) + self.evt.wait() + ftp.close() + + def testTimeoutValue(self): + # a value + ftp = ftplib.FTP(HOST, timeout=30) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutConnect(self): + ftp = ftplib.FTP() + ftp.connect(HOST, timeout=30) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutDifferentOrder(self): + ftp = ftplib.FTP(timeout=30) + ftp.connect(HOST) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + def testTimeoutDirectAccess(self): + ftp = ftplib.FTP() + ftp.timeout = 30 + ftp.connect(HOST) + self.assertEqual(ftp.sock.gettimeout(), 30) + self.evt.wait() + ftp.close() + + +class MiscTestCase(TestCase): + def test__all__(self): + blacklist = {'MSG_OOB', 'FTP_PORT', 'MAXLINE', 'CRLF', 'B_CRLF', + 'Error', 'parse150', 'parse227', 'parse229', 'parse257', + 'print_line', 'ftpcp', 'test'} + support.check__all__(self, ftplib, blacklist=blacklist) + + +def test_main(): + tests = [TestFTPClass, TestTimeouts, + TestIPv6Environment, + TestTLS_FTPClassMixin, TestTLS_FTPClass, + MiscTestCase] + + thread_info = support.threading_setup() + try: + support.run_unittest(*tests) + finally: + support.threading_cleanup(*thread_info) + + +if __name__ == '__main__': + test_main() diff --git a/src/greentest/3.7/test_httplib.py b/src/greentest/3.7/test_httplib.py new file mode 100644 index 0000000..a3f8194 --- /dev/null +++ b/src/greentest/3.7/test_httplib.py @@ -0,0 +1,1923 @@ +import errno +from http import client +import io +import itertools +import os +import array +import socket +import threading + +import unittest +TestCase = unittest.TestCase + +from test import support + +here = os.path.dirname(__file__) +# Self-signed cert file for 'localhost' +CERT_localhost = os.path.join(here, 'keycert.pem') +# Self-signed cert file for 'fakehostname' +CERT_fakehostname = os.path.join(here, 'keycert2.pem') +# Self-signed cert file for self-signed.pythontest.net +CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') + +# constants for testing chunked encoding +chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd! \r\n' + '8\r\n' + 'and now \r\n' + '22\r\n' + 'for something completely different\r\n' +) +chunked_expected = b'hello world! and now for something completely different' +chunk_extension = ";foo=bar" +last_chunk = "0\r\n" +last_chunk_extended = "0" + chunk_extension + "\r\n" +trailers = "X-Dummy: foo\r\nX-Dumm2: bar\r\n" +chunked_end = "\r\n" + +HOST = support.HOST + +class FakeSocket: + def __init__(self, text, fileclass=io.BytesIO, host=None, port=None): + if isinstance(text, str): + text = text.encode("ascii") + self.text = text + self.fileclass = fileclass + self.data = b'' + self.sendall_calls = 0 + self.file_closed = False + self.host = host + self.port = port + + def sendall(self, data): + self.sendall_calls += 1 + self.data += data + + def makefile(self, mode, bufsize=None): + if mode != 'r' and mode != 'rb': + raise client.UnimplementedFileMode() + # keep the file around so we can check how much was read from it + self.file = self.fileclass(self.text) + self.file.close = self.file_close #nerf close () + return self.file + + def file_close(self): + self.file_closed = True + + def close(self): + pass + + def setsockopt(self, level, optname, value): + pass + +class EPipeSocket(FakeSocket): + + def __init__(self, text, pipe_trigger): + # When sendall() is called with pipe_trigger, raise EPIPE. + FakeSocket.__init__(self, text) + self.pipe_trigger = pipe_trigger + + def sendall(self, data): + if self.pipe_trigger in data: + raise OSError(errno.EPIPE, "gotcha") + self.data += data + + def close(self): + pass + +class NoEOFBytesIO(io.BytesIO): + """Like BytesIO, but raises AssertionError on EOF. + + This is used below to test that http.client doesn't try to read + more from the underlying file than it should. + """ + def read(self, n=-1): + data = io.BytesIO.read(self, n) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + + def readline(self, length=None): + data = io.BytesIO.readline(self, length) + if data == b'': + raise AssertionError('caller tried to read past EOF') + return data + +class FakeSocketHTTPConnection(client.HTTPConnection): + """HTTPConnection subclass using FakeSocket; counts connect() calls""" + + def __init__(self, *args): + self.connections = 0 + super().__init__('example.com') + self.fake_socket_args = args + self._create_connection = self.create_connection + + def connect(self): + """Count the number of times connect() is invoked""" + self.connections += 1 + return super().connect() + + def create_connection(self, *pos, **kw): + return FakeSocket(*self.fake_socket_args) + +class HeaderTests(TestCase): + def test_auto_headers(self): + # Some headers are added automatically, but should not be added by + # .request() if they are explicitly set. + + class HeaderCountingBuffer(list): + def __init__(self): + self.count = {} + def append(self, item): + kv = item.split(b':') + if len(kv) > 1: + # item is a 'Key: Value' header string + lcKey = kv[0].decode('ascii').lower() + self.count.setdefault(lcKey, 0) + self.count[lcKey] += 1 + list.append(self, item) + + for explicit_header in True, False: + for header in 'Content-length', 'Host', 'Accept-encoding': + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('blahblahblah') + conn._buffer = HeaderCountingBuffer() + + body = 'spamspamspam' + headers = {} + if explicit_header: + headers[header] = str(len(body)) + conn.request('POST', '/', body, headers) + self.assertEqual(conn._buffer.count[header.lower()], 1) + + def test_content_length_0(self): + + class ContentLengthChecker(list): + def __init__(self): + list.__init__(self) + self.content_length = None + def append(self, item): + kv = item.split(b':', 1) + if len(kv) > 1 and kv[0].lower() == b'content-length': + self.content_length = kv[1].strip() + list.append(self, item) + + # Here, we're testing that methods expecting a body get a + # content-length set to zero if the body is empty (either None or '') + bodies = (None, '') + methods_with_body = ('PUT', 'POST', 'PATCH') + for method, body in itertools.product(methods_with_body, bodies): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', body) + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # For these methods, we make sure that content-length is not set when + # the body is None because it might cause unexpected behaviour on the + # server. + methods_without_body = ( + 'GET', 'CONNECT', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', + ) + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', None) + self.assertEqual( + conn._buffer.content_length, None, + 'Header Content-Length set for empty body on {}'.format(method) + ) + + # If the body is set to '', that's considered to be "present but + # empty" rather than "missing", so content length would be set, even + # for methods that don't expect a body. + for method in methods_without_body: + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', '') + self.assertEqual( + conn._buffer.content_length, b'0', + 'Header Content-Length incorrect on {}'.format(method) + ) + + # If the body is set, make sure Content-Length is set. + for method in itertools.chain(methods_without_body, methods_with_body): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn._buffer = ContentLengthChecker() + conn.request(method, '/', ' ') + self.assertEqual( + conn._buffer.content_length, b'1', + 'Header Content-Length incorrect on {}'.format(method) + ) + + def test_putheader(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.putrequest('GET','/') + conn.putheader('Content-length', 42) + self.assertIn(b'Content-length: 42', conn._buffer) + + conn.putheader('Foo', ' bar ') + self.assertIn(b'Foo: bar ', conn._buffer) + conn.putheader('Bar', '\tbaz\t') + self.assertIn(b'Bar: \tbaz\t', conn._buffer) + conn.putheader('Authorization', 'Bearer mytoken') + self.assertIn(b'Authorization: Bearer mytoken', conn._buffer) + conn.putheader('IterHeader', 'IterA', 'IterB') + self.assertIn(b'IterHeader: IterA\r\n\tIterB', conn._buffer) + conn.putheader('LatinHeader', b'\xFF') + self.assertIn(b'LatinHeader: \xFF', conn._buffer) + conn.putheader('Utf8Header', b'\xc3\x80') + self.assertIn(b'Utf8Header: \xc3\x80', conn._buffer) + conn.putheader('C1-Control', b'next\x85line') + self.assertIn(b'C1-Control: next\x85line', conn._buffer) + conn.putheader('Embedded-Fold-Space', 'is\r\n allowed') + self.assertIn(b'Embedded-Fold-Space: is\r\n allowed', conn._buffer) + conn.putheader('Embedded-Fold-Tab', 'is\r\n\tallowed') + self.assertIn(b'Embedded-Fold-Tab: is\r\n\tallowed', conn._buffer) + conn.putheader('Key Space', 'value') + self.assertIn(b'Key Space: value', conn._buffer) + conn.putheader('KeySpace ', 'value') + self.assertIn(b'KeySpace : value', conn._buffer) + conn.putheader(b'Nonbreak\xa0Space', 'value') + self.assertIn(b'Nonbreak\xa0Space: value', conn._buffer) + conn.putheader(b'\xa0NonbreakSpace', 'value') + self.assertIn(b'\xa0NonbreakSpace: value', conn._buffer) + + def test_ipv6host_header(self): + # Default host header on IPv6 transaction should be wrapped by [] if + # it is an IPv6 address + expected = b'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001::]:81') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + expected = b'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \ + b'Accept-Encoding: identity\r\n\r\n' + conn = client.HTTPConnection('[2001:102A::]') + sock = FakeSocket('') + conn.sock = sock + conn.request('GET', '/foo') + self.assertTrue(sock.data.startswith(expected)) + + def test_malformed_headers_coped_with(self): + # Issue 19996 + body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + + self.assertEqual(resp.getheader('First'), 'val') + self.assertEqual(resp.getheader('Second'), 'val') + + def test_parse_all_octets(self): + # Ensure no valid header field octet breaks the parser + body = ( + b'HTTP/1.1 200 OK\r\n' + b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters + b'VCHAR: ' + bytes(range(0x21, 0x7E + 1)) + b'\r\n' + b'obs-text: ' + bytes(range(0x80, 0xFF + 1)) + b'\r\n' + b'obs-fold: text\r\n' + b' folded with space\r\n' + b'\tfolded with tab\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.getheader('Content-Length'), '0') + self.assertEqual(resp.msg['Content-Length'], '0') + self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value') + self.assertEqual(resp.msg["!#$%&'*+-.^_`|~"], 'value') + vchar = ''.join(map(chr, range(0x21, 0x7E + 1))) + self.assertEqual(resp.getheader('VCHAR'), vchar) + self.assertEqual(resp.msg['VCHAR'], vchar) + self.assertIsNotNone(resp.getheader('obs-text')) + self.assertIn('obs-text', resp.msg) + for folded in (resp.getheader('obs-fold'), resp.msg['obs-fold']): + self.assertTrue(folded.startswith('text')) + self.assertIn(' folded with space', folded) + self.assertTrue(folded.endswith('folded with tab')) + + def test_invalid_headers(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/') + + # http://tools.ietf.org/html/rfc7230#section-3.2.4, whitespace is no + # longer allowed in header names + cases = ( + (b'Invalid\r\nName', b'ValidValue'), + (b'Invalid\rName', b'ValidValue'), + (b'Invalid\nName', b'ValidValue'), + (b'\r\nInvalidName', b'ValidValue'), + (b'\rInvalidName', b'ValidValue'), + (b'\nInvalidName', b'ValidValue'), + (b' InvalidName', b'ValidValue'), + (b'\tInvalidName', b'ValidValue'), + (b'Invalid:Name', b'ValidValue'), + (b':InvalidName', b'ValidValue'), + (b'ValidName', b'Invalid\r\nValue'), + (b'ValidName', b'Invalid\rValue'), + (b'ValidName', b'Invalid\nValue'), + (b'ValidName', b'InvalidValue\r\n'), + (b'ValidName', b'InvalidValue\r'), + (b'ValidName', b'InvalidValue\n'), + ) + for name, value in cases: + with self.subTest((name, value)): + with self.assertRaisesRegex(ValueError, 'Invalid header'): + conn.putheader(name, value) + + +class TransferEncodingTest(TestCase): + expected_body = b"It's just a flesh wound" + + def test_endheaders_chunked(self): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.putrequest('POST', '/') + conn.endheaders(self._make_body(), encode_chunked=True) + + _, _, body = self._parse_request(conn.sock.data) + body = self._parse_chunked(body) + self.assertEqual(body, self.expected_body) + + def test_explicit_headers(self): + # explicit chunked + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + # this shouldn't actually be automatically chunk-encoded because the + # calling code has explicitly stated that it's taking care of it + conn.request( + 'POST', '/', self._make_body(), {'Transfer-Encoding': 'chunked'}) + + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers.keys()]) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertEqual(body, self.expected_body) + + # explicit chunked, string body + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request( + 'POST', '/', self.expected_body.decode('latin-1'), + {'Transfer-Encoding': 'chunked'}) + + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers.keys()]) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertEqual(body, self.expected_body) + + # User-specified TE, but request() does the chunk encoding + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request('POST', '/', + headers={'Transfer-Encoding': 'gzip, chunked'}, + encode_chunked=True, + body=self._make_body()) + _, headers, body = self._parse_request(conn.sock.data) + self.assertNotIn('content-length', [k.lower() for k in headers]) + self.assertEqual(headers['Transfer-Encoding'], 'gzip, chunked') + self.assertEqual(self._parse_chunked(body), self.expected_body) + + def test_request(self): + for empty_lines in (False, True,): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request( + 'POST', '/', self._make_body(empty_lines=empty_lines)) + + _, headers, body = self._parse_request(conn.sock.data) + body = self._parse_chunked(body) + self.assertEqual(body, self.expected_body) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + + # Content-Length and Transfer-Encoding SHOULD not be sent in the + # same request + self.assertNotIn('content-length', [k.lower() for k in headers]) + + def test_empty_body(self): + # Zero-length iterable should be treated like any other iterable + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(b'') + conn.request('POST', '/', ()) + _, headers, body = self._parse_request(conn.sock.data) + self.assertEqual(headers['Transfer-Encoding'], 'chunked') + self.assertNotIn('content-length', [k.lower() for k in headers]) + self.assertEqual(body, b"0\r\n\r\n") + + def _make_body(self, empty_lines=False): + lines = self.expected_body.split(b' ') + for idx, line in enumerate(lines): + # for testing handling empty lines + if empty_lines and idx % 2: + yield b'' + if idx < len(lines) - 1: + yield line + b' ' + else: + yield line + + def _parse_request(self, data): + lines = data.split(b'\r\n') + request = lines[0] + headers = {} + n = 1 + while n < len(lines) and len(lines[n]) > 0: + key, val = lines[n].split(b':') + key = key.decode('latin-1').strip() + headers[key] = val.decode('latin-1').strip() + n += 1 + + return request, headers, b'\r\n'.join(lines[n + 1:]) + + def _parse_chunked(self, data): + body = [] + trailers = {} + n = 0 + lines = data.split(b'\r\n') + # parse body + while True: + size, chunk = lines[n:n+2] + size = int(size, 16) + + if size == 0: + n += 1 + break + + self.assertEqual(size, len(chunk)) + body.append(chunk) + + n += 2 + # we /should/ hit the end chunk, but check against the size of + # lines so we're not stuck in an infinite loop should we get + # malformed data + if n > len(lines): + break + + return b''.join(body) + + +class BasicTest(TestCase): + def test_status_lines(self): + # Test HTTP status lines + + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(0), b'') # Issue #20007 + self.assertFalse(resp.isclosed()) + self.assertFalse(resp.closed) + self.assertEqual(resp.read(), b"Text") + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + self.assertRaises(client.BadStatusLine, resp.begin) + + def test_bad_status_repr(self): + exc = client.BadStatusLine('') + self.assertEqual(repr(exc), '''BadStatusLine("''")''') + + def test_partial_reads(self): + # if we have Content-Length, HTTPResponse knows when to close itself, + # the same behaviour as when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_mixed_reads(self): + # readline() should update the remaining length, so that read() knows + # how much data is left and does not raise IncompleteRead + body = "HTTP/1.1 200 Ok\r\nContent-Length: 13\r\n\r\nText\r\nAnother" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.readline(), b'Text\r\n') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(), b'Another') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos(self): + # if we have Content-Length, HTTPResponse knows when to close itself, + # the same behaviour as when we read the whole thing with read() + body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_reads_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_partial_readintos_no_content_length(self): + # when no length is present, the socket should be gracefully closed when + # all data was read + body = "HTTP/1.1 200 Ok\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + + def test_partial_reads_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(2), b'Te') + self.assertFalse(resp.isclosed()) + self.assertEqual(resp.read(2), b'xt') + self.assertEqual(resp.read(1), b'') + self.assertTrue(resp.isclosed()) + + def test_partial_readintos_incomplete_body(self): + # if the server shuts down the connection before the whole + # content-length is delivered, the socket is gracefully closed + body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + b = bytearray(2) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'Te') + self.assertFalse(resp.isclosed()) + n = resp.readinto(b) + self.assertEqual(n, 2) + self.assertEqual(bytes(b), b'xt') + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:80", "www.python.org", 80), + ("www.python.org:", "www.python.org", 80), + ("www.python.org", "www.python.org", 80), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", 80)): + c = client.HTTPConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + def test_response_headers(self): + # test response with multiple message headers with the same field name. + text = ('HTTP/1.1 200 OK\r\n' + 'Set-Cookie: Customer="WILE_E_COYOTE"; ' + 'Version="1"; Path="/acme"\r\n' + 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";' + ' Path="/acme"\r\n' + '\r\n' + 'No body\r\n') + hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"' + ', ' + 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"') + s = FakeSocket(text) + r = client.HTTPResponse(s) + r.begin() + cookies = r.getheader("Set-Cookie") + self.assertEqual(cookies, hdr) + + def test_read_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + if resp.read(): + self.fail("Did not expect response from HEAD request") + + def test_readinto_head(self): + # Test that the library doesn't attempt to read any data + # from a HEAD request. (Tickles SF bug #622042.) + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 14432\r\n' + '\r\n', + NoEOFBytesIO) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + if resp.readinto(b) != 0: + self.fail("Did not expect response from HEAD request") + self.assertEqual(bytes(b), b'\x00'*5) + + def test_too_many_headers(self): + headers = '\r\n'.join('Header%d: foo' % i + for i in range(client._MAXHEADERS + 1)) + '\r\n' + text = ('HTTP/1.1 200 OK\r\n' + headers) + s = FakeSocket(text) + r = client.HTTPResponse(s) + self.assertRaisesRegex(client.HTTPException, + r"got more than \d+ headers", r.begin) + + def test_send_file(self): + expected = (b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' + b'Accept-Encoding: identity\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n') + + with open(__file__, 'rb') as body: + conn = client.HTTPConnection('example.com') + sock = FakeSocket(body) + conn.sock = sock + conn.request('GET', '/foo', body) + self.assertTrue(sock.data.startswith(expected), '%r != %r' % + (sock.data[:len(expected)], expected)) + + def test_send(self): + expected = b'this is a test this is only a test' + conn = client.HTTPConnection('example.com') + sock = FakeSocket(None) + conn.sock = sock + conn.send(expected) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(array.array('b', expected)) + self.assertEqual(expected, sock.data) + sock.data = b'' + conn.send(io.BytesIO(expected)) + self.assertEqual(expected, sock.data) + + def test_send_updating_file(self): + def data(): + yield 'data' + yield None + yield 'data_two' + + class UpdatingFile(io.TextIOBase): + mode = 'r' + d = data() + def read(self, blocksize=-1): + return next(self.d) + + expected = b'data' + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.send(UpdatingFile()) + self.assertEqual(sock.data, expected) + + + def test_send_iter(self): + expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \ + b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \ + b'\r\nonetwothree' + + def body(): + yield b"one" + yield b"two" + yield b"three" + + conn = client.HTTPConnection('example.com') + sock = FakeSocket("") + conn.sock = sock + conn.request('GET', '/foo', body(), {'Content-Length': '11'}) + self.assertEqual(sock.data, expected) + + def test_blocksize_request(self): + """Check that request() respects the configured block size.""" + blocksize = 8 # For easy debugging. + conn = client.HTTPConnection('example.com', blocksize=blocksize) + sock = FakeSocket(None) + conn.sock = sock + expected = b"a" * blocksize + b"b" + conn.request("PUT", "/", io.BytesIO(expected), {"Content-Length": "9"}) + self.assertEqual(sock.sendall_calls, 3) + body = sock.data.split(b"\r\n\r\n", 1)[1] + self.assertEqual(body, expected) + + def test_blocksize_send(self): + """Check that send() respects the configured block size.""" + blocksize = 8 # For easy debugging. + conn = client.HTTPConnection('example.com', blocksize=blocksize) + sock = FakeSocket(None) + conn.sock = sock + expected = b"a" * blocksize + b"b" + conn.send(io.BytesIO(expected)) + self.assertEqual(sock.sendall_calls, 2) + self.assertEqual(sock.data, expected) + + def test_send_type_error(self): + # See: Issue #12676 + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket('') + with self.assertRaises(TypeError): + conn.request('POST', 'test', conn) + + def test_chunked(self): + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(n) + resp.read(n) + resp.read(), expected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_readinto_chunked(self): + + expected = chunked_expected + nexpected = len(expected) + b = bytearray(128) + + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + n = resp.readinto(b) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(n, nexpected) + resp.close() + + # Various read sizes + for n in range(1, 12): + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + m = memoryview(b) + i = resp.readinto(m[0:n]) + i += resp.readinto(m[i:n + i]) + i += resp.readinto(m[i:]) + self.assertEqual(b[:nexpected], expected) + self.assertEqual(i, nexpected) + resp.close() + + for x in ('', 'foo\r\n'): + sock = FakeSocket(chunked_start + x) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + n = resp.readinto(b) + except client.IncompleteRead as i: + self.assertEqual(i.partial, expected) + expected_message = 'IncompleteRead(%d bytes read)' % len(expected) + self.assertEqual(repr(i), expected_message) + self.assertEqual(str(i), expected_message) + else: + self.fail('IncompleteRead expected') + finally: + resp.close() + + def test_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_readinto_chunked_head(self): + chunked_start = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello world\r\n' + '1\r\n' + 'd\r\n' + ) + sock = FakeSocket(chunked_start + last_chunk + chunked_end) + resp = client.HTTPResponse(sock, method="HEAD") + resp.begin() + b = bytearray(5) + n = resp.readinto(b) + self.assertEqual(n, 0) + self.assertEqual(bytes(b), b'\x00'*5) + self.assertEqual(resp.status, 200) + self.assertEqual(resp.reason, 'OK') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_negative_content_length(self): + sock = FakeSocket( + 'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), b'Hello\r\n') + self.assertTrue(resp.isclosed()) + + def test_incomplete_read(self): + sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n') + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + try: + resp.read() + except client.IncompleteRead as i: + self.assertEqual(i.partial, b'Hello\r\n') + self.assertEqual(repr(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertEqual(str(i), + "IncompleteRead(7 bytes read, 3 more expected)") + self.assertTrue(resp.isclosed()) + else: + self.fail('IncompleteRead expected') + + def test_epipe(self): + sock = EPipeSocket( + "HTTP/1.0 401 Authorization Required\r\n" + "Content-type: text/html\r\n" + "WWW-Authenticate: Basic realm=\"example\"\r\n", + b"Content-Length") + conn = client.HTTPConnection("example.com") + conn.sock = sock + self.assertRaises(OSError, + lambda: conn.request("PUT", "/url", "body")) + resp = conn.getresponse() + self.assertEqual(401, resp.status) + self.assertEqual("Basic realm=\"example\"", + resp.getheader("www-authenticate")) + + # Test lines overflowing the max line size (_MAXLINE in http.client) + + def test_overflowing_status_line(self): + body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n" + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises((client.LineTooLong, client.BadStatusLine), resp.begin) + + def test_overflowing_header_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + self.assertRaises(client.LineTooLong, resp.begin) + + def test_overflowing_chunked_line(self): + body = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + + '0' * 65536 + 'a\r\n' + 'hello world\r\n' + '0\r\n' + '\r\n' + ) + resp = client.HTTPResponse(FakeSocket(body)) + resp.begin() + self.assertRaises(client.LineTooLong, resp.read) + + def test_early_eof(self): + # Test httpresponse with no \r\n termination, + body = "HTTP/1.1 200 Ok" + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.read(), b'') + self.assertTrue(resp.isclosed()) + self.assertFalse(resp.closed) + resp.close() + self.assertTrue(resp.closed) + + def test_error_leak(self): + # Test that the socket is not leaked if getresponse() fails + conn = client.HTTPConnection('example.com') + response = None + class Response(client.HTTPResponse): + def __init__(self, *pos, **kw): + nonlocal response + response = self # Avoid garbage collector closing the socket + client.HTTPResponse.__init__(self, *pos, **kw) + conn.response_class = Response + conn.sock = FakeSocket('Invalid status line') + conn.request('GET', '/') + self.assertRaises(client.BadStatusLine, conn.getresponse) + self.assertTrue(response.closed) + self.assertTrue(conn.sock.file_closed) + + def test_chunked_extension(self): + extra = '3;foo=bar\r\n' + 'abc\r\n' + expected = chunked_expected + b'abc' + + sock = FakeSocket(chunked_start + extra + last_chunk_extended + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + def test_chunked_missing_end(self): + """some servers may serve up a short chunked encoding stream""" + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk) #no terminating crlf + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + resp.close() + + def test_chunked_trailers(self): + """See that trailers are read and ignored""" + expected = chunked_expected + sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # we should have reached the end of the file + self.assertEqual(sock.file.read(), b"") #we read to the end + resp.close() + + def test_chunked_sync(self): + """Check that we don't read past the end of the chunked-encoding stream""" + expected = chunked_expected + extradata = "extradata" + sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata.encode("ascii")) #we read to the end + resp.close() + + def test_content_length_sync(self): + """Check that we don't read past the end of the Content-Length stream""" + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_readlines_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.readlines(2000), [expected]) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_read1_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read1(2000), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_readline_bound_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.readline(10), expected) + self.assertEqual(resp.readline(10), b"") + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_read1_bound_content_length(self): + extradata = b"extradata" + expected = b"Hello123\r\n" + sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 30\r\n\r\n' + expected*3 + extradata) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + self.assertEqual(resp.read1(20), expected*2) + self.assertEqual(resp.read(), expected) + # the file should now have our extradata ready to be read + self.assertEqual(sock.file.read(), extradata) #we read to the end + resp.close() + + def test_response_fileno(self): + # Make sure fd returned by fileno is valid. + serv = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) + self.addCleanup(serv.close) + serv.bind((HOST, 0)) + serv.listen() + + result = None + def run_server(): + [conn, address] = serv.accept() + with conn, conn.makefile("rb") as reader: + # Read the request header until a blank line + while True: + line = reader.readline() + if not line.rstrip(b"\r\n"): + break + conn.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n") + nonlocal result + result = reader.read() + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, float(1)) + conn = client.HTTPConnection(*serv.getsockname()) + conn.request("CONNECT", "dummy:1234") + response = conn.getresponse() + try: + self.assertEqual(response.status, client.OK) + s = socket.socket(fileno=response.fileno()) + try: + s.sendall(b"proxied data\n") + finally: + s.detach() + finally: + response.close() + conn.close() + thread.join() + self.assertEqual(result, b"proxied data\n") + +class ExtendedReadTest(TestCase): + """ + Test peek(), read1(), readline() + """ + lines = ( + 'HTTP/1.1 200 OK\r\n' + '\r\n' + 'hello world!\n' + 'and now \n' + 'for something completely different\n' + 'foo' + ) + lines_expected = lines[lines.find('hello'):].encode("ascii") + lines_chunked = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd!\n\r\n' + '9\r\n' + 'and now \n\r\n' + '23\r\n' + 'for something completely different\n\r\n' + '3\r\n' + 'foo\r\n' + '0\r\n' # terminating chunk + '\r\n' # end of trailers + ) + + def setUp(self): + sock = FakeSocket(self.lines) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + resp.fp = io.BufferedReader(resp.fp) + self.resp = resp + + + + def test_peek(self): + resp = self.resp + # patch up the buffered peek so that it returns not too much stuff + oldpeek = resp.fp.peek + def mypeek(n=-1): + p = oldpeek(n) + if n >= 0: + return p[:n] + return p[:10] + resp.fp.peek = mypeek + + all = [] + while True: + # try a short peek + p = resp.peek(3) + if p: + self.assertGreater(len(p), 0) + # then unbounded peek + p2 = resp.peek() + self.assertGreaterEqual(len(p2), len(p)) + self.assertTrue(p2.startswith(p)) + next = resp.read(len(p2)) + self.assertEqual(next, p2) + else: + next = resp.read() + self.assertFalse(next) + all.append(next) + if not next: + break + self.assertEqual(b"".join(all), self.lines_expected) + + def test_readline(self): + resp = self.resp + self._verify_readline(self.resp.readline, self.lines_expected) + + def _verify_readline(self, readline, expected): + all = [] + while True: + # short readlines + line = readline(5) + if line and line != b"foo": + if len(line) < 5: + self.assertTrue(line.endswith(b"\n")) + all.append(line) + if not line: + break + self.assertEqual(b"".join(all), expected) + + def test_read1(self): + resp = self.resp + def r(): + res = resp.read1(4) + self.assertLessEqual(len(res), 4) + return res + readliner = Readliner(r) + self._verify_readline(readliner.readline, self.lines_expected) + + def test_read1_unbounded(self): + resp = self.resp + all = [] + while True: + data = resp.read1() + if not data: + break + all.append(data) + self.assertEqual(b"".join(all), self.lines_expected) + + def test_read1_bounded(self): + resp = self.resp + all = [] + while True: + data = resp.read1(10) + if not data: + break + self.assertLessEqual(len(data), 10) + all.append(data) + self.assertEqual(b"".join(all), self.lines_expected) + + def test_read1_0(self): + self.assertEqual(self.resp.read1(0), b"") + + def test_peek_0(self): + p = self.resp.peek(0) + self.assertLessEqual(0, len(p)) + +class ExtendedReadTestChunked(ExtendedReadTest): + """ + Test peek(), read1(), readline() in chunked mode + """ + lines = ( + 'HTTP/1.1 200 OK\r\n' + 'Transfer-Encoding: chunked\r\n\r\n' + 'a\r\n' + 'hello worl\r\n' + '3\r\n' + 'd!\n\r\n' + '9\r\n' + 'and now \n\r\n' + '23\r\n' + 'for something completely different\n\r\n' + '3\r\n' + 'foo\r\n' + '0\r\n' # terminating chunk + '\r\n' # end of trailers + ) + + +class Readliner: + """ + a simple readline class that uses an arbitrary read function and buffering + """ + def __init__(self, readfunc): + self.readfunc = readfunc + self.remainder = b"" + + def readline(self, limit): + data = [] + datalen = 0 + read = self.remainder + try: + while True: + idx = read.find(b'\n') + if idx != -1: + break + if datalen + len(read) >= limit: + idx = limit - datalen - 1 + # read more data + data.append(read) + read = self.readfunc() + if not read: + idx = 0 #eof condition + break + idx += 1 + data.append(read[:idx]) + self.remainder = read[idx:] + return b"".join(data) + except: + self.remainder = b"".join(data) + raise + + +class OfflineTest(TestCase): + def test_all(self): + # Documented objects defined in the module should be in __all__ + expected = {"responses"} # White-list documented dict() object + # HTTPMessage, parse_headers(), and the HTTP status code constants are + # intentionally omitted for simplicity + blacklist = {"HTTPMessage", "parse_headers"} + for name in dir(client): + if name.startswith("_") or name in blacklist: + continue + module_object = getattr(client, name) + if getattr(module_object, "__module__", None) == "http.client": + expected.add(name) + self.assertCountEqual(client.__all__, expected) + + def test_responses(self): + self.assertEqual(client.responses[client.NOT_FOUND], "Not Found") + + def test_client_constants(self): + # Make sure we don't break backward compatibility with 3.4 + expected = [ + 'CONTINUE', + 'SWITCHING_PROTOCOLS', + 'PROCESSING', + 'OK', + 'CREATED', + 'ACCEPTED', + 'NON_AUTHORITATIVE_INFORMATION', + 'NO_CONTENT', + 'RESET_CONTENT', + 'PARTIAL_CONTENT', + 'MULTI_STATUS', + 'IM_USED', + 'MULTIPLE_CHOICES', + 'MOVED_PERMANENTLY', + 'FOUND', + 'SEE_OTHER', + 'NOT_MODIFIED', + 'USE_PROXY', + 'TEMPORARY_REDIRECT', + 'BAD_REQUEST', + 'UNAUTHORIZED', + 'PAYMENT_REQUIRED', + 'FORBIDDEN', + 'NOT_FOUND', + 'METHOD_NOT_ALLOWED', + 'NOT_ACCEPTABLE', + 'PROXY_AUTHENTICATION_REQUIRED', + 'REQUEST_TIMEOUT', + 'CONFLICT', + 'GONE', + 'LENGTH_REQUIRED', + 'PRECONDITION_FAILED', + 'REQUEST_ENTITY_TOO_LARGE', + 'REQUEST_URI_TOO_LONG', + 'UNSUPPORTED_MEDIA_TYPE', + 'REQUESTED_RANGE_NOT_SATISFIABLE', + 'EXPECTATION_FAILED', + 'MISDIRECTED_REQUEST', + 'UNPROCESSABLE_ENTITY', + 'LOCKED', + 'FAILED_DEPENDENCY', + 'UPGRADE_REQUIRED', + 'PRECONDITION_REQUIRED', + 'TOO_MANY_REQUESTS', + 'REQUEST_HEADER_FIELDS_TOO_LARGE', + 'INTERNAL_SERVER_ERROR', + 'NOT_IMPLEMENTED', + 'BAD_GATEWAY', + 'SERVICE_UNAVAILABLE', + 'GATEWAY_TIMEOUT', + 'HTTP_VERSION_NOT_SUPPORTED', + 'INSUFFICIENT_STORAGE', + 'NOT_EXTENDED', + 'NETWORK_AUTHENTICATION_REQUIRED', + ] + for const in expected: + with self.subTest(constant=const): + self.assertTrue(hasattr(client, const)) + + +class SourceAddressTest(TestCase): + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.source_port = support.find_unused_port() + self.serv.listen() + self.conn = None + + def tearDown(self): + if self.conn: + self.conn.close() + self.conn = None + self.serv.close() + self.serv = None + + def testHTTPConnectionSourceAddress(self): + self.conn = client.HTTPConnection(HOST, self.port, + source_address=('', self.source_port)) + self.conn.connect() + self.assertEqual(self.conn.sock.getsockname()[1], self.source_port) + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not defined') + def testHTTPSConnectionSourceAddress(self): + self.conn = client.HTTPSConnection(HOST, self.port, + source_address=('', self.source_port)) + # We don't test anything here other than the constructor not barfing as + # this code doesn't deal with setting up an active running SSL server + # for an ssl_wrapped connect() to actually return from. + + +class TimeoutTest(TestCase): + PORT = None + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + TimeoutTest.PORT = support.bind_port(self.serv) + self.serv.listen() + + def tearDown(self): + self.serv.close() + self.serv = None + + def testTimeoutAttribute(self): + # This will prove that the timeout gets through HTTPConnection + # and into the socket. + + # default -- use global socket timeout + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + # no timeout -- do not use global socket default + self.assertIsNone(socket.getdefaulttimeout()) + socket.setdefaulttimeout(30) + try: + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, + timeout=None) + httpConn.connect() + finally: + socket.setdefaulttimeout(None) + self.assertEqual(httpConn.sock.gettimeout(), None) + httpConn.close() + + # a value + httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30) + httpConn.connect() + self.assertEqual(httpConn.sock.gettimeout(), 30) + httpConn.close() + + +class PersistenceTest(TestCase): + + def test_reuse_reconnect(self): + # Should reuse or reconnect depending on header from server + tests = ( + ('1.0', '', False), + ('1.0', 'Connection: keep-alive\r\n', True), + ('1.1', '', True), + ('1.1', 'Connection: close\r\n', False), + ('1.0', 'Connection: keep-ALIVE\r\n', True), + ('1.1', 'Connection: cloSE\r\n', False), + ) + for version, header, reuse in tests: + with self.subTest(version=version, header=header): + msg = ( + 'HTTP/{} 200 OK\r\n' + '{}' + 'Content-Length: 12\r\n' + '\r\n' + 'Dummy body\r\n' + ).format(version, header) + conn = FakeSocketHTTPConnection(msg) + self.assertIsNone(conn.sock) + conn.request('GET', '/open-connection') + with conn.getresponse() as response: + self.assertEqual(conn.sock is None, not reuse) + response.read() + self.assertEqual(conn.sock is None, not reuse) + self.assertEqual(conn.connections, 1) + conn.request('GET', '/subsequent-request') + self.assertEqual(conn.connections, 1 if reuse else 2) + + def test_disconnected(self): + + def make_reset_reader(text): + """Return BufferedReader that raises ECONNRESET at EOF""" + stream = io.BytesIO(text) + def readinto(buffer): + size = io.BytesIO.readinto(stream, buffer) + if size == 0: + raise ConnectionResetError() + return size + stream.readinto = readinto + return io.BufferedReader(stream) + + tests = ( + (io.BytesIO, client.RemoteDisconnected), + (make_reset_reader, ConnectionResetError), + ) + for stream_factory, exception in tests: + with self.subTest(exception=exception): + conn = FakeSocketHTTPConnection(b'', stream_factory) + conn.request('GET', '/eof-response') + self.assertRaises(exception, conn.getresponse) + self.assertIsNone(conn.sock) + # HTTPConnection.connect() should be automatically invoked + conn.request('GET', '/reconnect') + self.assertEqual(conn.connections, 2) + + def test_100_close(self): + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + # Missing final response + ) + conn.request('GET', '/', headers={'Expect': '100-continue'}) + self.assertRaises(client.RemoteDisconnected, conn.getresponse) + self.assertIsNone(conn.sock) + conn.request('GET', '/reconnect') + self.assertEqual(conn.connections, 2) + + +class HTTPSTest(TestCase): + + def setUp(self): + if not hasattr(client, 'HTTPSConnection'): + self.skipTest('ssl support required') + + def make_server(self, certfile): + from test.ssl_servers import make_https_server + return make_https_server(self, certfile=certfile) + + def test_attributes(self): + # simple test to check it's storing the timeout + h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30) + self.assertEqual(h.timeout, 30) + + def test_networked(self): + # Default settings: requires a valid cert from a trusted CA + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + h = client.HTTPSConnection('self-signed.pythontest.net', 443) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_networked_noverification(self): + # Switch off cert verification + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl._create_unverified_context() + h = client.HTTPSConnection('self-signed.pythontest.net', 443, + context=context) + h.request('GET', '/') + resp = h.getresponse() + h.close() + self.assertIn('nginx', resp.getheader('server')) + resp.close() + + @support.system_must_validate_cert + def test_networked_trusted_by_default_cert(self): + # Default settings: requires a valid cert from a trusted CA + support.requires('network') + with support.transient_internet('www.python.org'): + h = client.HTTPSConnection('www.python.org', 443) + h.request('GET', '/') + resp = h.getresponse() + content_type = resp.getheader('content-type') + resp.close() + h.close() + self.assertIn('text/html', content_type) + + def test_networked_good_cert(self): + # We feed the server's cert as a validating cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(context.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(context.check_hostname, True) + context.load_verify_locations(CERT_selfsigned_pythontestdotnet) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + h.request('GET', '/') + resp = h.getresponse() + server_string = resp.getheader('server') + resp.close() + h.close() + self.assertIn('nginx', server_string) + + def test_networked_bad_cert(self): + # We feed a "CA" cert that is unrelated to the server's cert + import ssl + support.requires('network') + with support.transient_internet('self-signed.pythontest.net'): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_unknown_cert(self): + # The custom cert isn't known to the default trust bundle + import ssl + server = self.make_server(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port) + with self.assertRaises(ssl.SSLError) as exc_info: + h.request('GET', '/') + self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') + + def test_local_good_hostname(self): + # The (valid) cert validates the HTTP hostname + import ssl + server = self.make_server(CERT_localhost) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(CERT_localhost) + h = client.HTTPSConnection('localhost', server.port, context=context) + self.addCleanup(h.close) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.addCleanup(resp.close) + self.assertEqual(resp.status, 404) + + def test_local_bad_hostname(self): + # The (valid) cert doesn't validate the HTTP hostname + import ssl + server = self.make_server(CERT_fakehostname) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations(CERT_fakehostname) + h = client.HTTPSConnection('localhost', server.port, context=context) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # Same with explicit check_hostname=True + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + # With check_hostname=False, the mismatching is ignored + context.check_hostname = False + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=False) + h.request('GET', '/nonexistent') + resp = h.getresponse() + resp.close() + h.close() + self.assertEqual(resp.status, 404) + # The context's check_hostname setting is used if one isn't passed to + # HTTPSConnection. + context.check_hostname = False + h = client.HTTPSConnection('localhost', server.port, context=context) + h.request('GET', '/nonexistent') + resp = h.getresponse() + self.assertEqual(resp.status, 404) + resp.close() + h.close() + # Passing check_hostname to HTTPSConnection should override the + # context's setting. + with support.check_warnings(('', DeprecationWarning)): + h = client.HTTPSConnection('localhost', server.port, + context=context, check_hostname=True) + with self.assertRaises(ssl.CertificateError): + h.request('GET', '/') + + @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), + 'http.client.HTTPSConnection not available') + def test_host_port(self): + # Check invalid host_port + + for hp in ("www.python.org:abc", "user:password@www.python.org"): + self.assertRaises(client.InvalidURL, client.HTTPSConnection, hp) + + for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", + "fe80::207:e9ff:fe9b", 8000), + ("www.python.org:443", "www.python.org", 443), + ("www.python.org:", "www.python.org", 443), + ("www.python.org", "www.python.org", 443), + ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 443), + ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", + 443)): + c = client.HTTPSConnection(hp) + self.assertEqual(h, c.host) + self.assertEqual(p, c.port) + + +class RequestBodyTest(TestCase): + """Test cases where a request includes a message body.""" + + def setUp(self): + self.conn = client.HTTPConnection('example.com') + self.conn.sock = self.sock = FakeSocket("") + self.conn.sock = self.sock + + def get_headers_and_fp(self): + f = io.BytesIO(self.sock.data) + f.readline() # read the request line + message = client.parse_headers(f) + return message, f + + def test_list_body(self): + # Note that no content-length is automatically calculated for + # an iterable. The request will fall back to send chunked + # transfer encoding. + cases = ( + ([b'foo', b'bar'], b'3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n'), + ((b'foo', b'bar'), b'3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n'), + ) + for body, expected in cases: + with self.subTest(body): + self.conn = client.HTTPConnection('example.com') + self.conn.sock = self.sock = FakeSocket('') + + self.conn.request('PUT', '/url', body) + msg, f = self.get_headers_and_fp() + self.assertNotIn('Content-Type', msg) + self.assertNotIn('Content-Length', msg) + self.assertEqual(msg.get('Transfer-Encoding'), 'chunked') + self.assertEqual(expected, f.read()) + + def test_manual_content_length(self): + # Set an incorrect content-length so that we can verify that + # it will not be over-ridden by the library. + self.conn.request("PUT", "/url", "body", + {"Content-Length": "42"}) + message, f = self.get_headers_and_fp() + self.assertEqual("42", message.get("content-length")) + self.assertEqual(4, len(f.read())) + + def test_ascii_body(self): + self.conn.request("PUT", "/url", "body") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("4", message.get("content-length")) + self.assertEqual(b'body', f.read()) + + def test_latin1_body(self): + self.conn.request("PUT", "/url", "body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_bytes_body(self): + self.conn.request("PUT", "/url", b"body\xc1") + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("5", message.get("content-length")) + self.assertEqual(b'body\xc1', f.read()) + + def test_text_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as f: + f.write("body") + with open(support.TESTFN) as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + # No content-length will be determined for files; the body + # will be sent using chunked transfer encoding instead. + self.assertIsNone(message.get("content-length")) + self.assertEqual("chunked", message.get("transfer-encoding")) + self.assertEqual(b'4\r\nbody\r\n0\r\n\r\n', f.read()) + + def test_binary_file_body(self): + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "wb") as f: + f.write(b"body\xc1") + with open(support.TESTFN, "rb") as f: + self.conn.request("PUT", "/url", f) + message, f = self.get_headers_and_fp() + self.assertEqual("text/plain", message.get_content_type()) + self.assertIsNone(message.get_charset()) + self.assertEqual("chunked", message.get("Transfer-Encoding")) + self.assertNotIn("Content-Length", message) + self.assertEqual(b'5\r\nbody\xc1\r\n0\r\n\r\n', f.read()) + + +class HTTPResponseTest(TestCase): + + def setUp(self): + body = "HTTP/1.1 200 Ok\r\nMy-Header: first-value\r\nMy-Header: \ + second-value\r\n\r\nText" + sock = FakeSocket(body) + self.resp = client.HTTPResponse(sock) + self.resp.begin() + + def test_getting_header(self): + header = self.resp.getheader('My-Header') + self.assertEqual(header, 'first-value, second-value') + + header = self.resp.getheader('My-Header', 'some default') + self.assertEqual(header, 'first-value, second-value') + + def test_getting_nonexistent_header_with_string_default(self): + header = self.resp.getheader('No-Such-Header', 'default-value') + self.assertEqual(header, 'default-value') + + def test_getting_nonexistent_header_with_iterable_default(self): + header = self.resp.getheader('No-Such-Header', ['default', 'values']) + self.assertEqual(header, 'default, values') + + header = self.resp.getheader('No-Such-Header', ('default', 'values')) + self.assertEqual(header, 'default, values') + + def test_getting_nonexistent_header_without_default(self): + header = self.resp.getheader('No-Such-Header') + self.assertEqual(header, None) + + def test_getting_header_defaultint(self): + header = self.resp.getheader('No-Such-Header',default=42) + self.assertEqual(header, 42) + +class TunnelTests(TestCase): + def setUp(self): + response_text = ( + 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n' # Reply to HEAD + 'Content-Length: 42\r\n\r\n' + ) + self.host = 'proxy.com' + self.conn = client.HTTPConnection(self.host) + self.conn._create_connection = self._create_connection(response_text) + + def tearDown(self): + self.conn.close() + + def _create_connection(self, response_text): + def create_connection(address, timeout=None, source_address=None): + return FakeSocket(response_text, host=address[0], port=address[1]) + return create_connection + + def test_set_tunnel_host_port_headers(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers) + + def test_disallow_set_tunnel_after_connect(self): + # Once connected, we shouldn't be able to tunnel anymore + self.conn.connect() + self.assertRaises(RuntimeError, self.conn.set_tunnel, + 'destination.com') + + def test_connect_with_tunnel(self): + self.conn.set_tunnel('destination.com') + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + # issue22095 + self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + # This test should be removed when CONNECT gets the HTTP/1.1 blessing + self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) + + def test_connect_put_request(self): + self.conn.set_tunnel('destination.com') + self.conn.request('PUT', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertIn(b'CONNECT destination.com', self.conn.sock.data) + self.assertIn(b'Host: destination.com', self.conn.sock.data) + + def test_tunnel_debuglog(self): + expected_header = 'X-Dummy: 1' + response_text = 'HTTP/1.0 200 OK\r\n{}\r\n\r\n'.format(expected_header) + + self.conn.set_debuglevel(1) + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + with support.captured_stdout() as output: + self.conn.request('PUT', '/', '') + lines = output.getvalue().splitlines() + self.assertIn('header: {}'.format(expected_header), lines) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/greentest/3.7/test_select.py b/src/greentest/3.7/test_select.py new file mode 100644 index 0000000..a973f3f --- /dev/null +++ b/src/greentest/3.7/test_select.py @@ -0,0 +1,82 @@ +import errno +import os +import select +import sys +import unittest +from test import support + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectTestCase(unittest.TestCase): + + class Nope: + pass + + class Almost: + def fileno(self): + return 'fileno' + + def test_error_conditions(self): + self.assertRaises(TypeError, select.select, 1, 2, 3) + self.assertRaises(TypeError, select.select, [self.Nope()], [], []) + self.assertRaises(TypeError, select.select, [self.Almost()], [], []) + self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + try: + select.select([fd], [], [], 0) + except OSError as err: + self.assertEqual(err.errno, errno.EBADF) + else: + self.fail("exception not raised") + + def test_returned_list_identity(self): + # See issue #8329 + r, w, x = select.select([], [], [], 1) + self.assertIsNot(r, w) + self.assertIsNot(r, x) + self.assertIsNot(w, x) + + def test_select(self): + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print('timeout =', tout) + rfd, wfd, xfd = select.select([p], [], [], tout) + if (rfd, wfd, xfd) == ([], [], []): + continue + if (rfd, wfd, xfd) == ([p], [], []): + line = p.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select():', rfd, wfd, xfd) + p.close() + + # Issue 16230: Crash on select resized list + def test_select_mutated(self): + a = [] + class F: + def fileno(self): + del a[-1] + return sys.__stdout__.fileno() + a[:] = [F()] * 10 + self.assertEqual(select.select([], a, []), ([], a[:5], [])) + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.7/test_selectors.py b/src/greentest/3.7/test_selectors.py new file mode 100644 index 0000000..3161122 --- /dev/null +++ b/src/greentest/3.7/test_selectors.py @@ -0,0 +1,564 @@ +import errno +import os +import random +import selectors +import signal +import socket +import sys +from test import support +from time import sleep +import unittest +import unittest.mock +import tempfile +from time import monotonic as time +try: + import resource +except ImportError: + resource = None + + +if hasattr(socket, 'socketpair'): + socketpair = socket.socketpair +else: + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): + with socket.socket(family, type, proto) as l: + l.bind((support.HOST, 0)) + l.listen() + c = socket.socket(family, type, proto) + try: + c.connect(l.getsockname()) + caddr = c.getsockname() + while True: + a, addr = l.accept() + # check that we've got the correct client + if addr == caddr: + return c, a + a.close() + except OSError: + c.close() + raise + + +def find_ready_matching(ready, flag): + match = [] + for key, events in ready: + if events & flag: + match.append(key.fileobj) + return match + + +class BaseSelectorTestCase(unittest.TestCase): + + def make_socketpair(self): + rd, wr = socketpair() + self.addCleanup(rd.close) + self.addCleanup(wr.close) + return rd, wr + + def test_register(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIsInstance(key, selectors.SelectorKey) + self.assertEqual(key.fileobj, rd) + self.assertEqual(key.fd, rd.fileno()) + self.assertEqual(key.events, selectors.EVENT_READ) + self.assertEqual(key.data, "data") + + # register an unknown event + self.assertRaises(ValueError, s.register, 0, 999999) + + # register an invalid FD + self.assertRaises(ValueError, s.register, -10, selectors.EVENT_READ) + + # register twice + self.assertRaises(KeyError, s.register, rd, selectors.EVENT_READ) + + # register the same FD, but with a different object + self.assertRaises(KeyError, s.register, rd.fileno(), + selectors.EVENT_READ) + + def test_unregister(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + s.unregister(rd) + + # unregister an unknown file obj + self.assertRaises(KeyError, s.unregister, 999999) + + # unregister twice + self.assertRaises(KeyError, s.unregister, rd) + + def test_unregister_after_fd_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + r, w = rd.fileno(), wr.fileno() + s.register(r, selectors.EVENT_READ) + s.register(w, selectors.EVENT_WRITE) + rd.close() + wr.close() + s.unregister(r) + s.unregister(w) + + @unittest.skipUnless(os.name == 'posix', "requires posix") + def test_unregister_after_fd_close_and_reuse(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + r, w = rd.fileno(), wr.fileno() + s.register(r, selectors.EVENT_READ) + s.register(w, selectors.EVENT_WRITE) + rd2, wr2 = self.make_socketpair() + rd.close() + wr.close() + os.dup2(rd2.fileno(), r) + os.dup2(wr2.fileno(), w) + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + s.unregister(r) + s.unregister(w) + + def test_unregister_after_socket_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + rd.close() + wr.close() + s.unregister(rd) + s.unregister(wr) + + def test_modify(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ) + + # modify events + key2 = s.modify(rd, selectors.EVENT_WRITE) + self.assertNotEqual(key.events, key2.events) + self.assertEqual(key2, s.get_key(rd)) + + s.unregister(rd) + + # modify data + d1 = object() + d2 = object() + + key = s.register(rd, selectors.EVENT_READ, d1) + key2 = s.modify(rd, selectors.EVENT_READ, d2) + self.assertEqual(key.events, key2.events) + self.assertNotEqual(key.data, key2.data) + self.assertEqual(key2, s.get_key(rd)) + self.assertEqual(key2.data, d2) + + # modify unknown file obj + self.assertRaises(KeyError, s.modify, 999999, selectors.EVENT_READ) + + # modify use a shortcut + d3 = object() + s.register = unittest.mock.Mock() + s.unregister = unittest.mock.Mock() + + s.modify(rd, selectors.EVENT_READ, d3) + self.assertFalse(s.register.called) + self.assertFalse(s.unregister.called) + + def test_modify_unregister(self): + # Make sure the fd is unregister()ed in case of error on + # modify(): http://bugs.python.org/issue30014 + if self.SELECTOR.__name__ == 'EpollSelector': + patch = unittest.mock.patch( + 'selectors.EpollSelector._selector_cls') + elif self.SELECTOR.__name__ == 'PollSelector': + patch = unittest.mock.patch( + 'selectors.PollSelector._selector_cls') + elif self.SELECTOR.__name__ == 'DevpollSelector': + patch = unittest.mock.patch( + 'selectors.DevpollSelector._selector_cls') + else: + raise self.skipTest("") + + with patch as m: + m.return_value.modify = unittest.mock.Mock( + side_effect=ZeroDivisionError) + s = self.SELECTOR() + self.addCleanup(s.close) + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + self.assertEqual(len(s._map), 1) + with self.assertRaises(ZeroDivisionError): + s.modify(rd, selectors.EVENT_WRITE) + self.assertEqual(len(s._map), 0) + + def test_close(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + mapping = s.get_map() + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + + s.close() + self.assertRaises(RuntimeError, s.get_key, rd) + self.assertRaises(RuntimeError, s.get_key, wr) + self.assertRaises(KeyError, mapping.__getitem__, rd) + self.assertRaises(KeyError, mapping.__getitem__, wr) + + def test_get_key(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertEqual(key, s.get_key(rd)) + + # unknown file obj + self.assertRaises(KeyError, s.get_key, 999999) + + def test_get_map(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + keys = s.get_map() + self.assertFalse(keys) + self.assertEqual(len(keys), 0) + self.assertEqual(list(keys), []) + key = s.register(rd, selectors.EVENT_READ, "data") + self.assertIn(rd, keys) + self.assertEqual(key, keys[rd]) + self.assertEqual(len(keys), 1) + self.assertEqual(list(keys), [rd.fileno()]) + self.assertEqual(list(keys.values()), [key]) + + # unknown file obj + with self.assertRaises(KeyError): + keys[999999] + + # Read-only mapping + with self.assertRaises(TypeError): + del keys[rd] + + def test_select(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(rd, selectors.EVENT_READ) + wr_key = s.register(wr, selectors.EVENT_WRITE) + + result = s.select() + for key, events in result: + self.assertTrue(isinstance(key, selectors.SelectorKey)) + self.assertTrue(events) + self.assertFalse(events & ~(selectors.EVENT_READ | + selectors.EVENT_WRITE)) + + self.assertEqual([(wr_key, selectors.EVENT_WRITE)], result) + + def test_context_manager(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + with s as sel: + sel.register(rd, selectors.EVENT_READ) + sel.register(wr, selectors.EVENT_WRITE) + + self.assertRaises(RuntimeError, s.get_key, rd) + self.assertRaises(RuntimeError, s.get_key, wr) + + def test_fileno(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + if hasattr(s, 'fileno'): + fd = s.fileno() + self.assertTrue(isinstance(fd, int)) + self.assertGreaterEqual(fd, 0) + + def test_selector(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + NUM_SOCKETS = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_SOCKETS): + rd, wr = self.make_socketpair() + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = s.select() + ready_writers = find_ready_matching(ready, selectors.EVENT_WRITE) + if not ready_writers: + self.fail("no sockets ready for writing") + wr = random.choice(ready_writers) + wr.send(MSG) + + for i in range(10): + ready = s.select() + ready_readers = find_ready_matching(ready, + selectors.EVENT_READ) + if ready_readers: + break + # there might be a delay between the write to the write end and + # the read end is reported ready + sleep(0.1) + else: + self.fail("no sockets ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = rd.recv(MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + s.unregister(r2w[rd]) + s.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_SOCKETS) + + @unittest.skipIf(sys.platform == 'win32', + 'select.select() cannot be used with empty fd sets') + def test_empty_select(self): + # Issue #23009: Make sure EpollSelector.select() works when no FD is + # registered. + s = self.SELECTOR() + self.addCleanup(s.close) + self.assertEqual(s.select(timeout=0), []) + + def test_timeout(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + s.register(wr, selectors.EVENT_WRITE) + t = time() + self.assertEqual(1, len(s.select(0))) + self.assertEqual(1, len(s.select(-1))) + self.assertLess(time() - t, 0.5) + + s.unregister(wr) + s.register(rd, selectors.EVENT_READ) + t = time() + self.assertFalse(s.select(0)) + self.assertFalse(s.select(-1)) + self.assertLess(time() - t, 0.5) + + t0 = time() + self.assertFalse(s.select(1)) + t1 = time() + dt = t1 - t0 + # Tolerate 2.0 seconds for very slow buildbots + self.assertTrue(0.8 <= dt <= 2.0, dt) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_select_interrupt_exc(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + class InterruptSelect(Exception): + pass + + def handler(*args): + raise InterruptSelect + + orig_alrm_handler = signal.signal(signal.SIGALRM, handler) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + + try: + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + # select() is interrupted by a signal which raises an exception + with self.assertRaises(InterruptSelect): + s.select(30) + # select() was interrupted before the timeout of 30 seconds + self.assertLess(time() - t, 5.0) + finally: + signal.alarm(0) + + @unittest.skipUnless(hasattr(signal, "alarm"), + "signal.alarm() required for this test") + def test_select_interrupt_noraise(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + rd, wr = self.make_socketpair() + + orig_alrm_handler = signal.signal(signal.SIGALRM, lambda *args: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + + try: + signal.alarm(1) + + s.register(rd, selectors.EVENT_READ) + t = time() + # select() is interrupted by a signal, but the signal handler doesn't + # raise an exception, so select() should by retries with a recomputed + # timeout + self.assertFalse(s.select(1.5)) + self.assertGreaterEqual(time() - t, 1.0) + finally: + signal.alarm(0) + + +class ScalableSelectorMixIn: + + # see issue #18963 for why it's skipped on older OS X versions + @support.requires_mac_ver(10, 5) + @unittest.skipUnless(resource, "Test needs resource module") + def test_above_fd_setsize(self): + # A scalable implementation should have no problem with more than + # FD_SETSIZE file descriptors. Since we don't know the value, we just + # try to set the soft RLIMIT_NOFILE to the hard RLIMIT_NOFILE ceiling. + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard)) + self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE, + (soft, hard)) + NUM_FDS = min(hard, 2**16) + except (OSError, ValueError): + NUM_FDS = soft + + # guard for already allocated FDs (stdin, stdout...) + NUM_FDS -= 32 + + s = self.SELECTOR() + self.addCleanup(s.close) + + for i in range(NUM_FDS // 2): + try: + rd, wr = self.make_socketpair() + except OSError: + # too many FDs, skip - note that we should only catch EMFILE + # here, but apparently *BSD and Solaris can fail upon connect() + # or bind() with EADDRNOTAVAIL, so let's be safe + self.skipTest("FD limit reached") + + try: + s.register(rd, selectors.EVENT_READ) + s.register(wr, selectors.EVENT_WRITE) + except OSError as e: + if e.errno == errno.ENOSPC: + # this can be raised by epoll if we go over + # fs.epoll.max_user_watches sysctl + self.skipTest("FD limit reached") + raise + + try: + fds = s.select() + except OSError as e: + if e.errno == errno.EINVAL and sys.platform == 'darwin': + # unexplainable errors on macOS don't need to fail the test + self.skipTest("Invalid argument error calling poll()") + raise + self.assertEqual(NUM_FDS // 2, len(fds)) + + +class DefaultSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.DefaultSelector + + +class SelectSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = selectors.SelectSelector + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class PollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'PollSelector', None) + + +@unittest.skipUnless(hasattr(selectors, 'EpollSelector'), + "Test needs selectors.EpollSelector") +class EpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'EpollSelector', None) + + def test_register_file(self): + # epoll(7) returns EPERM when given a file to watch + s = self.SELECTOR() + with tempfile.NamedTemporaryFile() as f: + with self.assertRaises(IOError): + s.register(f, selectors.EVENT_READ) + # the SelectorKey has been removed + with self.assertRaises(KeyError): + s.get_key(f) + + +@unittest.skipUnless(hasattr(selectors, 'KqueueSelector'), + "Test needs selectors.KqueueSelector)") +class KqueueSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'KqueueSelector', None) + + def test_register_bad_fd(self): + # a file descriptor that's been closed should raise an OSError + # with EBADF + s = self.SELECTOR() + bad_f = support.make_bad_fd() + with self.assertRaises(OSError) as cm: + s.register(bad_f, selectors.EVENT_READ) + self.assertEqual(cm.exception.errno, errno.EBADF) + # the SelectorKey has been removed + with self.assertRaises(KeyError): + s.get_key(bad_f) + + +@unittest.skipUnless(hasattr(selectors, 'DevpollSelector'), + "Test needs selectors.DevpollSelector") +class DevpollSelectorTestCase(BaseSelectorTestCase, ScalableSelectorMixIn): + + SELECTOR = getattr(selectors, 'DevpollSelector', None) + + + +def test_main(): + tests = [DefaultSelectorTestCase, SelectSelectorTestCase, + PollSelectorTestCase, EpollSelectorTestCase, + KqueueSelectorTestCase, DevpollSelectorTestCase] + support.run_unittest(*tests) + support.reap_children() + + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.7/test_smtpd.py b/src/greentest/3.7/test_smtpd.py new file mode 100644 index 0000000..a9f7d5a --- /dev/null +++ b/src/greentest/3.7/test_smtpd.py @@ -0,0 +1,1013 @@ +import unittest +import textwrap +from test import support, mock_socket +import socket +import io +import smtpd +import asyncore + + +class DummyServer(smtpd.SMTPServer): + def __init__(self, *args, **kwargs): + smtpd.SMTPServer.__init__(self, *args, **kwargs) + self.messages = [] + if self._decode_data: + self.return_status = 'return status' + else: + self.return_status = b'return status' + + def process_message(self, peer, mailfrom, rcpttos, data, **kw): + self.messages.append((peer, mailfrom, rcpttos, data)) + if data == self.return_status: + return '250 Okish' + if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']: + return '250 SMTPUTF8 message okish' + + +class DummyDispatcherBroken(Exception): + pass + + +class BrokenDummyServer(DummyServer): + def listen(self, num): + raise DummyDispatcherBroken() + + +class SMTPDServerTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def test_process_message_unimplemented(self): + server = smtpd.SMTPServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + + write_line(b'HELO example') + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') + + def test_decode_data_and_enable_SMTPUTF8_raises(self): + self.assertRaises( + ValueError, + smtpd.SMTPServer, + (support.HOST, 0), + ('b', 0), + enable_SMTPUTF8=True, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class DebuggingServerTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def send_data(self, channel, data, enable_SMTPUTF8=False): + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + write_line(b'EHLO example') + if enable_SMTPUTF8: + write_line(b'MAIL From:eggs@example BODY=8BITMIME SMTPUTF8') + else: + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + write_line(data) + write_line(b'.') + + def test_process_message_with_decode_data_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nhello\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + From: test + X-Peer: peer-address + + hello + ------------ END MESSAGE ------------ + """)) + + def test_process_message_with_decode_data_false(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def test_process_message_with_enable_SMTPUTF8_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def test_process_SMTPUTF8_message_with_enable_SMTPUTF8_true(self): + server = smtpd.DebuggingServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n', + enable_SMTPUTF8=True) + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + mail options: ['BODY=8BITMIME', 'SMTPUTF8'] + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class TestFamilyDetection(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + @unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") + def test_socket_uses_IPv6(self): + server = smtpd.SMTPServer((support.HOSTv6, 0), (support.HOSTv4, 0)) + self.assertEqual(server.socket.family, socket.AF_INET6) + + def test_socket_uses_IPv4(self): + server = smtpd.SMTPServer((support.HOSTv4, 0), (support.HOSTv6, 0)) + self.assertEqual(server.socket.family, socket.AF_INET) + + +class TestRcptOptionParsing(unittest.TestCase): + error_response = (b'555 RCPT TO parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_params_rejected(self): + server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: foo=bar') + self.assertEqual(channel.socket.last, self.error_response) + + def test_nothing_accepted(self): + server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: ') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class TestMailOptionParsing(unittest.TestCase): + error_response = (b'555 MAIL FROM parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_with_decode_data_true(self): + server = DummyServer((support.HOST, 0), ('b', 0), decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + b'MAIL from: size=20 BODY=UNKNOWN', + b'MAIL from: size=20 body=8bitmime', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line(channel, b'MAIL from: size=20') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_decode_data_false(self): + server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line( + channel, + b'MAIL from: size=20 SMTPUTF8 BODY=UNKNOWN') + self.assertEqual( + channel.socket.last, + b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n') + self.write_line( + channel, b'MAIL from: size=20 body=8bitmime') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_enable_smtputf8_true(self): + server = DummyServer((support.HOST, 0), ('b', 0), enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + self.write_line(channel, b'EHLO example') + self.write_line( + channel, + b'MAIL from: size=20 body=8bitmime smtputf8') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class SMTPDChannelTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_broken_connect(self): + self.assertRaises( + DummyDispatcherBroken, BrokenDummyServer, + (support.HOST, 0), ('b', 0), decode_data=True) + + def test_decode_data_and_enable_SMTPUTF8_raises(self): + self.assertRaises( + ValueError, smtpd.SMTPChannel, + self.server, self.channel.conn, self.channel.addr, + enable_SMTPUTF8=True, decode_data=True) + + def test_server_accept(self): + self.server.handle_accept() + + def test_missing_data(self): + self.write_line(b'') + self.assertEqual(self.channel.socket.last, + b'500 Error: bad syntax\r\n') + + def test_EHLO(self): + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, b'250 HELP\r\n') + + def test_EHLO_bad_syntax(self): + self.write_line(b'EHLO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: EHLO hostname\r\n') + + def test_EHLO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_EHLO_HELO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO(self): + name = smtpd.socket.getfqdn() + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + '250 {}\r\n'.format(name).encode('ascii')) + + def test_HELO_EHLO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELP(self): + self.write_line(b'HELP') + self.assertEqual(self.channel.socket.last, + b'250 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELP_command(self): + self.write_line(b'HELP MAIL') + self.assertEqual(self.channel.socket.last, + b'250 Syntax: MAIL FROM:
\r\n') + + def test_HELP_command_unknown(self): + self.write_line(b'HELP SPAM') + self.assertEqual(self.channel.socket.last, + b'501 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELO_bad_syntax(self): + self.write_line(b'HELO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: HELO hostname\r\n') + + def test_HELO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO_parameter_rejected_when_extensions_not_enabled(self): + self.extended_smtp = False + self.write_line(b'HELO example') + self.write_line(b'MAIL from: SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_allows_space_after_colon(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_extended_MAIL_allows_space_after_colon(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: size=20') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_NOOP(self): + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_HELO_NOOP(self): + self.write_line(b'HELO example') + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_NOOP_bad_syntax(self): + self.write_line(b'NOOP hi') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: NOOP\r\n') + + def test_QUIT(self): + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_HELO_QUIT(self): + self.write_line(b'HELO example') + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_QUIT_arg_ignored(self): + self.write_line(b'QUIT bye bye') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_bad_state(self): + self.channel.smtp_state = 'BAD STATE' + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'451 Internal confusion\r\n') + + def test_command_too_long(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ' + + b'a' * self.channel.command_size_limit + + b'@example') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_limit_extended_with_SIZE(self): + self.write_line(b'EHLO example') + fill_len = self.channel.command_size_limit - len('MAIL from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 26) + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_rejects_SMTPUTF8_by_default(self): + self.write_line(b'EHLO example') + self.write_line( + b'MAIL from: BODY=8BITMIME SMTPUTF8') + self.assertEqual(self.channel.socket.last[0:1], b'5') + + def test_data_longer_than_default_data_size_limit(self): + # Hack the default so we don't have to generate so much data. + self.channel.data_size_limit = 1048 + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'A' * self.channel.data_size_limit + + b'A\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + def test_MAIL_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=512') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_MAIL_invalid_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=invalid') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_RCPT_unknown_parameters(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 MAIL FROM parameters not recognized or not implemented\r\n') + + self.write_line(b'MAIL FROM:') + self.write_line(b'RCPT TO: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 RCPT TO parameters not recognized or not implemented\r\n') + + def test_MAIL_size_parameter_larger_than_default_data_size_limit(self): + self.channel.data_size_limit = 1048 + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=2096') + self.assertEqual(self.channel.socket.last, + b'552 Error: message size exceeds fixed maximum message size\r\n') + + def test_need_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'RCPT to:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: need MAIL command\r\n') + + def test_MAIL_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_missing_address(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_chevrons(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_empty_chevrons(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from:<>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_quoted_localpart(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_nested_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:eggs@example') + self.write_line(b'MAIL from:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: nested MAIL command\r\n') + + def test_VRFY(self): + self.write_line(b'VRFY eggs@example') + self.assertEqual(self.channel.socket.last, + b'252 Cannot VRFY user, but will accept message and attempt ' + \ + b'delivery\r\n') + + def test_VRFY_syntax(self): + self.write_line(b'VRFY') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: VRFY
\r\n') + + def test_EXPN_not_implemented(self): + self.write_line(b'EXPN') + self.assertEqual(self.channel.socket.last, + b'502 EXPN not implemented\r\n') + + def test_no_HELO_MAIL(self): + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_need_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'503 Error: need RCPT command\r\n') + + def test_RCPT_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
\r\n') + + def test_RCPT_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
[SP ]\r\n') + + def test_RCPT_lowercase_to_OK(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to: ') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_no_HELO_RCPT(self): + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example'], + 'data\nmore')]) + + def test_DATA_syntax(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n') + + def test_no_HELO_DATA(self): + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_transparency_section_4_5_2(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'..\r\n.\r\n') + self.assertEqual(self.channel.received_data, '.') + + def test_multiple_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RCPT To:ham@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example','ham@example'], + 'data')]) + + def test_manual_status(self): + # checks that the Channel is able to return a custom status message + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'return status\r\n.') + self.assertEqual(self.channel.socket.last, b'250 Okish\r\n') + + def test_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'MAIL From:foo@example') + self.write_line(b'RCPT To:eggs@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'foo@example', + ['eggs@example'], + 'data')]) + + def test_HELO_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_RSET_syntax(self): + self.write_line(b'RSET hi') + self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n') + + def test_unknown_command(self): + self.write_line(b'UNKNOWN_CMD') + self.assertEqual(self.channel.socket.last, + b'500 Error: command "UNKNOWN_CMD" not ' + \ + b'recognized\r\n') + + def test_attribute_deprecations(self): + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__server + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__server = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__line + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__line = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__state + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__state = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__greeting + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__greeting = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__mailfrom + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__mailfrom = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__rcpttos + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__rcpttos = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__data + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__data = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__fqdn + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__fqdn = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__peer + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__peer = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__conn + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__conn = 'spam' + with support.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__addr + with support.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__addr = 'spam' + +@unittest.skipUnless(support.IPV6_ENABLED, "IPv6 not enabled") +class SMTPDChannelIPv6Test(SMTPDChannelTest): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOSTv6, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + +class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set DATA size limit to 32 bytes for easy testing + self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_data_limit_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example'], + 'data\nmore')]) + + def test_data_limit_dialog_too_much_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'This message is longer than 32 bytes\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + +class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0)) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, b'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87\n' + b'and some plain ascii') + + +class SMTPDChannelWithDecodeDataTrue(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set decode_data to True + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, 'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + 'utf8 enriched text: żźć\nand some plain ascii') + + +class SMTPDChannelTestWithEnableSMTPUTF8True(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((support.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + enable_SMTPUTF8=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_MAIL_command_accepts_SMTPUTF8_when_announced(self): + self.write_line(b'EHLO example') + self.write_line( + 'MAIL from: BODY=8BITMIME SMTPUTF8'.encode( + 'utf-8') + ) + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_process_smtputf8_message(self): + self.write_line(b'EHLO example') + for mail_parameters in [b'', b'BODY=8BITMIME SMTPUTF8']: + self.write_line(b'MAIL from: ' + mail_parameters) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'rcpt to:') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'data') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'c\r\n.') + if mail_parameters == b'': + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + else: + self.assertEqual(self.channel.socket.last, + b'250 SMTPUTF8 message okish\r\n') + + def test_utf8_data(self): + self.write_line(b'EHLO example') + self.write_line( + 'MAIL From: naïve@examplé BODY=8BITMIME SMTPUTF8'.encode('utf-8')) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line('RCPT To:späm@examplé'.encode('utf-8')) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + + def test_MAIL_command_limit_extended_with_SIZE_and_SMTPUTF8(self): + self.write_line(b'ehlo example') + fill_len = (512 + 26 + 10) - len('mail from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 1) + + b'@example>') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_multiple_emails_with_extended_command_length(self): + self.write_line(b'ehlo example') + fill_len = (512 + 26 + 10) - len('mail from:<@example>') + for char in [b'a', b'b', b'c']: + self.write_line(b'MAIL from:<' + char * fill_len + b'a@example>') + self.assertEqual(self.channel.socket.last[0:3], b'500') + self.write_line(b'MAIL from:<' + char * fill_len + b'@example>') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'rcpt to:') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'data') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'test\r\n.') + self.assertEqual(self.channel.socket.last[0:3], b'250') + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = { + "program", "Devnull", "DEBUGSTREAM", "NEWLINE", "COMMASPACE", + "DATA_SIZE_DEFAULT", "usage", "Options", "parseargs", + + } + support.check__all__(self, smtpd, blacklist=blacklist) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.7/test_socket.py b/src/greentest/3.7/test_socket.py new file mode 100644 index 0000000..f377ebc --- /dev/null +++ b/src/greentest/3.7/test_socket.py @@ -0,0 +1,6036 @@ +import unittest +from test import support + +import errno +import io +import itertools +import socket +import select +import tempfile +import time +import traceback +import queue +import sys +import os +import array +import contextlib +from weakref import proxy +import signal +import math +import pickle +import struct +import random +import shutil +import string +import _thread as thread +import threading +try: + import multiprocessing +except ImportError: + multiprocessing = False +try: + import fcntl +except ImportError: + fcntl = None + +HOST = support.HOST +MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return + +VSOCKPORT = 1234 + +try: + import _socket +except ImportError: + _socket = None + +def get_cid(): + if fcntl is None: + return None + try: + with open("/dev/vsock", "rb") as f: + r = fcntl.ioctl(f, socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID, " ") + except OSError: + return None + else: + return struct.unpack("I", r)[0] + +def _have_socket_can(): + """Check whether CAN sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_can_isotp(): + """Check whether CAN ISOTP sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_rds(): + """Check whether RDS sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_alg(): + """Check whether AF_ALG sockets are supported on this host.""" + try: + s = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +def _have_socket_vsock(): + """Check whether AF_VSOCK sockets are supported on this host.""" + ret = get_cid() is not None + return ret + + +def _is_fd_in_blocking_mode(sock): + return not bool( + fcntl.fcntl(sock, fcntl.F_GETFL, os.O_NONBLOCK) & os.O_NONBLOCK) + + +HAVE_SOCKET_CAN = _have_socket_can() + +HAVE_SOCKET_CAN_ISOTP = _have_socket_can_isotp() + +HAVE_SOCKET_RDS = _have_socket_rds() + +HAVE_SOCKET_ALG = _have_socket_alg() + +HAVE_SOCKET_VSOCK = _have_socket_vsock() + +# Size in bytes of the int type +SIZEOF_INT = array.array("i").itemsize + +class SocketTCPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(self.serv) + self.serv.listen() + + def tearDown(self): + self.serv.close() + self.serv = None + +class SocketUDPTest(unittest.TestCase): + + def setUp(self): + self.serv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.port = support.bind_port(self.serv) + + def tearDown(self): + self.serv.close() + self.serv = None + +class ThreadSafeCleanupTestCase(unittest.TestCase): + """Subclass of unittest.TestCase with thread-safe cleanup methods. + + This subclass protects the addCleanup() and doCleanups() methods + with a recursive lock. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cleanup_lock = threading.RLock() + + def addCleanup(self, *args, **kwargs): + with self._cleanup_lock: + return super().addCleanup(*args, **kwargs) + + def doCleanups(self, *args, **kwargs): + with self._cleanup_lock: + return super().doCleanups(*args, **kwargs) + +class SocketCANTest(unittest.TestCase): + + """To be able to run this test, a `vcan0` CAN interface can be created with + the following commands: + # modprobe vcan + # ip link add dev vcan0 type vcan + # ifconfig vcan0 up + """ + interface = 'vcan0' + bufsize = 128 + + """The CAN frame structure is defined in : + + struct can_frame { + canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ + __u8 can_dlc; /* data length code: 0 .. 8 */ + __u8 data[8] __attribute__((aligned(8))); + }; + """ + can_frame_fmt = "=IB3x8s" + can_frame_size = struct.calcsize(can_frame_fmt) + + """The Broadcast Management Command frame structure is defined + in : + + struct bcm_msg_head { + __u32 opcode; + __u32 flags; + __u32 count; + struct timeval ival1, ival2; + canid_t can_id; + __u32 nframes; + struct can_frame frames[0]; + } + + `bcm_msg_head` must be 8 bytes aligned because of the `frames` member (see + `struct can_frame` definition). Must use native not standard types for packing. + """ + bcm_cmd_msg_fmt = "@3I4l2I" + bcm_cmd_msg_fmt += "x" * (struct.calcsize(bcm_cmd_msg_fmt) % 8) + + def setUp(self): + self.s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + self.addCleanup(self.s.close) + try: + self.s.bind((self.interface,)) + except OSError: + self.skipTest('network interface `%s` does not exist' % + self.interface) + + +class SocketRDSTest(unittest.TestCase): + + """To be able to run this test, the `rds` kernel module must be loaded: + # modprobe rds + """ + bufsize = 8192 + + def setUp(self): + self.serv = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + self.addCleanup(self.serv.close) + try: + self.port = support.bind_port(self.serv) + except OSError: + self.skipTest('unable to bind RDS socket') + + +class ThreadableTest: + """Threadable Test class + + The ThreadableTest class makes it easy to create a threaded + client/server pair from an existing unit test. To create a + new threaded class from an existing unit test, use multiple + inheritance: + + class NewClass (OldClass, ThreadableTest): + pass + + This class defines two new fixture functions with obvious + purposes for overriding: + + clientSetUp () + clientTearDown () + + Any new test functions within the class must then define + tests in pairs, where the test name is preceded with a + '_' to indicate the client portion of the test. Ex: + + def testFoo(self): + # Server portion + + def _testFoo(self): + # Client portion + + Any exceptions raised by the clients during their tests + are caught and transferred to the main thread to alert + the testing framework. + + Note, the server setup function cannot call any blocking + functions that rely on the client thread during setup, + unless serverExplicitReady() is called just before + the blocking call (such as in setting up a client/server + connection and performing the accept() in setUp(). + """ + + def __init__(self): + # Swap the true setup function + self.__setUp = self.setUp + self.__tearDown = self.tearDown + self.setUp = self._setUp + self.tearDown = self._tearDown + + def serverExplicitReady(self): + """This method allows the server to explicitly indicate that + it wants the client thread to proceed. This is useful if the + server is about to execute a blocking routine that is + dependent upon the client thread during its setup routine.""" + self.server_ready.set() + + def _setUp(self): + self.wait_threads = support.wait_threads_exit() + self.wait_threads.__enter__() + + self.server_ready = threading.Event() + self.client_ready = threading.Event() + self.done = threading.Event() + self.queue = queue.Queue(1) + self.server_crashed = False + + # Do some munging to start the client test. + methodname = self.id() + i = methodname.rfind('.') + methodname = methodname[i+1:] + test_method = getattr(self, '_' + methodname) + self.client_thread = thread.start_new_thread( + self.clientRun, (test_method,)) + + try: + self.__setUp() + except: + self.server_crashed = True + raise + finally: + self.server_ready.set() + self.client_ready.wait() + + def _tearDown(self): + self.__tearDown() + self.done.wait() + self.wait_threads.__exit__(None, None, None) + + if self.queue.qsize(): + exc = self.queue.get() + raise exc + + def clientRun(self, test_func): + self.server_ready.wait() + try: + self.clientSetUp() + except BaseException as e: + self.queue.put(e) + self.clientTearDown() + return + finally: + self.client_ready.set() + if self.server_crashed: + self.clientTearDown() + return + if not hasattr(test_func, '__call__'): + raise TypeError("test_func must be a callable function") + try: + test_func() + except BaseException as e: + self.queue.put(e) + finally: + self.clientTearDown() + + def clientSetUp(self): + raise NotImplementedError("clientSetUp must be implemented.") + + def clientTearDown(self): + self.done.set() + thread.exit() + +class ThreadedTCPSocketTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketUDPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedCANSocketTest(SocketCANTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketCANTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + try: + self.cli.bind((self.interface,)) + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedRDSSocketTest(SocketRDSTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketRDSTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + try: + # RDS sockets must be bound explicitly to send or receive data + self.cli.bind((HOST, 0)) + self.cli_addr = self.cli.getsockname() + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +@unittest.skipIf(fcntl is None, "need fcntl") +@unittest.skipUnless(HAVE_SOCKET_VSOCK, + 'VSOCK sockets required for this test.') +@unittest.skipUnless(get_cid() != 2, + "This test can only be run on a virtual guest.") +class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): + + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.serv = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) + self.addCleanup(self.serv.close) + self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) + self.serv.listen() + self.serverExplicitReady() + self.conn, self.connaddr = self.serv.accept() + self.addCleanup(self.conn.close) + + def clientSetUp(self): + time.sleep(0.1) + self.cli = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) + self.addCleanup(self.cli.close) + cid = get_cid() + self.cli.connect((cid, VSOCKPORT)) + + def testStream(self): + msg = self.conn.recv(1024) + self.assertEqual(msg, MSG) + + def _testStream(self): + self.cli.send(MSG) + self.cli.close() + +class SocketConnectedTest(ThreadedTCPSocketTest): + """Socket tests for client-server connection. + + self.cli_conn is a client socket connected to the server. The + setUp() method guarantees that it is connected to the server. + """ + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + ThreadedTCPSocketTest.setUp(self) + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + ThreadedTCPSocketTest.tearDown(self) + + def clientSetUp(self): + ThreadedTCPSocketTest.clientSetUp(self) + self.cli.connect((HOST, self.port)) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + ThreadedTCPSocketTest.clientTearDown(self) + +class SocketPairTest(unittest.TestCase, ThreadableTest): + + def __init__(self, methodName='runTest'): + unittest.TestCase.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.serv, self.cli = socket.socketpair() + + def tearDown(self): + self.serv.close() + self.serv = None + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +# The following classes are used by the sendmsg()/recvmsg() tests. +# Combining, for instance, ConnectedStreamTestMixin and TCPTestBase +# gives a drop-in replacement for SocketConnectedTest, but different +# address families can be used, and the attributes serv_addr and +# cli_addr will be set to the addresses of the endpoints. + +class SocketTestBase(unittest.TestCase): + """A base class for socket tests. + + Subclasses must provide methods newSocket() to return a new socket + and bindSock(sock) to bind it to an unused address. + + Creates a socket self.serv and sets self.serv_addr to its address. + """ + + def setUp(self): + self.serv = self.newSocket() + self.bindServer() + + def bindServer(self): + """Bind server socket and set self.serv_addr to its address.""" + self.bindSock(self.serv) + self.serv_addr = self.serv.getsockname() + + def tearDown(self): + self.serv.close() + self.serv = None + + +class SocketListeningTestMixin(SocketTestBase): + """Mixin to listen on the server socket.""" + + def setUp(self): + super().setUp() + self.serv.listen() + + +class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, + ThreadableTest): + """Mixin to add client socket and allow client/server tests. + + Client socket is self.cli and its address is self.cli_addr. See + ThreadableTest for usage information. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = self.newClientSocket() + self.bindClient() + + def newClientSocket(self): + """Return a new socket for use as client.""" + return self.newSocket() + + def bindClient(self): + """Bind client socket and set self.cli_addr to its address.""" + self.bindSock(self.cli) + self.cli_addr = self.cli.getsockname() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +class ConnectedStreamTestMixin(SocketListeningTestMixin, + ThreadedSocketTestMixin): + """Mixin to allow client/server stream tests with connected client. + + Server's socket representing connection to client is self.cli_conn + and client's connection to server is self.serv_conn. (Based on + SocketConnectedTest.) + """ + + def setUp(self): + super().setUp() + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + super().tearDown() + + def clientSetUp(self): + super().clientSetUp() + self.cli.connect(self.serv_addr) + self.serv_conn = self.cli + + def clientTearDown(self): + try: + self.serv_conn.close() + self.serv_conn = None + except AttributeError: + pass + super().clientTearDown() + + +class UnixSocketTestBase(SocketTestBase): + """Base class for Unix-domain socket tests.""" + + # This class is used for file descriptor passing tests, so we + # create the sockets in a private directory so that other users + # can't send anything that might be problematic for a privileged + # user running the tests. + + def setUp(self): + self.dir_path = tempfile.mkdtemp() + self.addCleanup(os.rmdir, self.dir_path) + super().setUp() + + def bindSock(self, sock): + path = tempfile.mktemp(dir=self.dir_path) + support.bind_unix_socket(sock, path) + self.addCleanup(support.unlink, path) + +class UnixStreamBase(UnixSocketTestBase): + """Base class for Unix-domain SOCK_STREAM tests.""" + + def newSocket(self): + return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + +class InetTestBase(SocketTestBase): + """Base class for IPv4 socket tests.""" + + host = HOST + + def setUp(self): + super().setUp() + self.port = self.serv_addr[1] + + def bindSock(self, sock): + support.bind_port(sock, host=self.host) + +class TCPTestBase(InetTestBase): + """Base class for TCP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + +class UDPTestBase(InetTestBase): + """Base class for UDP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + +class SCTPStreamBase(InetTestBase): + """Base class for SCTP tests in one-to-one (SOCK_STREAM) mode.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM, + socket.IPPROTO_SCTP) + + +class Inet6TestBase(InetTestBase): + """Base class for IPv6 socket tests.""" + + host = support.HOSTv6 + +class UDP6TestBase(Inet6TestBase): + """Base class for UDP-over-IPv6 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + + +# Test-skipping decorators for use with ThreadableTest. + +def skipWithClientIf(condition, reason): + """Skip decorated test if condition is true, add client_skip decorator. + + If the decorated object is not a class, sets its attribute + "client_skip" to a decorator which will return an empty function + if the test is to be skipped, or the original function if it is + not. This can be used to avoid running the client part of a + skipped test when using ThreadableTest. + """ + def client_pass(*args, **kwargs): + pass + def skipdec(obj): + retval = unittest.skip(reason)(obj) + if not isinstance(obj, type): + retval.client_skip = lambda f: client_pass + return retval + def noskipdec(obj): + if not (isinstance(obj, type) or hasattr(obj, "client_skip")): + obj.client_skip = lambda f: f + return obj + return skipdec if condition else noskipdec + + +def requireAttrs(obj, *attributes): + """Skip decorated test if obj is missing any of the given attributes. + + Sets client_skip attribute as skipWithClientIf() does. + """ + missing = [name for name in attributes if not hasattr(obj, name)] + return skipWithClientIf( + missing, "don't have " + ", ".join(name for name in missing)) + + +def requireSocket(*args): + """Skip decorated test if a socket cannot be created with given arguments. + + When an argument is given as a string, will use the value of that + attribute of the socket module, or skip the test if it doesn't + exist. Sets client_skip attribute as skipWithClientIf() does. + """ + err = None + missing = [obj for obj in args if + isinstance(obj, str) and not hasattr(socket, obj)] + if missing: + err = "don't have " + ", ".join(name for name in missing) + else: + callargs = [getattr(socket, obj) if isinstance(obj, str) else obj + for obj in args] + try: + s = socket.socket(*callargs) + except OSError as e: + # XXX: check errno? + err = str(e) + else: + s.close() + return skipWithClientIf( + err is not None, + "can't create socket({0}): {1}".format( + ", ".join(str(o) for o in args), err)) + + +####################################################################### +## Begin Tests + +class GeneralModuleTests(unittest.TestCase): + + def test_SocketType_is_socketobject(self): + import _socket + self.assertTrue(socket.SocketType is _socket.socket) + s = socket.socket() + self.assertIsInstance(s, socket.SocketType) + s.close() + + def test_repr(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with s: + self.assertIn('fd=%i' % s.fileno(), repr(s)) + self.assertIn('family=%s' % socket.AF_INET, repr(s)) + self.assertIn('type=%s' % socket.SOCK_STREAM, repr(s)) + self.assertIn('proto=0', repr(s)) + self.assertNotIn('raddr', repr(s)) + s.bind(('127.0.0.1', 0)) + self.assertIn('laddr', repr(s)) + self.assertIn(str(s.getsockname()), repr(s)) + self.assertIn('[closed]', repr(s)) + self.assertNotIn('laddr', repr(s)) + + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_csocket_repr(self): + s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) + try: + expected = ('' + % (s.fileno(), s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + finally: + s.close() + expected = ('' + % (s.family, s.type, s.proto)) + self.assertEqual(repr(s), expected) + + def test_weakref(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + p = proxy(s) + self.assertEqual(p.fileno(), s.fileno()) + s.close() + s = None + try: + p.fileno() + except ReferenceError: + pass + else: + self.fail('Socket proxy still exists') + + def testSocketError(self): + # Testing socket module exceptions + msg = "Error raising socket exception (%s)." + with self.assertRaises(OSError, msg=msg % 'OSError'): + raise OSError + with self.assertRaises(OSError, msg=msg % 'socket.herror'): + raise socket.herror + with self.assertRaises(OSError, msg=msg % 'socket.gaierror'): + raise socket.gaierror + + def testSendtoErrors(self): + # Testing that sendto doesn't mask failures. See #10169. + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind(('', 0)) + sockname = s.getsockname() + # 2 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'str'") + with self.assertRaises(TypeError) as cm: + s.sendto(5j, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'complex'") + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None) + self.assertIn('not NoneType',str(cm.exception)) + # 3 args + with self.assertRaises(TypeError) as cm: + s.sendto('\u2620', 0, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'str'") + with self.assertRaises(TypeError) as cm: + s.sendto(5j, 0, sockname) + self.assertEqual(str(cm.exception), + "a bytes-like object is required, not 'complex'") + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, None) + self.assertIn('not NoneType', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 'bar', sockname) + self.assertIn('an integer is required', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', None, None) + self.assertIn('an integer is required', str(cm.exception)) + # wrong number of args + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo') + self.assertIn('(1 given)', str(cm.exception)) + with self.assertRaises(TypeError) as cm: + s.sendto(b'foo', 0, sockname, 4) + self.assertIn('(4 given)', str(cm.exception)) + + def testCrucialConstants(self): + # Testing for mission critical constants + socket.AF_INET + socket.SOCK_STREAM + socket.SOCK_DGRAM + socket.SOCK_RAW + socket.SOCK_RDM + socket.SOCK_SEQPACKET + socket.SOL_SOCKET + socket.SO_REUSEADDR + + def testHostnameRes(self): + # Testing hostname resolution mechanisms + hostname = socket.gethostname() + try: + ip = socket.gethostbyname(hostname) + except OSError: + # Probably name lookup wasn't set up right; skip this test + self.skipTest('name lookup failure') + self.assertTrue(ip.find('.') >= 0, "Error resolving host to ip.") + try: + hname, aliases, ipaddrs = socket.gethostbyaddr(ip) + except OSError: + # Probably a similar problem as above; skip this test + self.skipTest('name lookup failure') + all_host_names = [hostname, hname] + aliases + fqhn = socket.getfqdn(ip) + if not fqhn in all_host_names: + self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + + def test_host_resolution(self): + for addr in [support.HOSTv4, '10.0.0.1', '255.255.255.255']: + self.assertEqual(socket.gethostbyname(addr), addr) + + # we don't test support.HOSTv6 because there's a chance it doesn't have + # a matching name entry (e.g. 'ip6-localhost') + for host in [support.HOSTv4]: + self.assertIn(host, socket.gethostbyaddr(host)[2]) + + def test_host_resolution_bad_address(self): + # These are all malformed IP addresses and expected not to resolve to + # any result. But some ISPs, e.g. AWS, may successfully resolve these + # IPs. + explanation = ( + "resolving an invalid IP address did not raise OSError; " + "can be caused by a broken DNS server" + ) + for addr in ['0.1.1.~1', '1+.1.1.1', '::1q', '::1::2', + '1:1:1:1:1:1:1:1:1']: + with self.assertRaises(OSError, msg=addr): + socket.gethostbyname(addr) + with self.assertRaises(OSError, msg=explanation): + socket.gethostbyaddr(addr) + + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") + @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") + def test_sethostname(self): + oldhn = socket.gethostname() + try: + socket.sethostname('new') + except OSError as e: + if e.errno == errno.EPERM: + self.skipTest("test should be run as root") + else: + raise + try: + # running test as root! + self.assertEqual(socket.gethostname(), 'new') + # Should work with bytes objects too + socket.sethostname(b'bar') + self.assertEqual(socket.gethostname(), 'bar') + finally: + socket.sethostname(oldhn) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInterfaceNameIndex(self): + interfaces = socket.if_nameindex() + for index, name in interfaces: + self.assertIsInstance(index, int) + self.assertIsInstance(name, str) + # interface indices are non-zero integers + self.assertGreater(index, 0) + _index = socket.if_nametoindex(name) + self.assertIsInstance(_index, int) + self.assertEqual(index, _index) + _name = socket.if_indextoname(index) + self.assertIsInstance(_name, str) + self.assertEqual(name, _name) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInvalidInterfaceNameIndex(self): + # test nonexistent interface index/name + self.assertRaises(OSError, socket.if_indextoname, 0) + self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') + # test with invalid values + self.assertRaises(TypeError, socket.if_nametoindex, 0) + self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') + def testRefCountGetNameInfo(self): + # Testing reference count for getnameinfo + try: + # On some versions, this loses a reference + orig = sys.getrefcount(__name__) + socket.getnameinfo(__name__,0) + except TypeError: + if sys.getrefcount(__name__) != orig: + self.fail("socket.getnameinfo loses a reference") + + def testInterpreterCrash(self): + # Making sure getnameinfo doesn't crash the interpreter + try: + # On some versions, this crashes the interpreter. + socket.getnameinfo(('x', 0, 0, 0), 0) + except OSError: + pass + + def testNtoH(self): + # This just checks that htons etc. are their own inverse, + # when looking at the lower 16 or 32 bits. + sizes = {socket.htonl: 32, socket.ntohl: 32, + socket.htons: 16, socket.ntohs: 16} + for func, size in sizes.items(): + mask = (1<= 23): + port2 = socket.getservbyname(service) + eq(port, port2) + # Try udp, but don't barf if it doesn't exist + try: + udpport = socket.getservbyname(service, 'udp') + except OSError: + udpport = None + else: + eq(udpport, port) + # Now make sure the lookup by port returns the same service name + # Issue #26936: Android getservbyport() is broken. + if not support.is_android: + eq(socket.getservbyport(port2), service) + eq(socket.getservbyport(port, 'tcp'), service) + if udpport is not None: + eq(socket.getservbyport(udpport, 'udp'), service) + # Make sure getservbyport does not accept out of range ports. + self.assertRaises(OverflowError, socket.getservbyport, -1) + self.assertRaises(OverflowError, socket.getservbyport, 65536) + + def testDefaultTimeout(self): + # Testing default timeout + # The default timeout should initially be None + self.assertEqual(socket.getdefaulttimeout(), None) + s = socket.socket() + self.assertEqual(s.gettimeout(), None) + s.close() + + # Set the default timeout to 10, and see if it propagates + socket.setdefaulttimeout(10) + self.assertEqual(socket.getdefaulttimeout(), 10) + s = socket.socket() + self.assertEqual(s.gettimeout(), 10) + s.close() + + # Reset the default timeout to None, and see if it propagates + socket.setdefaulttimeout(None) + self.assertEqual(socket.getdefaulttimeout(), None) + s = socket.socket() + self.assertEqual(s.gettimeout(), None) + s.close() + + # Check that setting it to an invalid value raises ValueError + self.assertRaises(ValueError, socket.setdefaulttimeout, -1) + + # Check that setting it to an invalid type raises TypeError + self.assertRaises(TypeError, socket.setdefaulttimeout, "spam") + + @unittest.skipUnless(hasattr(socket, 'inet_aton'), + 'test needs socket.inet_aton()') + def testIPv4_inet_aton_fourbytes(self): + # Test that issue1008086 and issue767150 are fixed. + # It must return 4 bytes. + self.assertEqual(b'\x00'*4, socket.inet_aton('0.0.0.0')) + self.assertEqual(b'\xff'*4, socket.inet_aton('255.255.255.255')) + + @unittest.skipUnless(hasattr(socket, 'inet_pton'), + 'test needs socket.inet_pton()') + def testIPv4toString(self): + from socket import inet_aton as f, inet_pton, AF_INET + g = lambda a: inet_pton(AF_INET, a) + + assertInvalid = lambda func,a: self.assertRaises( + (OSError, ValueError), func, a + ) + + self.assertEqual(b'\x00\x00\x00\x00', f('0.0.0.0')) + self.assertEqual(b'\xff\x00\xff\x00', f('255.0.255.0')) + self.assertEqual(b'\xaa\xaa\xaa\xaa', f('170.170.170.170')) + self.assertEqual(b'\x01\x02\x03\x04', f('1.2.3.4')) + self.assertEqual(b'\xff\xff\xff\xff', f('255.255.255.255')) + # bpo-29972: inet_pton() doesn't fail on AIX + if not sys.platform.startswith('aix'): + assertInvalid(f, '0.0.0.') + assertInvalid(f, '300.0.0.0') + assertInvalid(f, 'a.0.0.0') + assertInvalid(f, '1.2.3.4.5') + assertInvalid(f, '::1') + + self.assertEqual(b'\x00\x00\x00\x00', g('0.0.0.0')) + self.assertEqual(b'\xff\x00\xff\x00', g('255.0.255.0')) + self.assertEqual(b'\xaa\xaa\xaa\xaa', g('170.170.170.170')) + self.assertEqual(b'\xff\xff\xff\xff', g('255.255.255.255')) + assertInvalid(g, '0.0.0.') + assertInvalid(g, '300.0.0.0') + assertInvalid(g, 'a.0.0.0') + assertInvalid(g, '1.2.3.4.5') + assertInvalid(g, '::1') + + @unittest.skipUnless(hasattr(socket, 'inet_pton'), + 'test needs socket.inet_pton()') + def testIPv6toString(self): + try: + from socket import inet_pton, AF_INET6, has_ipv6 + if not has_ipv6: + self.skipTest('IPv6 not available') + except ImportError: + self.skipTest('could not import needed symbols from socket') + + if sys.platform == "win32": + try: + inet_pton(AF_INET6, '::') + except OSError as e: + if e.winerror == 10022: + self.skipTest('IPv6 might not be supported') + + f = lambda a: inet_pton(AF_INET6, a) + assertInvalid = lambda a: self.assertRaises( + (OSError, ValueError), f, a + ) + + self.assertEqual(b'\x00' * 16, f('::')) + self.assertEqual(b'\x00' * 16, f('0::0')) + self.assertEqual(b'\x00\x01' + b'\x00' * 14, f('1::')) + self.assertEqual( + b'\x45\xef\x76\xcb\x00\x1a\x56\xef\xaf\xeb\x0b\xac\x19\x24\xae\xae', + f('45ef:76cb:1a:56ef:afeb:bac:1924:aeae') + ) + self.assertEqual( + b'\xad\x42\x0a\xbc' + b'\x00' * 4 + b'\x01\x27\x00\x00\x02\x54\x00\x02', + f('ad42:abc::127:0:254:2') + ) + self.assertEqual(b'\x00\x12\x00\x0a' + b'\x00' * 12, f('12:a::')) + assertInvalid('0x20::') + assertInvalid(':::') + assertInvalid('::0::') + assertInvalid('1::abc::') + assertInvalid('1::abc::def') + assertInvalid('1:2:3:4:5:6') + assertInvalid('1:2:3:4:5:6:7:8:0') + # bpo-29972: inet_pton() doesn't fail on AIX + if not sys.platform.startswith('aix'): + assertInvalid('1:2:3:4:5:6:') + assertInvalid('1:2:3:4:5:6:7:8:') + + self.assertEqual(b'\x00' * 12 + b'\xfe\x2a\x17\x40', + f('::254.42.23.64') + ) + self.assertEqual( + b'\x00\x42' + b'\x00' * 8 + b'\xa2\x9b\xfe\x2a\x17\x40', + f('42::a29b:254.42.23.64') + ) + self.assertEqual( + b'\x00\x42\xa8\xb9\x00\x00\x00\x02\xff\xff\xa2\x9b\xfe\x2a\x17\x40', + f('42:a8b9:0:2:ffff:a29b:254.42.23.64') + ) + assertInvalid('255.254.253.252') + assertInvalid('1::260.2.3.0') + assertInvalid('1::0.be.e.0') + assertInvalid('1:2:3:4:5:6:7:1.2.3.4') + assertInvalid('::1.2.3.4:0') + assertInvalid('0.100.200.0:3:4:5:6:7:8') + + @unittest.skipUnless(hasattr(socket, 'inet_ntop'), + 'test needs socket.inet_ntop()') + def testStringToIPv4(self): + from socket import inet_ntoa as f, inet_ntop, AF_INET + g = lambda a: inet_ntop(AF_INET, a) + assertInvalid = lambda func,a: self.assertRaises( + (OSError, ValueError), func, a + ) + + self.assertEqual('1.0.1.0', f(b'\x01\x00\x01\x00')) + self.assertEqual('170.85.170.85', f(b'\xaa\x55\xaa\x55')) + self.assertEqual('255.255.255.255', f(b'\xff\xff\xff\xff')) + self.assertEqual('1.2.3.4', f(b'\x01\x02\x03\x04')) + assertInvalid(f, b'\x00' * 3) + assertInvalid(f, b'\x00' * 5) + assertInvalid(f, b'\x00' * 16) + self.assertEqual('170.85.170.85', f(bytearray(b'\xaa\x55\xaa\x55'))) + + self.assertEqual('1.0.1.0', g(b'\x01\x00\x01\x00')) + self.assertEqual('170.85.170.85', g(b'\xaa\x55\xaa\x55')) + self.assertEqual('255.255.255.255', g(b'\xff\xff\xff\xff')) + assertInvalid(g, b'\x00' * 3) + assertInvalid(g, b'\x00' * 5) + assertInvalid(g, b'\x00' * 16) + self.assertEqual('170.85.170.85', g(bytearray(b'\xaa\x55\xaa\x55'))) + + @unittest.skipUnless(hasattr(socket, 'inet_ntop'), + 'test needs socket.inet_ntop()') + def testStringToIPv6(self): + try: + from socket import inet_ntop, AF_INET6, has_ipv6 + if not has_ipv6: + self.skipTest('IPv6 not available') + except ImportError: + self.skipTest('could not import needed symbols from socket') + + if sys.platform == "win32": + try: + inet_ntop(AF_INET6, b'\x00' * 16) + except OSError as e: + if e.winerror == 10022: + self.skipTest('IPv6 might not be supported') + + f = lambda a: inet_ntop(AF_INET6, a) + assertInvalid = lambda a: self.assertRaises( + (OSError, ValueError), f, a + ) + + self.assertEqual('::', f(b'\x00' * 16)) + self.assertEqual('::1', f(b'\x00' * 15 + b'\x01')) + self.assertEqual( + 'aef:b01:506:1001:ffff:9997:55:170', + f(b'\x0a\xef\x0b\x01\x05\x06\x10\x01\xff\xff\x99\x97\x00\x55\x01\x70') + ) + self.assertEqual('::1', f(bytearray(b'\x00' * 15 + b'\x01'))) + + assertInvalid(b'\x12' * 15) + assertInvalid(b'\x12' * 17) + assertInvalid(b'\x12' * 4) + + # XXX The following don't test module-level functionality... + + def testSockName(self): + # Testing getsockname() + port = support.find_unused_port() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + sock.bind(("0.0.0.0", port)) + name = sock.getsockname() + # XXX(nnorwitz): http://tinyurl.com/os5jz seems to indicate + # it reasonable to get the host's addr in addition to 0.0.0.0. + # At least for eCos. This is required for the S/390 to pass. + try: + my_ip_addr = socket.gethostbyname(socket.gethostname()) + except OSError: + # Probably name lookup wasn't set up right; skip this test + self.skipTest('name lookup failure') + self.assertIn(name[0], ("0.0.0.0", my_ip_addr), '%s invalid' % name[0]) + self.assertEqual(name[1], port) + + def testGetSockOpt(self): + # Testing getsockopt() + # We know a socket should start without reuse==0 + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + reuse = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) + self.assertFalse(reuse != 0, "initial mode is reuse") + + def testSetSockOpt(self): + # Testing setsockopt() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + reuse = sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) + self.assertFalse(reuse == 0, "failed to set reuse mode") + + def testSendAfterClose(self): + # testing send() after close() with timeout + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(1) + sock.close() + self.assertRaises(OSError, sock.send, b"spam") + + def testCloseException(self): + sock = socket.socket() + sock.bind((socket._LOCALHOST, 0)) + socket.socket(fileno=sock.fileno()).close() + try: + sock.close() + except OSError as err: + # Winsock apparently raises ENOTSOCK + self.assertIn(err.errno, (errno.EBADF, errno.ENOTSOCK)) + else: + self.fail("close() should raise EBADF/ENOTSOCK") + + def testNewAttributes(self): + # testing .family, .type and .protocol + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.assertEqual(sock.family, socket.AF_INET) + if hasattr(socket, 'SOCK_CLOEXEC'): + self.assertIn(sock.type, + (socket.SOCK_STREAM | socket.SOCK_CLOEXEC, + socket.SOCK_STREAM)) + else: + self.assertEqual(sock.type, socket.SOCK_STREAM) + self.assertEqual(sock.proto, 0) + sock.close() + + def test_getsockaddrarg(self): + sock = socket.socket() + self.addCleanup(sock.close) + port = support.find_unused_port() + big_port = port + 65536 + neg_port = port - 65536 + self.assertRaises(OverflowError, sock.bind, (HOST, big_port)) + self.assertRaises(OverflowError, sock.bind, (HOST, neg_port)) + # Since find_unused_port() is inherently subject to race conditions, we + # call it a couple times if necessary. + for i in itertools.count(): + port = support.find_unused_port() + try: + sock.bind((HOST, port)) + except OSError as e: + if e.errno != errno.EADDRINUSE or i == 5: + raise + else: + break + + @unittest.skipUnless(os.name == "nt", "Windows specific") + def test_sock_ioctl(self): + self.assertTrue(hasattr(socket.socket, 'ioctl')) + self.assertTrue(hasattr(socket, 'SIO_RCVALL')) + self.assertTrue(hasattr(socket, 'RCVALL_ON')) + self.assertTrue(hasattr(socket, 'RCVALL_OFF')) + self.assertTrue(hasattr(socket, 'SIO_KEEPALIVE_VALS')) + s = socket.socket() + self.addCleanup(s.close) + self.assertRaises(ValueError, s.ioctl, -1, None) + s.ioctl(socket.SIO_KEEPALIVE_VALS, (1, 100, 100)) + + @unittest.skipUnless(os.name == "nt", "Windows specific") + @unittest.skipUnless(hasattr(socket, 'SIO_LOOPBACK_FAST_PATH'), + 'Loopback fast path support required for this test') + def test_sio_loopback_fast_path(self): + s = socket.socket() + self.addCleanup(s.close) + try: + s.ioctl(socket.SIO_LOOPBACK_FAST_PATH, True) + except OSError as exc: + WSAEOPNOTSUPP = 10045 + if exc.winerror == WSAEOPNOTSUPP: + self.skipTest("SIO_LOOPBACK_FAST_PATH is defined but " + "doesn't implemented in this Windows version") + raise + self.assertRaises(TypeError, s.ioctl, socket.SIO_LOOPBACK_FAST_PATH, None) + + def testGetaddrinfo(self): + try: + socket.getaddrinfo('localhost', 80) + except socket.gaierror as err: + if err.errno == socket.EAI_SERVICE: + # see http://bugs.python.org/issue1282647 + self.skipTest("buggy libc version") + raise + # len of every sequence is supposed to be == 5 + for info in socket.getaddrinfo(HOST, None): + self.assertEqual(len(info), 5) + # host can be a domain name, a string representation of an + # IPv4/v6 address or None + socket.getaddrinfo('localhost', 80) + socket.getaddrinfo('127.0.0.1', 80) + socket.getaddrinfo(None, 80) + if support.IPV6_ENABLED: + socket.getaddrinfo('::1', 80) + # port can be a string service name such as "http", a numeric + # port number or None + # Issue #26936: Android getaddrinfo() was broken before API level 23. + if (not hasattr(sys, 'getandroidapilevel') or + sys.getandroidapilevel() >= 23): + socket.getaddrinfo(HOST, "http") + socket.getaddrinfo(HOST, 80) + socket.getaddrinfo(HOST, None) + # test family and socktype filters + infos = socket.getaddrinfo(HOST, 80, socket.AF_INET, socket.SOCK_STREAM) + for family, type, _, _, _ in infos: + self.assertEqual(family, socket.AF_INET) + self.assertEqual(str(family), 'AddressFamily.AF_INET') + self.assertEqual(type, socket.SOCK_STREAM) + self.assertEqual(str(type), 'SocketKind.SOCK_STREAM') + infos = socket.getaddrinfo(HOST, None, 0, socket.SOCK_STREAM) + for _, socktype, _, _, _ in infos: + self.assertEqual(socktype, socket.SOCK_STREAM) + # test proto and flags arguments + socket.getaddrinfo(HOST, None, 0, 0, socket.SOL_TCP) + socket.getaddrinfo(HOST, None, 0, 0, 0, socket.AI_PASSIVE) + # a server willing to support both IPv4 and IPv6 will + # usually do this + socket.getaddrinfo(None, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, + socket.AI_PASSIVE) + # test keyword arguments + a = socket.getaddrinfo(HOST, None) + b = socket.getaddrinfo(host=HOST, port=None) + self.assertEqual(a, b) + a = socket.getaddrinfo(HOST, None, socket.AF_INET) + b = socket.getaddrinfo(HOST, None, family=socket.AF_INET) + self.assertEqual(a, b) + a = socket.getaddrinfo(HOST, None, 0, socket.SOCK_STREAM) + b = socket.getaddrinfo(HOST, None, type=socket.SOCK_STREAM) + self.assertEqual(a, b) + a = socket.getaddrinfo(HOST, None, 0, 0, socket.SOL_TCP) + b = socket.getaddrinfo(HOST, None, proto=socket.SOL_TCP) + self.assertEqual(a, b) + a = socket.getaddrinfo(HOST, None, 0, 0, 0, socket.AI_PASSIVE) + b = socket.getaddrinfo(HOST, None, flags=socket.AI_PASSIVE) + self.assertEqual(a, b) + a = socket.getaddrinfo(None, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, + socket.AI_PASSIVE) + b = socket.getaddrinfo(host=None, port=0, family=socket.AF_UNSPEC, + type=socket.SOCK_STREAM, proto=0, + flags=socket.AI_PASSIVE) + self.assertEqual(a, b) + # Issue #6697. + self.assertRaises(UnicodeEncodeError, socket.getaddrinfo, 'localhost', '\uD800') + + # Issue 17269: test workaround for OS X platform bug segfault + if hasattr(socket, 'AI_NUMERICSERV'): + try: + # The arguments here are undefined and the call may succeed + # or fail. All we care here is that it doesn't segfault. + socket.getaddrinfo("localhost", None, 0, 0, 0, + socket.AI_NUMERICSERV) + except socket.gaierror: + pass + + def test_getnameinfo(self): + # only IP addresses are allowed + self.assertRaises(OSError, socket.getnameinfo, ('mail.python.org',0), 0) + + @unittest.skipUnless(support.is_resource_enabled('network'), + 'network is not enabled') + def test_idna(self): + # Check for internet access before running test + # (issue #12804, issue #25138). + with support.transient_internet('python.org'): + socket.gethostbyname('python.org') + + # these should all be successful + domain = 'испытание.pythontest.net' + socket.gethostbyname(domain) + socket.gethostbyname_ex(domain) + socket.getaddrinfo(domain,0,socket.AF_UNSPEC,socket.SOCK_STREAM) + # this may not work if the forward lookup chooses the IPv6 address, as that doesn't + # have a reverse entry yet + # socket.gethostbyaddr('испытание.python.org') + + def check_sendall_interrupted(self, with_timeout): + # socketpair() is not strictly required, but it makes things easier. + if not hasattr(signal, 'alarm') or not hasattr(socket, 'socketpair'): + self.skipTest("signal.alarm and socket.socketpair required for this test") + # Our signal handlers clobber the C errno by calling a math function + # with an invalid domain value. + def ok_handler(*args): + self.assertRaises(ValueError, math.acosh, 0) + def raising_handler(*args): + self.assertRaises(ValueError, math.acosh, 0) + 1 // 0 + c, s = socket.socketpair() + old_alarm = signal.signal(signal.SIGALRM, raising_handler) + try: + if with_timeout: + # Just above the one second minimum for signal.alarm + c.settimeout(1.5) + with self.assertRaises(ZeroDivisionError): + signal.alarm(1) + c.sendall(b"x" * support.SOCK_MAX_SIZE) + if with_timeout: + signal.signal(signal.SIGALRM, ok_handler) + signal.alarm(1) + self.assertRaises(socket.timeout, c.sendall, + b"x" * support.SOCK_MAX_SIZE) + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_alarm) + c.close() + s.close() + + def test_sendall_interrupted(self): + self.check_sendall_interrupted(False) + + def test_sendall_interrupted_with_timeout(self): + self.check_sendall_interrupted(True) + + def test_dealloc_warn(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + r = repr(sock) + with self.assertWarns(ResourceWarning) as cm: + sock = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + # An open socket file object gets dereferenced after the socket + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + f = sock.makefile('rb') + r = repr(sock) + sock = None + support.gc_collect() + with self.assertWarns(ResourceWarning): + f = None + support.gc_collect() + + def test_name_closed_socketio(self): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + fp = sock.makefile("rb") + fp.close() + self.assertEqual(repr(fp), "<_io.BufferedReader name=-1>") + + def test_unusable_closed_socketio(self): + with socket.socket() as sock: + fp = sock.makefile("rb", buffering=0) + self.assertTrue(fp.readable()) + self.assertFalse(fp.writable()) + self.assertFalse(fp.seekable()) + fp.close() + self.assertRaises(ValueError, fp.readable) + self.assertRaises(ValueError, fp.writable) + self.assertRaises(ValueError, fp.seekable) + + def test_socket_close(self): + sock = socket.socket() + try: + sock.bind((HOST, 0)) + socket.close(sock.fileno()) + with self.assertRaises(OSError): + sock.listen(1) + finally: + with self.assertRaises(OSError): + # sock.close() fails with EBADF + sock.close() + with self.assertRaises(TypeError): + socket.close(None) + with self.assertRaises(OSError): + socket.close(-1) + + def test_makefile_mode(self): + for mode in 'r', 'rb', 'rw', 'w', 'wb': + with self.subTest(mode=mode): + with socket.socket() as sock: + with sock.makefile(mode) as fp: + self.assertEqual(fp.mode, mode) + + def test_makefile_invalid_mode(self): + for mode in 'rt', 'x', '+', 'a': + with self.subTest(mode=mode): + with socket.socket() as sock: + with self.assertRaisesRegex(ValueError, 'invalid mode'): + sock.makefile(mode) + + def test_pickle(self): + sock = socket.socket() + with sock: + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, sock, protocol) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + family = pickle.loads(pickle.dumps(socket.AF_INET, protocol)) + self.assertEqual(family, socket.AF_INET) + type = pickle.loads(pickle.dumps(socket.SOCK_STREAM, protocol)) + self.assertEqual(type, socket.SOCK_STREAM) + + def test_listen_backlog(self): + for backlog in 0, -1: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv: + srv.bind((HOST, 0)) + srv.listen(backlog) + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv: + srv.bind((HOST, 0)) + srv.listen() + + @support.cpython_only + def test_listen_backlog_overflow(self): + # Issue 15989 + import _testcapi + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.bind((HOST, 0)) + self.assertRaises(OverflowError, srv.listen, _testcapi.INT_MAX + 1) + srv.close() + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + def test_flowinfo(self): + self.assertRaises(OverflowError, socket.getnameinfo, + (support.HOSTv6, 0, 0xffffffff), 0) + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + self.assertRaises(OverflowError, s.bind, (support.HOSTv6, 0, -10)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + def test_getaddrinfo_ipv6_basic(self): + ((*_, sockaddr),) = socket.getaddrinfo( + 'ff02::1de:c0:face:8D', # Note capital letter `D`. + 1234, socket.AF_INET6, + socket.SOCK_DGRAM, + socket.IPPROTO_UDP + ) + self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, 0)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + @unittest.skipUnless( + hasattr(socket, 'if_nameindex'), + 'if_nameindex is not supported') + def test_getaddrinfo_ipv6_scopeid_symbolic(self): + # Just pick up any network interface (Linux, Mac OS X) + (ifindex, test_interface) = socket.if_nameindex()[0] + ((*_, sockaddr),) = socket.getaddrinfo( + 'ff02::1de:c0:face:8D%' + test_interface, + 1234, socket.AF_INET6, + socket.SOCK_DGRAM, + socket.IPPROTO_UDP + ) + # Note missing interface name part in IPv6 address + self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, ifindex)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + @unittest.skipUnless( + sys.platform == 'win32', + 'Numeric scope id does not work or undocumented') + def test_getaddrinfo_ipv6_scopeid_numeric(self): + # Also works on Linux and Mac OS X, but is not documented (?) + # Windows, Linux and Max OS X allow nonexistent interface numbers here. + ifindex = 42 + ((*_, sockaddr),) = socket.getaddrinfo( + 'ff02::1de:c0:face:8D%' + str(ifindex), + 1234, socket.AF_INET6, + socket.SOCK_DGRAM, + socket.IPPROTO_UDP + ) + # Note missing interface name part in IPv6 address + self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, ifindex)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + @unittest.skipUnless( + hasattr(socket, 'if_nameindex'), + 'if_nameindex is not supported') + def test_getnameinfo_ipv6_scopeid_symbolic(self): + # Just pick up any network interface. + (ifindex, test_interface) = socket.if_nameindex()[0] + sockaddr = ('ff02::1de:c0:face:8D', 1234, 0, ifindex) # Note capital letter `D`. + nameinfo = socket.getnameinfo(sockaddr, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV) + self.assertEqual(nameinfo, ('ff02::1de:c0:face:8d%' + test_interface, '1234')) + + @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') + @unittest.skipUnless( + sys.platform == 'win32', + 'Numeric scope id does not work or undocumented') + def test_getnameinfo_ipv6_scopeid_numeric(self): + # Also works on Linux (undocumented), but does not work on Mac OS X + # Windows and Linux allow nonexistent interface numbers here. + ifindex = 42 + sockaddr = ('ff02::1de:c0:face:8D', 1234, 0, ifindex) # Note capital letter `D`. + nameinfo = socket.getnameinfo(sockaddr, socket.NI_NUMERICHOST | socket.NI_NUMERICSERV) + self.assertEqual(nameinfo, ('ff02::1de:c0:face:8d%' + str(ifindex), '1234')) + + def test_str_for_enums(self): + # Make sure that the AF_* and SOCK_* constants have enum-like string + # reprs. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + self.assertEqual(str(s.family), 'AddressFamily.AF_INET') + self.assertEqual(str(s.type), 'SocketKind.SOCK_STREAM') + + def test_socket_consistent_sock_type(self): + SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) + SOCK_CLOEXEC = getattr(socket, 'SOCK_CLOEXEC', 0) + sock_type = socket.SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC + + with socket.socket(socket.AF_INET, sock_type) as s: + self.assertEqual(s.type, socket.SOCK_STREAM) + s.settimeout(1) + self.assertEqual(s.type, socket.SOCK_STREAM) + s.settimeout(0) + self.assertEqual(s.type, socket.SOCK_STREAM) + s.setblocking(True) + self.assertEqual(s.type, socket.SOCK_STREAM) + s.setblocking(False) + self.assertEqual(s.type, socket.SOCK_STREAM) + + @unittest.skipIf(os.name == 'nt', 'Will not work on Windows') + def test_unknown_socket_family_repr(self): + # Test that when created with a family that's not one of the known + # AF_*/SOCK_* constants, socket.family just returns the number. + # + # To do this we fool socket.socket into believing it already has an + # open fd because on this path it doesn't actually verify the family and + # type and populates the socket object. + # + # On Windows this trick won't work, so the test is skipped. + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + unknown_family = max(socket.AddressFamily.__members__.values()) + 1 + + unknown_type = max( + kind + for name, kind in socket.SocketKind.__members__.items() + if name not in {'SOCK_NONBLOCK', 'SOCK_CLOEXEC'} + ) + 1 + + with socket.socket( + family=unknown_family, type=unknown_type, proto=23, + fileno=fd) as s: + self.assertEqual(s.family, unknown_family) + self.assertEqual(s.type, unknown_type) + # some OS like macOS ignore proto + self.assertIn(s.proto, {0, 23}) + + @unittest.skipUnless(hasattr(os, 'sendfile'), 'test needs os.sendfile()') + def test__sendfile_use_sendfile(self): + class File: + def __init__(self, fd): + self.fd = fd + + def fileno(self): + return self.fd + with socket.socket() as sock: + fd = os.open(os.curdir, os.O_RDONLY) + os.close(fd) + with self.assertRaises(socket._GiveupOnSendfile): + sock._sendfile_use_sendfile(File(fd)) + with self.assertRaises(OverflowError): + sock._sendfile_use_sendfile(File(2**1000)) + with self.assertRaises(TypeError): + sock._sendfile_use_sendfile(File(None)) + + def _test_socket_fileno(self, s, family, stype): + self.assertEqual(s.family, family) + self.assertEqual(s.type, stype) + + fd = s.fileno() + s2 = socket.socket(fileno=fd) + self.addCleanup(s2.close) + # detach old fd to avoid double close + s.detach() + self.assertEqual(s2.family, family) + self.assertEqual(s2.type, stype) + self.assertEqual(s2.fileno(), fd) + + def test_socket_fileno(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(s.close) + s.bind((support.HOST, 0)) + self._test_socket_fileno(s, socket.AF_INET, socket.SOCK_STREAM) + + if hasattr(socket, "SOCK_DGRAM"): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind((support.HOST, 0)) + self._test_socket_fileno(s, socket.AF_INET, socket.SOCK_DGRAM) + + if support.IPV6_ENABLED: + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self.addCleanup(s.close) + s.bind((support.HOSTv6, 0, 0, 0)) + self._test_socket_fileno(s, socket.AF_INET6, socket.SOCK_STREAM) + + if hasattr(socket, "AF_UNIX"): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.addCleanup(s.close) + s.bind(os.path.join(tmpdir, 'socket')) + self._test_socket_fileno(s, socket.AF_UNIX, socket.SOCK_STREAM) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +class BasicCANTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_CAN + socket.PF_CAN + socket.CAN_RAW + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCMConstants(self): + socket.CAN_BCM + + # opcodes + socket.CAN_BCM_TX_SETUP # create (cyclic) transmission task + socket.CAN_BCM_TX_DELETE # remove (cyclic) transmission task + socket.CAN_BCM_TX_READ # read properties of (cyclic) transmission task + socket.CAN_BCM_TX_SEND # send one CAN frame + socket.CAN_BCM_RX_SETUP # create RX content filter subscription + socket.CAN_BCM_RX_DELETE # remove RX content filter subscription + socket.CAN_BCM_RX_READ # read properties of RX content filter subscription + socket.CAN_BCM_TX_STATUS # reply to TX_READ request + socket.CAN_BCM_TX_EXPIRED # notification on performed transmissions (count=0) + socket.CAN_BCM_RX_STATUS # reply to RX_READ request + socket.CAN_BCM_RX_TIMEOUT # cyclic message is absent + socket.CAN_BCM_RX_CHANGED # updated CAN frame (detected content change) + + def testCreateSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + pass + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testCreateBCMSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) as s: + pass + + def testBindAny(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.bind(('', )) + + def testTooLongInterfaceName(self): + # most systems limit IFNAMSIZ to 16, take 1024 to be sure + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + self.assertRaisesRegex(OSError, 'interface name too long', + s.bind, ('x' * 1024,)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_LOOPBACK"), + 'socket.CAN_RAW_LOOPBACK required for this test.') + def testLoopback(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + for loopback in (0, 1): + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK, + loopback) + self.assertEqual(loopback, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_FILTER"), + 'socket.CAN_RAW_FILTER required for this test.') + def testFilter(self): + can_id, can_mask = 0x200, 0x700 + can_filter = struct.pack("=II", can_id, can_mask) + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, can_filter) + self.assertEqual(can_filter, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, 8)) + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, bytearray(can_filter)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +class CANTest(ThreadedCANSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedCANSocketTest.__init__(self, methodName=methodName) + + @classmethod + def build_can_frame(cls, can_id, data): + """Build a CAN frame.""" + can_dlc = len(data) + data = data.ljust(8, b'\x00') + return struct.pack(cls.can_frame_fmt, can_id, can_dlc, data) + + @classmethod + def dissect_can_frame(cls, frame): + """Dissect a CAN frame.""" + can_id, can_dlc, data = struct.unpack(cls.can_frame_fmt, frame) + return (can_id, can_dlc, data[:can_dlc]) + + def testSendFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + self.assertEqual(addr[0], self.interface) + self.assertEqual(addr[1], socket.AF_CAN) + + def _testSendFrame(self): + self.cf = self.build_can_frame(0x00, b'\x01\x02\x03\x04\x05') + self.cli.send(self.cf) + + def testSendMaxFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + + def _testSendMaxFrame(self): + self.cf = self.build_can_frame(0x00, b'\x07' * 8) + self.cli.send(self.cf) + + def testSendMultiFrames(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf1, cf) + + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf2, cf) + + def _testSendMultiFrames(self): + self.cf1 = self.build_can_frame(0x07, b'\x44\x33\x22\x11') + self.cli.send(self.cf1) + + self.cf2 = self.build_can_frame(0x12, b'\x99\x22\x33') + self.cli.send(self.cf2) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def _testBCM(self): + cf, addr = self.cli.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + can_id, can_dlc, data = self.dissect_can_frame(cf) + self.assertEqual(self.can_id, can_id) + self.assertEqual(self.data, data) + + @unittest.skipUnless(hasattr(socket, "CAN_BCM"), + 'socket.CAN_BCM required for this test.') + def testBCM(self): + bcm = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) + self.addCleanup(bcm.close) + bcm.connect((self.interface,)) + self.can_id = 0x123 + self.data = bytes([0xc0, 0xff, 0xee]) + self.cf = self.build_can_frame(self.can_id, self.data) + opcode = socket.CAN_BCM_TX_SEND + flags = 0 + count = 0 + ival1_seconds = ival1_usec = ival2_seconds = ival2_usec = 0 + bcm_can_id = 0x0222 + nframes = 1 + assert len(self.cf) == 16 + header = struct.pack(self.bcm_cmd_msg_fmt, + opcode, + flags, + count, + ival1_seconds, + ival1_usec, + ival2_seconds, + ival2_usec, + bcm_can_id, + nframes, + ) + header_plus_frame = header + self.cf + bytes_sent = bcm.send(header_plus_frame) + self.assertEqual(bytes_sent, len(header_plus_frame)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN_ISOTP, 'CAN ISOTP required for this test.') +class ISOTPTest(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.interface = "vcan0" + + def testCrucialConstants(self): + socket.AF_CAN + socket.PF_CAN + socket.CAN_ISOTP + socket.SOCK_DGRAM + + def testCreateSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + pass + + @unittest.skipUnless(hasattr(socket, "CAN_ISOTP"), + 'socket.CAN_ISOTP required for this test.') + def testCreateISOTPSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: + pass + + def testTooLongInterfaceName(self): + # most systems limit IFNAMSIZ to 16, take 1024 to be sure + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: + with self.assertRaisesRegex(OSError, 'interface name too long'): + s.bind(('x' * 1024, 1, 2)) + + def testBind(self): + try: + with socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_ISOTP) as s: + addr = self.interface, 0x123, 0x456 + s.bind(addr) + self.assertEqual(s.getsockname(), addr) + except OSError as e: + if e.errno == errno.ENODEV: + self.skipTest('network interface `%s` does not exist' % + self.interface) + else: + raise + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +class BasicRDSTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_RDS + socket.PF_RDS + + def testCreateSocket(self): + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + pass + + def testSocketBufferSize(self): + bufsize = 16384 + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, bufsize) + s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, bufsize) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +class RDSTest(ThreadedRDSSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedRDSSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + super().setUp() + self.evt = threading.Event() + + def testSendAndRecv(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + self.assertEqual(self.cli_addr, addr) + + def _testSendAndRecv(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testPeek(self): + data, addr = self.serv.recvfrom(self.bufsize, socket.MSG_PEEK) + self.assertEqual(self.data, data) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testPeek(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + @requireAttrs(socket.socket, 'recvmsg') + def testSendAndRecvMsg(self): + data, ancdata, msg_flags, addr = self.serv.recvmsg(self.bufsize) + self.assertEqual(self.data, data) + + @requireAttrs(socket.socket, 'sendmsg') + def _testSendAndRecvMsg(self): + self.data = b'hello ' * 10 + self.cli.sendmsg([self.data], (), 0, (HOST, self.port)) + + def testSendAndRecvMulti(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data1, data) + + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data2, data) + + def _testSendAndRecvMulti(self): + self.data1 = b'bacon' + self.cli.sendto(self.data1, 0, (HOST, self.port)) + + self.data2 = b'egg' + self.cli.sendto(self.data2, 0, (HOST, self.port)) + + def testSelect(self): + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testSelect(self): + self.data = b'select' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testCongestion(self): + # wait until the sender is done + self.evt.wait() + + def _testCongestion(self): + # test the behavior in case of congestion + self.data = b'fill' + self.cli.setblocking(False) + try: + # try to lower the receiver's socket buffer size + self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384) + except OSError: + pass + with self.assertRaises(OSError) as cm: + try: + # fill the receiver's socket buffer + while True: + self.cli.sendto(self.data, 0, (HOST, self.port)) + finally: + # signal the receiver we're done + self.evt.set() + # sendto() should have failed with ENOBUFS + self.assertEqual(cm.exception.errno, errno.ENOBUFS) + # and we should have received a congestion notification through poll + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + + +@unittest.skipIf(fcntl is None, "need fcntl") +@unittest.skipUnless(HAVE_SOCKET_VSOCK, + 'VSOCK sockets required for this test.') +class BasicVSOCKTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_VSOCK + + def testVSOCKConstants(self): + socket.SO_VM_SOCKETS_BUFFER_SIZE + socket.SO_VM_SOCKETS_BUFFER_MIN_SIZE + socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE + socket.VMADDR_CID_ANY + socket.VMADDR_PORT_ANY + socket.VMADDR_CID_HOST + socket.VM_SOCKETS_INVALID_VERSION + socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID + + def testCreateSocket(self): + with socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) as s: + pass + + def testSocketBufferSize(self): + with socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) as s: + orig_max = s.getsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE) + orig = s.getsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_SIZE) + orig_min = s.getsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_MIN_SIZE) + + s.setsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE, orig_max * 2) + s.setsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_SIZE, orig * 2) + s.setsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_MIN_SIZE, orig_min * 2) + + self.assertEqual(orig_max * 2, + s.getsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE)) + self.assertEqual(orig * 2, + s.getsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_SIZE)) + self.assertEqual(orig_min * 2, + s.getsockopt(socket.AF_VSOCK, + socket.SO_VM_SOCKETS_BUFFER_MIN_SIZE)) + + +class BasicTCPTest(SocketConnectedTest): + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecv(self): + # Testing large receive over TCP + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.serv_conn.send(MSG) + + def testOverFlowRecv(self): + # Testing receive in chunks over TCP + seg1 = self.cli_conn.recv(len(MSG) - 3) + seg2 = self.cli_conn.recv(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecv(self): + self.serv_conn.send(MSG) + + def testRecvFrom(self): + # Testing large recvfrom() over TCP + msg, addr = self.cli_conn.recvfrom(1024) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.serv_conn.send(MSG) + + def testOverFlowRecvFrom(self): + # Testing recvfrom() in chunks over TCP + seg1, addr = self.cli_conn.recvfrom(len(MSG)-3) + seg2, addr = self.cli_conn.recvfrom(1024) + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testOverFlowRecvFrom(self): + self.serv_conn.send(MSG) + + def testSendAll(self): + # Testing sendall() with a 2048 byte string over TCP + msg = b'' + while 1: + read = self.cli_conn.recv(1024) + if not read: + break + msg += read + self.assertEqual(msg, b'f' * 2048) + + def _testSendAll(self): + big_chunk = b'f' * 2048 + self.serv_conn.sendall(big_chunk) + + def testFromFd(self): + # Testing fromfd() + fd = self.cli_conn.fileno() + sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + self.assertIsInstance(sock, socket.socket) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testFromFd(self): + self.serv_conn.send(MSG) + + def testDup(self): + # Testing dup() + sock = self.cli_conn.dup() + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDup(self): + self.serv_conn.send(MSG) + + def testShutdown(self): + # Testing shutdown() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, MSG) + # wait for _testShutdown to finish: on OS X, when the server + # closes the connection the client also becomes disconnected, + # and the client's shutdown call will fail. (Issue #4397.) + self.done.wait() + + def _testShutdown(self): + self.serv_conn.send(MSG) + self.serv_conn.shutdown(2) + + testShutdown_overflow = support.cpython_only(testShutdown) + + @support.cpython_only + def _testShutdown_overflow(self): + import _testcapi + self.serv_conn.send(MSG) + # Issue 15989 + self.assertRaises(OverflowError, self.serv_conn.shutdown, + _testcapi.INT_MAX + 1) + self.assertRaises(OverflowError, self.serv_conn.shutdown, + 2 + (_testcapi.UINT_MAX + 1)) + self.serv_conn.shutdown(2) + + def testDetach(self): + # Testing detach() + fileno = self.cli_conn.fileno() + f = self.cli_conn.detach() + self.assertEqual(f, fileno) + # cli_conn cannot be used anymore... + self.assertTrue(self.cli_conn._closed) + self.assertRaises(OSError, self.cli_conn.recv, 1024) + self.cli_conn.close() + # ...but we can create another socket using the (still open) + # file descriptor + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=f) + self.addCleanup(sock.close) + msg = sock.recv(1024) + self.assertEqual(msg, MSG) + + def _testDetach(self): + self.serv_conn.send(MSG) + + +class BasicUDPTest(ThreadedUDPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedUDPSocketTest.__init__(self, methodName=methodName) + + def testSendtoAndRecv(self): + # Testing sendto() and Recv() over UDP + msg = self.serv.recv(len(MSG)) + self.assertEqual(msg, MSG) + + def _testSendtoAndRecv(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFrom(self): + # Testing recvfrom() over UDP + msg, addr = self.serv.recvfrom(len(MSG)) + self.assertEqual(msg, MSG) + + def _testRecvFrom(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + + def testRecvFromNegative(self): + # Negative lengths passed to recvfrom should give ValueError. + self.assertRaises(ValueError, self.serv.recvfrom, -1) + + def _testRecvFromNegative(self): + self.cli.sendto(MSG, 0, (HOST, self.port)) + +# Tests for the sendmsg()/recvmsg() interface. Where possible, the +# same test code is used with different families and types of socket +# (e.g. stream, datagram), and tests using recvmsg() are repeated +# using recvmsg_into(). +# +# The generic test classes such as SendmsgTests and +# RecvmsgGenericTests inherit from SendrecvmsgBase and expect to be +# supplied with sockets cli_sock and serv_sock representing the +# client's and the server's end of the connection respectively, and +# attributes cli_addr and serv_addr holding their (numeric where +# appropriate) addresses. +# +# The final concrete test classes combine these with subclasses of +# SocketTestBase which set up client and server sockets of a specific +# type, and with subclasses of SendrecvmsgBase such as +# SendrecvmsgDgramBase and SendrecvmsgConnectedBase which map these +# sockets to cli_sock and serv_sock and override the methods and +# attributes of SendrecvmsgBase to fill in destination addresses if +# needed when sending, check for specific flags in msg_flags, etc. +# +# RecvmsgIntoMixin provides a version of doRecvmsg() implemented using +# recvmsg_into(). + +# XXX: like the other datagram (UDP) tests in this module, the code +# here assumes that datagram delivery on the local machine will be +# reliable. + +class SendrecvmsgBase(ThreadSafeCleanupTestCase): + # Base class for sendmsg()/recvmsg() tests. + + # Time in seconds to wait before considering a test failed, or + # None for no timeout. Not all tests actually set a timeout. + fail_timeout = 3.0 + + def setUp(self): + self.misc_event = threading.Event() + super().setUp() + + def sendToServer(self, msg): + # Send msg to the server. + return self.cli_sock.send(msg) + + # Tuple of alternative default arguments for sendmsg() when called + # via sendmsgToServer() (e.g. to include a destination address). + sendmsg_to_server_defaults = () + + def sendmsgToServer(self, *args): + # Call sendmsg() on self.cli_sock with the given arguments, + # filling in any arguments which are not supplied with the + # corresponding items of self.sendmsg_to_server_defaults, if + # any. + return self.cli_sock.sendmsg( + *(args + self.sendmsg_to_server_defaults[len(args):])) + + def doRecvmsg(self, sock, bufsize, *args): + # Call recvmsg() on sock with given arguments and return its + # result. Should be used for tests which can use either + # recvmsg() or recvmsg_into() - RecvmsgIntoMixin overrides + # this method with one which emulates it using recvmsg_into(), + # thus allowing the same test to be used for both methods. + result = sock.recvmsg(bufsize, *args) + self.registerRecvmsgResult(result) + return result + + def registerRecvmsgResult(self, result): + # Called by doRecvmsg() with the return value of recvmsg() or + # recvmsg_into(). Can be overridden to arrange cleanup based + # on the returned ancillary data, for instance. + pass + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer. + self.assertEqual(addr1, addr2) + + # Flags that are normally unset in msg_flags + msg_flags_common_unset = 0 + for name in ("MSG_CTRUNC", "MSG_OOB"): + msg_flags_common_unset |= getattr(socket, name, 0) + + # Flags that are normally set + msg_flags_common_set = 0 + + # Flags set when a complete record has been received (e.g. MSG_EOR + # for SCTP) + msg_flags_eor_indicator = 0 + + # Flags set when a complete record has not been received + # (e.g. MSG_TRUNC for datagram sockets) + msg_flags_non_eor_indicator = 0 + + def checkFlags(self, flags, eor=None, checkset=0, checkunset=0, ignore=0): + # Method to check the value of msg_flags returned by recvmsg[_into](). + # + # Checks that all bits in msg_flags_common_set attribute are + # set in "flags" and all bits in msg_flags_common_unset are + # unset. + # + # The "eor" argument specifies whether the flags should + # indicate that a full record (or datagram) has been received. + # If "eor" is None, no checks are done; otherwise, checks + # that: + # + # * if "eor" is true, all bits in msg_flags_eor_indicator are + # set and all bits in msg_flags_non_eor_indicator are unset + # + # * if "eor" is false, all bits in msg_flags_non_eor_indicator + # are set and all bits in msg_flags_eor_indicator are unset + # + # If "checkset" and/or "checkunset" are supplied, they require + # the given bits to be set or unset respectively, overriding + # what the attributes require for those bits. + # + # If any bits are set in "ignore", they will not be checked, + # regardless of the other inputs. + # + # Will raise Exception if the inputs require a bit to be both + # set and unset, and it is not ignored. + + defaultset = self.msg_flags_common_set + defaultunset = self.msg_flags_common_unset + + if eor: + defaultset |= self.msg_flags_eor_indicator + defaultunset |= self.msg_flags_non_eor_indicator + elif eor is not None: + defaultset |= self.msg_flags_non_eor_indicator + defaultunset |= self.msg_flags_eor_indicator + + # Function arguments override defaults + defaultset &= ~checkunset + defaultunset &= ~checkset + + # Merge arguments with remaining defaults, and check for conflicts + checkset |= defaultset + checkunset |= defaultunset + inboth = checkset & checkunset & ~ignore + if inboth: + raise Exception("contradictory set, unset requirements for flags " + "{0:#x}".format(inboth)) + + # Compare with given msg_flags value + mask = (checkset | checkunset) & ~ignore + self.assertEqual(flags & mask, checkset & mask) + + +class RecvmsgIntoMixin(SendrecvmsgBase): + # Mixin to implement doRecvmsg() using recvmsg_into(). + + def doRecvmsg(self, sock, bufsize, *args): + buf = bytearray(bufsize) + result = sock.recvmsg_into([buf], *args) + self.registerRecvmsgResult(result) + self.assertGreaterEqual(result[0], 0) + self.assertLessEqual(result[0], bufsize) + return (bytes(buf[:result[0]]),) + result[1:] + + +class SendrecvmsgDgramFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for datagram sockets. + + @property + def msg_flags_non_eor_indicator(self): + return super().msg_flags_non_eor_indicator | socket.MSG_TRUNC + + +class SendrecvmsgSCTPFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for SCTP sockets. + + @property + def msg_flags_eor_indicator(self): + return super().msg_flags_eor_indicator | socket.MSG_EOR + + +class SendrecvmsgConnectionlessBase(SendrecvmsgBase): + # Base class for tests on connectionless-mode sockets. Users must + # supply sockets on attributes cli and serv to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.serv + + @property + def cli_sock(self): + return self.cli + + @property + def sendmsg_to_server_defaults(self): + return ([], [], 0, self.serv_addr) + + def sendToServer(self, msg): + return self.cli_sock.sendto(msg, self.serv_addr) + + +class SendrecvmsgConnectedBase(SendrecvmsgBase): + # Base class for tests on connected sockets. Users must supply + # sockets on attributes serv_conn and cli_conn (representing the + # connections *to* the server and the client), to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.cli_conn + + @property + def cli_sock(self): + return self.serv_conn + + def checkRecvmsgAddress(self, addr1, addr2): + # Address is currently "unspecified" for a connected socket, + # so we don't examine it + pass + + +class SendrecvmsgServerTimeoutBase(SendrecvmsgBase): + # Base class to set a timeout on server's socket. + + def setUp(self): + super().setUp() + self.serv_sock.settimeout(self.fail_timeout) + + +class SendmsgTests(SendrecvmsgServerTimeoutBase): + # Tests for sendmsg() which can use any socket type and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsg(self): + # Send a simple message with sendmsg(). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG]), len(MSG)) + + def testSendmsgDataGenerator(self): + # Send from buffer obtained from a generator (not a sequence). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgDataGenerator(self): + self.assertEqual(self.sendmsgToServer((o for o in [MSG])), + len(MSG)) + + def testSendmsgAncillaryGenerator(self): + # Gather (empty) ancillary data from a generator. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgAncillaryGenerator(self): + self.assertEqual(self.sendmsgToServer([MSG], (o for o in [])), + len(MSG)) + + def testSendmsgArray(self): + # Send data from an array instead of the usual bytes object. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgArray(self): + self.assertEqual(self.sendmsgToServer([array.array("B", MSG)]), + len(MSG)) + + def testSendmsgGather(self): + # Send message data from more than one buffer (gather write). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgGather(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + def testSendmsgBadArgs(self): + # Check that sendmsg() rejects invalid arguments. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadArgs(self): + self.assertRaises(TypeError, self.cli_sock.sendmsg) + self.assertRaises(TypeError, self.sendmsgToServer, + b"not in an iterable") + self.assertRaises(TypeError, self.sendmsgToServer, + object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG, object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], 0, object()) + self.sendToServer(b"done") + + def testSendmsgBadCmsg(self): + # Check that invalid ancillary data items are rejected. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(object(), 0, b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, object(), b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, object())]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0)]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b"data", 42)]) + self.sendToServer(b"done") + + @requireAttrs(socket, "CMSG_SPACE") + def testSendmsgBadMultiCmsg(self): + # Check that invalid ancillary data items are rejected when + # more than one item is present. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + @testSendmsgBadMultiCmsg.client_skip + def _testSendmsgBadMultiCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [0, 0, b""]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b""), object()]) + self.sendToServer(b"done") + + def testSendmsgExcessCmsgReject(self): + # Check that sendmsg() rejects excess ancillary data items + # when the number that can be sent is limited. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgExcessCmsgReject(self): + if not hasattr(socket, "CMSG_SPACE"): + # Can only send one item + with self.assertRaises(OSError) as cm: + self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")]) + self.assertIsNone(cm.exception.errno) + self.sendToServer(b"done") + + def testSendmsgAfterClose(self): + # Check that sendmsg() fails on a closed socket. + pass + + def _testSendmsgAfterClose(self): + self.cli_sock.close() + self.assertRaises(OSError, self.sendmsgToServer, [MSG]) + + +class SendmsgStreamTests(SendmsgTests): + # Tests for sendmsg() which require a stream socket and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsgExplicitNoneAddr(self): + # Check that peer address can be specified as None. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgExplicitNoneAddr(self): + self.assertEqual(self.sendmsgToServer([MSG], [], 0, None), len(MSG)) + + def testSendmsgTimeout(self): + # Check that timeout works with sendmsg(). + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + def _testSendmsgTimeout(self): + try: + self.cli_sock.settimeout(0.03) + with self.assertRaises(socket.timeout): + while True: + self.sendmsgToServer([b"a"*512]) + finally: + self.misc_event.set() + + # XXX: would be nice to have more tests for sendmsg flags argument. + + # Linux supports MSG_DONTWAIT when sending, but in general, it + # only works when receiving. Could add other platforms if they + # support it too. + @skipWithClientIf(sys.platform not in {"linux"}, + "MSG_DONTWAIT not known to work on this platform when " + "sending") + def testSendmsgDontWait(self): + # Check that MSG_DONTWAIT in flags causes non-blocking behaviour. + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @testSendmsgDontWait.client_skip + def _testSendmsgDontWait(self): + try: + with self.assertRaises(OSError) as cm: + while True: + self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT) + self.assertIn(cm.exception.errno, + (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + self.misc_event.set() + + +class SendmsgConnectionlessTests(SendmsgTests): + # Tests for sendmsg() which require a connectionless-mode + # (e.g. datagram) socket, and do not involve recvmsg() or + # recvmsg_into(). + + def testSendmsgNoDestAddr(self): + # Check that sendmsg() fails when no destination address is + # given for unconnected socket. + pass + + def _testSendmsgNoDestAddr(self): + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG]) + self.assertRaises(OSError, self.cli_sock.sendmsg, + [MSG], [], 0, None) + + +class RecvmsgGenericTests(SendrecvmsgBase): + # Tests for recvmsg() which can also be emulated using + # recvmsg_into(), and can use any socket type. + + def testRecvmsg(self): + # Receive a simple message with recvmsg[_into](). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsg(self): + self.sendToServer(MSG) + + def testRecvmsgExplicitDefaults(self): + # Test recvmsg[_into]() with default arguments provided explicitly. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgExplicitDefaults(self): + self.sendToServer(MSG) + + def testRecvmsgShorter(self): + # Receive a message smaller than buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) + 42) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShorter(self): + self.sendToServer(MSG) + + def testRecvmsgTrunc(self): + # Receive part of message, check for truncation indicators. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + def _testRecvmsgTrunc(self): + self.sendToServer(MSG) + + def testRecvmsgShortAncillaryBuf(self): + # Test ancillary data buffer too small to hold any ancillary data. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 1) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShortAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgLongAncillaryBuf(self): + # Test large ancillary data buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgLongAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgAfterClose(self): + # Check that recvmsg[_into]() fails on a closed socket. + self.serv_sock.close() + self.assertRaises(OSError, self.doRecvmsg, self.serv_sock, 1024) + + def _testRecvmsgAfterClose(self): + pass + + def testRecvmsgTimeout(self): + # Check that timeout works. + try: + self.serv_sock.settimeout(0.03) + self.assertRaises(socket.timeout, + self.doRecvmsg, self.serv_sock, len(MSG)) + finally: + self.misc_event.set() + + def _testRecvmsgTimeout(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @requireAttrs(socket, "MSG_PEEK") + def testRecvmsgPeek(self): + # Check that MSG_PEEK in flags enables examination of pending + # data without consuming it. + + # Receive part of data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3, 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + # Ignoring MSG_TRUNC here (so this test is the same for stream + # and datagram sockets). Some wording in POSIX seems to + # suggest that it needn't be set when peeking, but that may + # just be a slip. + self.checkFlags(flags, eor=False, + ignore=getattr(socket, "MSG_TRUNC", 0)) + + # Receive all data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + # Check that the same data can still be received normally. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgPeek.client_skip + def _testRecvmsgPeek(self): + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + def testRecvmsgFromSendmsg(self): + # Test receiving with recvmsg[_into]() when message is sent + # using sendmsg(). + self.serv_sock.settimeout(self.fail_timeout) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgFromSendmsg.client_skip + def _testRecvmsgFromSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + +class RecvmsgGenericStreamTests(RecvmsgGenericTests): + # Tests which require a stream socket and can use either recvmsg() + # or recvmsg_into(). + + def testRecvmsgEOF(self): + # Receive end-of-stream indicator (b"", peer socket closed). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.assertEqual(msg, b"") + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=None) # Might not have end-of-record marker + + def _testRecvmsgEOF(self): + self.cli_sock.close() + + def testRecvmsgOverflow(self): + # Receive a message in more than one chunk. + seg1, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + seg2, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testRecvmsgOverflow(self): + self.sendToServer(MSG) + + +class RecvmsgTests(RecvmsgGenericTests): + # Tests for recvmsg() which can use any socket type. + + def testRecvmsgBadArgs(self): + # Check that recvmsg() rejects invalid arguments. + self.assertRaises(TypeError, self.serv_sock.recvmsg) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + -1, 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + len(MSG), -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + [bytearray(10)], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + object(), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), 0, object()) + + msg, ancdata, flags, addr = self.serv_sock.recvmsg(len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgBadArgs(self): + self.sendToServer(MSG) + + +class RecvmsgIntoTests(RecvmsgIntoMixin, RecvmsgGenericTests): + # Tests for recvmsg_into() which can use any socket type. + + def testRecvmsgIntoBadArgs(self): + # Check that recvmsg_into() rejects invalid arguments. + buf = bytearray(len(MSG)) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + len(MSG), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + buf, 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [object()], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [b"I'm not writable"], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf, object()], 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg_into, + [buf], -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], 0, object()) + + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf], 0, 0) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoBadArgs(self): + self.sendToServer(MSG) + + def testRecvmsgIntoGenerator(self): + # Receive into buffer obtained from a generator (not a sequence). + buf = bytearray(len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + (o for o in [buf])) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoGenerator(self): + self.sendToServer(MSG) + + def testRecvmsgIntoArray(self): + # Receive into an array rather than the usual bytearray. + buf = array.array("B", [0] * len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf]) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf.tobytes(), MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoArray(self): + self.sendToServer(MSG) + + def testRecvmsgIntoScatter(self): + # Receive into multiple buffers (scatter write). + b1 = bytearray(b"----") + b2 = bytearray(b"0123456789") + b3 = bytearray(b"--------------") + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + [b1, memoryview(b2)[2:9], b3]) + self.assertEqual(nbytes, len(b"Mary had a little lamb")) + self.assertEqual(b1, bytearray(b"Mary")) + self.assertEqual(b2, bytearray(b"01 had a 9")) + self.assertEqual(b3, bytearray(b"little lamb---")) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoScatter(self): + self.sendToServer(b"Mary had a little lamb") + + +class CmsgMacroTests(unittest.TestCase): + # Test the functions CMSG_LEN() and CMSG_SPACE(). Tests + # assumptions used by sendmsg() and recvmsg[_into](), which share + # code with these functions. + + # Match the definition in socketmodule.c + try: + import _testcapi + except ImportError: + socklen_t_limit = 0x7fffffff + else: + socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) + + @requireAttrs(socket, "CMSG_LEN") + def testCMSG_LEN(self): + # Test CMSG_LEN() with various valid and invalid values, + # checking the assumptions used by recvmsg() and sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_LEN(0) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(socket.CMSG_LEN(0), array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_LEN(n) + # This is how recvmsg() calculates the data size + self.assertEqual(ret - socket.CMSG_LEN(0), n) + self.assertLessEqual(ret, self.socklen_t_limit) + + self.assertRaises(OverflowError, socket.CMSG_LEN, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_LEN, toobig) + self.assertRaises(OverflowError, socket.CMSG_LEN, sys.maxsize) + + @requireAttrs(socket, "CMSG_SPACE") + def testCMSG_SPACE(self): + # Test CMSG_SPACE() with various valid and invalid values, + # checking the assumptions used by sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + last = socket.CMSG_SPACE(0) + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(last, array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_SPACE(n) + self.assertGreaterEqual(ret, last) + self.assertGreaterEqual(ret, socket.CMSG_LEN(n)) + self.assertGreaterEqual(ret, n + socket.CMSG_LEN(0)) + self.assertLessEqual(ret, self.socklen_t_limit) + last = ret + + self.assertRaises(OverflowError, socket.CMSG_SPACE, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_SPACE, toobig) + self.assertRaises(OverflowError, socket.CMSG_SPACE, sys.maxsize) + + +class SCMRightsTest(SendrecvmsgServerTimeoutBase): + # Tests for file descriptor passing on Unix-domain sockets. + + # Invalid file descriptor value that's unlikely to evaluate to a + # real FD even if one of its bytes is replaced with a different + # value (which shouldn't actually happen). + badfd = -0x5555 + + def newFDs(self, n): + # Return a list of n file descriptors for newly-created files + # containing their list indices as ASCII numbers. + fds = [] + for i in range(n): + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + self.addCleanup(os.close, fd) + os.write(fd, str(i).encode()) + fds.append(fd) + return fds + + def checkFDs(self, fds): + # Check that the file descriptors in the given list contain + # their correct list indices as ASCII numbers. + for n, fd in enumerate(fds): + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(os.read(fd, 1024), str(n).encode()) + + def registerRecvmsgResult(self, result): + self.addCleanup(self.closeRecvmsgFDs, result) + + def closeRecvmsgFDs(self, recvmsg_result): + # Close all file descriptors specified in the ancillary data + # of the given return value from recvmsg() or recvmsg_into(). + for cmsg_level, cmsg_type, cmsg_data in recvmsg_result[1]: + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + for fd in fds: + os.close(fd) + + def createAndSendFDs(self, n): + # Send n new file descriptors created by newFDs() to the + # server, with the constant MSG as the non-ancillary data. + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(n)))]), + len(MSG)) + + def checkRecvmsgFDs(self, numfds, result, maxcmsgs=1, ignoreflags=0): + # Check that constant MSG was received with numfds file + # descriptors in a maximum of maxcmsgs control messages (which + # must contain only complete integers). By default, check + # that MSG_CTRUNC is unset, but ignore any flags in + # ignoreflags. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertIsInstance(ancdata, list) + self.assertLessEqual(len(ancdata), maxcmsgs) + fds = array.array("i") + for item in ancdata: + self.assertIsInstance(item, tuple) + cmsg_level, cmsg_type, cmsg_data = item + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data) % SIZEOF_INT, 0) + fds.frombytes(cmsg_data) + + self.assertEqual(len(fds), numfds) + self.checkFDs(fds) + + def testFDPassSimple(self): + # Pass a single FD (array read from bytes object). + self.checkRecvmsgFDs(1, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testFDPassSimple(self): + self.assertEqual( + self.sendmsgToServer( + [MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(1)).tobytes())]), + len(MSG)) + + def testMultipleFDPass(self): + # Pass multiple FDs in a single array. + self.checkRecvmsgFDs(4, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testMultipleFDPass(self): + self.createAndSendFDs(4) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassCMSG_SPACE(self): + # Test using CMSG_SPACE() to calculate ancillary buffer size. + self.checkRecvmsgFDs( + 4, self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(4 * SIZEOF_INT))) + + @testFDPassCMSG_SPACE.client_skip + def _testFDPassCMSG_SPACE(self): + self.createAndSendFDs(4) + + def testFDPassCMSG_LEN(self): + # Test using CMSG_LEN() to calculate ancillary buffer size. + self.checkRecvmsgFDs(1, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(4 * SIZEOF_INT)), + # RFC 3542 says implementations may set + # MSG_CTRUNC if there isn't enough space + # for trailing padding. + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassCMSG_LEN(self): + self.createAndSendFDs(1) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparate(self): + # Pass two FDs in two separate arrays. Arrays may be combined + # into a single control message by the OS. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), 10240), + maxcmsgs=2) + + @testFDPassSeparate.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparate(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparateMinSpace(self): + # Pass two FDs in two separate arrays, receiving them into the + # minimum space for two arrays. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(SIZEOF_INT)), + maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) + + @testFDPassSeparateMinSpace.client_skip + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #12958") + @unittest.skipIf(sys.platform.startswith("aix"), "skipping, see issue #22397") + def _testFDPassSeparateMinSpace(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + def sendAncillaryIfPossible(self, msg, ancdata): + # Try to send msg and ancdata to server, but if the system + # call fails, just send msg with no ancillary data. + try: + nbytes = self.sendmsgToServer([msg], ancdata) + except OSError as e: + # Check that it was the system call that failed + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer([msg]) + self.assertEqual(nbytes, len(msg)) + + @unittest.skipIf(sys.platform == "darwin", "see issue #24725") + def testFDPassEmpty(self): + # Try to pass an empty FD array. Can receive either no array + # or an empty array. + self.checkRecvmsgFDs(0, self.doRecvmsg(self.serv_sock, + len(MSG), 10240), + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassEmpty(self): + self.sendAncillaryIfPossible(MSG, [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + b"")]) + + def testFDPassPartialInt(self): + # Try to pass a truncated FD array. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 1) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + def _testFDPassPartialInt(self): + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [self.badfd]).tobytes()[:-1])]) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassPartialIntInMiddle(self): + # Try to pass two FD arrays, the first of which is truncated. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 2) + fds = array.array("i") + # Arrays may have been combined in a single control message + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.assertLessEqual(len(fds), 2) + self.checkFDs(fds) + + @testFDPassPartialIntInMiddle.client_skip + def _testFDPassPartialIntInMiddle(self): + fd0, fd1 = self.newFDs(2) + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0, self.badfd]).tobytes()[:-1]), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]) + + def checkTruncatedHeader(self, result, ignoreflags=0): + # Check that no ancillary data items are returned when data is + # truncated inside the cmsghdr structure. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no buffer size + # is specified. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG)), + # BSD seems to set MSG_CTRUNC only + # if an item has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTruncNoBufSize(self): + self.createAndSendFDs(1) + + def testCmsgTrunc0(self): + # Check that no ancillary data is received when buffer size is 0. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTrunc0(self): + self.createAndSendFDs(1) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + def testCmsgTrunc1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + + def _testCmsgTrunc1(self): + self.createAndSendFDs(1) + + def testCmsgTrunc2Int(self): + # The cmsghdr structure has at least three members, two of + # which are ints, so we still shouldn't see any ancillary + # data. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + SIZEOF_INT * 2)) + + def _testCmsgTrunc2Int(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Minus1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(0) - 1)) + + def _testCmsgTruncLen0Minus1(self): + self.createAndSendFDs(1) + + # The following tests try to truncate the control message in the + # middle of the FD array. + + def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): + # Check that file descriptor data is truncated to between + # mindata and maxdata bytes when received with buffer size + # ancbuf, and that any complete file descriptor numbers are + # valid. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + if mindata == 0 and ancdata == []: + return + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertGreaterEqual(len(cmsg_data), mindata) + self.assertLessEqual(len(cmsg_data), maxdata) + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.checkFDs(fds) + + def testCmsgTruncLen0(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + + def _testCmsgTruncLen0(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Plus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + + def _testCmsgTruncLen0Plus1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), + maxdata=SIZEOF_INT) + + def _testCmsgTruncLen1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen2Minus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, + maxdata=(2 * SIZEOF_INT) - 1) + + def _testCmsgTruncLen2Minus1(self): + self.createAndSendFDs(2) + + +class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase): + # Test sendmsg() and recvmsg[_into]() using the ancillary data + # features of the RFC 3542 Advanced Sockets API for IPv6. + # Currently we can only handle certain data items (e.g. traffic + # class, hop limit, MTU discovery and fragmentation settings) + # without resorting to unportable means such as the struct module, + # but the tests here are aimed at testing the ancillary data + # handling in sendmsg() and recvmsg() rather than the IPv6 API + # itself. + + # Test value to use when setting hop limit of packet + hop_limit = 2 + + # Test value to use when setting traffic class of packet. + # -1 means "use kernel default". + traffic_class = -1 + + def ancillaryMapping(self, ancdata): + # Given ancillary data list ancdata, return a mapping from + # pairs (cmsg_level, cmsg_type) to corresponding cmsg_data. + # Check that no (level, type) pair appears more than once. + d = {} + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertNotIn((cmsg_level, cmsg_type), d) + d[(cmsg_level, cmsg_type)] = cmsg_data + return d + + def checkHopLimit(self, ancbufsize, maxhop=255, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space. Check that data is MSG, ancillary data is not + # truncated (but ignore any flags in ignoreflags), and hop + # limit is between 0 and maxhop inclusive. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + self.assertIsInstance(ancdata[0], tuple) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimit(self): + # Test receiving the packet hop limit as ancillary data. + self.checkHopLimit(ancbufsize=10240) + + @testRecvHopLimit.client_skip + def _testRecvHopLimit(self): + # Need to wait until server has asked to receive ancillary + # data, as implementations are not required to buffer it + # otherwise. + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimitCMSG_SPACE(self): + # Test receiving hop limit, using CMSG_SPACE to calculate buffer size. + self.checkHopLimit(ancbufsize=socket.CMSG_SPACE(SIZEOF_INT)) + + @testRecvHopLimitCMSG_SPACE.client_skip + def _testRecvHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Could test receiving into buffer sized using CMSG_LEN, but RFC + # 3542 says portable applications must provide space for trailing + # padding. Implementations may set MSG_CTRUNC if there isn't + # enough space for the padding. + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSetHopLimit(self): + # Test setting hop limit on outgoing packet and receiving it + # at the other end. + self.checkHopLimit(ancbufsize=10240, maxhop=self.hop_limit) + + @testSetHopLimit.client_skip + def _testSetHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + def checkTrafficClassAndHopLimit(self, ancbufsize, maxhop=255, + ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space. Check that data is MSG, ancillary + # data is not truncated (but ignore any flags in ignoreflags), + # and traffic class and hop limit are in range (hop limit no + # more than maxhop). + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + self.assertEqual(len(ancdata), 2) + ancmap = self.ancillaryMapping(ancdata) + + tcdata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_TCLASS)] + self.assertEqual(len(tcdata), SIZEOF_INT) + a = array.array("i") + a.frombytes(tcdata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + hldata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT)] + self.assertEqual(len(hldata), SIZEOF_INT) + a = array.array("i") + a.frombytes(hldata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimit(self): + # Test receiving traffic class and hop limit as ancillary data. + self.checkTrafficClassAndHopLimit(ancbufsize=10240) + + @testRecvTrafficClassAndHopLimit.client_skip + def _testRecvTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + # Test receiving traffic class and hop limit, using + # CMSG_SPACE() to calculate buffer size. + self.checkTrafficClassAndHopLimit( + ancbufsize=socket.CMSG_SPACE(SIZEOF_INT) * 2) + + @testRecvTrafficClassAndHopLimitCMSG_SPACE.client_skip + def _testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSetTrafficClassAndHopLimit(self): + # Test setting traffic class and hop limit on outgoing packet, + # and receiving them at the other end. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testSetTrafficClassAndHopLimit.client_skip + def _testSetTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testOddCmsgSize(self): + # Try to send ancillary data with first item one byte too + # long. Fall back to sending with correct size if this fails, + # and check that second item was handled correctly. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testOddCmsgSize.client_skip + def _testOddCmsgSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + try: + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class]).tobytes() + b"\x00"), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + except OSError as e: + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + self.assertEqual(nbytes, len(MSG)) + + # Tests for proper handling of truncated ancillary data + + def checkHopLimitTruncatedHeader(self, ancbufsize, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space, which should be too small to contain the ancillary + # data header (if ancbufsize is None, pass no second argument + # to recvmsg()). Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and no ancillary data is + # returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + args = () if ancbufsize is None else (ancbufsize,) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), *args) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no ancillary + # buffer size is provided. + self.checkHopLimitTruncatedHeader(ancbufsize=None, + # BSD seems to set + # MSG_CTRUNC only if an item + # has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + @testCmsgTruncNoBufSize.client_skip + def _testCmsgTruncNoBufSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc0(self): + # Check that no ancillary data is received when ancillary + # buffer size is zero. + self.checkHopLimitTruncatedHeader(ancbufsize=0, + ignoreflags=socket.MSG_CTRUNC) + + @testSingleCmsgTrunc0.client_skip + def _testSingleCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=1) + + @testSingleCmsgTrunc1.client_skip + def _testSingleCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc2Int(self): + self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) + + @testSingleCmsgTrunc2Int.client_skip + def _testSingleCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncLen0Minus1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) + + @testSingleCmsgTruncLen0Minus1.client_skip + def _testSingleCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncInData(self): + # Test truncation of a control message inside its associated + # data. The message may be returned with its data truncated, + # or not returned at all. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + self.assertLessEqual(len(ancdata), 1) + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + @testSingleCmsgTruncInData.client_skip + def _testSingleCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + def checkTruncatedSecondHeader(self, ancbufsize, ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space, which should be large enough to + # contain the first item, but too small to contain the header + # of the second. Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and only one ancillary + # data item is returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertIn(cmsg_type, {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT}) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + # Try the above test with various buffer sizes. + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc0(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT), + ignoreflags=socket.MSG_CTRUNC) + + @testSecondCmsgTrunc0.client_skip + def _testSecondCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) + + @testSecondCmsgTrunc1.client_skip + def _testSecondCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc2Int(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + 2 * SIZEOF_INT) + + @testSecondCmsgTrunc2Int.client_skip + def _testSecondCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTruncLen0Minus1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(0) - 1) + + @testSecondCmsgTruncLen0Minus1.client_skip + def _testSecondCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecomdCmsgTruncInData(self): + # Test truncation of the second of two control messages inside + # its associated data. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + cmsg_types = {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT} + + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + self.assertEqual(ancdata, []) + + @testSecomdCmsgTruncInData.client_skip + def _testSecomdCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + +# Derive concrete test classes for different socket types. + +class SendrecvmsgUDPTestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +class SendmsgUDPTest(SendmsgConnectionlessTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +class RecvmsgUDPTest(RecvmsgTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +class RecvmsgIntoUDPTest(RecvmsgIntoTests, SendrecvmsgUDPTestBase): + pass + + +class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDP6TestBase): + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer, ignoring scope ID + self.assertEqual(addr1[:-1], addr2[:-1]) + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +class SendmsgUDP6Test(SendmsgConnectionlessTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +class RecvmsgUDP6Test(RecvmsgTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireSocket("AF_INET6", "SOCK_DGRAM") +class RecvmsgIntoUDP6Test(RecvmsgIntoTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +class RecvmsgRFC3542AncillaryUDP6Test(RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 required for this test.') +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +class RecvmsgIntoRFC3542AncillaryUDP6Test(RecvmsgIntoMixin, + RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + + +class SendrecvmsgTCPTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, TCPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +class SendmsgTCPTest(SendmsgStreamTests, SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +class RecvmsgTCPTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +class RecvmsgIntoTCPTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + + +class SendrecvmsgSCTPStreamTestBase(SendrecvmsgSCTPFlagsBase, + SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, SCTPStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +class SendmsgSCTPStreamTest(SendmsgStreamTests, SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +class RecvmsgSCTPStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + +@requireAttrs(socket.socket, "recvmsg_into") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +class RecvmsgIntoSCTPStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + + def testRecvmsgEOF(self): + try: + super(RecvmsgIntoSCTPStreamTest, self).testRecvmsgEOF() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + self.skipTest("sporadic ENOTCONN (kernel issue?) - see issue #13876") + + +class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, UnixStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireAttrs(socket, "AF_UNIX") +class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireAttrs(socket, "AF_UNIX") +class RecvmsgUnixStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireAttrs(socket, "AF_UNIX") +class RecvmsgIntoUnixStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +class RecvmsgSCMRightsStreamTest(SCMRightsTest, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg_into") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +class RecvmsgIntoSCMRightsStreamTest(RecvmsgIntoMixin, SCMRightsTest, + SendrecvmsgUnixStreamTestBase): + pass + + +# Test interrupting the interruptible send/receive methods with a +# signal when a timeout is set. These tests avoid having multiple +# threads alive during the test so that the OS cannot deliver the +# signal to the wrong one. + +class InterruptedTimeoutBase(unittest.TestCase): + # Base class for interrupted send/receive tests. Installs an + # empty handler for SIGALRM and removes it on teardown, along with + # any scheduled alarms. + + def setUp(self): + super().setUp() + orig_alrm_handler = signal.signal(signal.SIGALRM, + lambda signum, frame: 1 / 0) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + + # Timeout for socket operations + timeout = 4.0 + + # Provide setAlarm() method to schedule delivery of SIGALRM after + # given number of seconds, or cancel it if zero, and an + # appropriate time value to use. Use setitimer() if available. + if hasattr(signal, "setitimer"): + alarm_time = 0.05 + + def setAlarm(self, seconds): + signal.setitimer(signal.ITIMER_REAL, seconds) + else: + # Old systems may deliver the alarm up to one second early + alarm_time = 2 + + def setAlarm(self, seconds): + signal.alarm(seconds) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase): + # Test interrupting the recv*() methods with signals when a + # timeout is set. + + def setUp(self): + super().setUp() + self.serv.settimeout(self.timeout) + + def checkInterruptedRecv(self, func, *args, **kwargs): + # Check that func(*args, **kwargs) raises + # errno of EINTR when interrupted by a signal. + try: + self.setAlarm(self.alarm_time) + with self.assertRaises(ZeroDivisionError) as cm: + func(*args, **kwargs) + finally: + self.setAlarm(0) + + def testInterruptedRecvTimeout(self): + self.checkInterruptedRecv(self.serv.recv, 1024) + + def testInterruptedRecvIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recv_into, bytearray(1024)) + + def testInterruptedRecvfromTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom, 1024) + + def testInterruptedRecvfromIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom_into, bytearray(1024)) + + @requireAttrs(socket.socket, "recvmsg") + def testInterruptedRecvmsgTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg, 1024) + + @requireAttrs(socket.socket, "recvmsg_into") + def testInterruptedRecvmsgIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg_into, [bytearray(1024)]) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +class InterruptedSendTimeoutTest(InterruptedTimeoutBase, + ThreadSafeCleanupTestCase, + SocketListeningTestMixin, TCPTestBase): + # Test interrupting the interruptible send*() methods with signals + # when a timeout is set. + + def setUp(self): + super().setUp() + self.serv_conn = self.newSocket() + self.addCleanup(self.serv_conn.close) + # Use a thread to complete the connection, but wait for it to + # terminate before running the test, so that there is only one + # thread to accept the signal. + cli_thread = threading.Thread(target=self.doConnect) + cli_thread.start() + self.cli_conn, addr = self.serv.accept() + self.addCleanup(self.cli_conn.close) + cli_thread.join() + self.serv_conn.settimeout(self.timeout) + + def doConnect(self): + self.serv_conn.connect(self.serv_addr) + + def checkInterruptedSend(self, func, *args, **kwargs): + # Check that func(*args, **kwargs), run in a loop, raises + # OSError with an errno of EINTR when interrupted by a + # signal. + try: + with self.assertRaises(ZeroDivisionError) as cm: + while True: + self.setAlarm(self.alarm_time) + func(*args, **kwargs) + finally: + self.setAlarm(0) + + # Issue #12958: The following tests have problems on OS X prior to 10.7 + @support.requires_mac_ver(10, 7) + def testInterruptedSendTimeout(self): + self.checkInterruptedSend(self.serv_conn.send, b"a"*512) + + @support.requires_mac_ver(10, 7) + def testInterruptedSendtoTimeout(self): + # Passing an actual address here as Python's wrapper for + # sendto() doesn't allow passing a zero-length one; POSIX + # requires that the address is ignored since the socket is + # connection-mode, however. + self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512, + self.serv_addr) + + @support.requires_mac_ver(10, 7) + @requireAttrs(socket.socket, "sendmsg") + def testInterruptedSendmsgTimeout(self): + self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512]) + + +class TCPCloserTest(ThreadedTCPSocketTest): + + def testClose(self): + conn, addr = self.serv.accept() + conn.close() + + sd = self.cli + read, write, err = select.select([sd], [], [], 1.0) + self.assertEqual(read, [sd]) + self.assertEqual(sd.recv(1), b'') + + # Calling close() many times should be safe. + conn.close() + conn.close() + + def _testClose(self): + self.cli.connect((HOST, self.port)) + time.sleep(1.0) + + +class BasicSocketPairTest(SocketPairTest): + + def __init__(self, methodName='runTest'): + SocketPairTest.__init__(self, methodName=methodName) + + def _check_defaults(self, sock): + self.assertIsInstance(sock, socket.socket) + if hasattr(socket, 'AF_UNIX'): + self.assertEqual(sock.family, socket.AF_UNIX) + else: + self.assertEqual(sock.family, socket.AF_INET) + self.assertEqual(sock.type, socket.SOCK_STREAM) + self.assertEqual(sock.proto, 0) + + def _testDefaults(self): + self._check_defaults(self.cli) + + def testDefaults(self): + self._check_defaults(self.serv) + + def testRecv(self): + msg = self.serv.recv(1024) + self.assertEqual(msg, MSG) + + def _testRecv(self): + self.cli.send(MSG) + + def testSend(self): + self.serv.send(MSG) + + def _testSend(self): + msg = self.cli.recv(1024) + self.assertEqual(msg, MSG) + + +class NonBlockingTCPTests(ThreadedTCPSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedTCPSocketTest.__init__(self, methodName=methodName) + + def testSetBlocking(self): + # Testing whether set blocking works + self.serv.setblocking(True) + self.assertIsNone(self.serv.gettimeout()) + self.assertTrue(self.serv.getblocking()) + if fcntl: + self.assertTrue(_is_fd_in_blocking_mode(self.serv)) + + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.assertFalse(self.serv.getblocking()) + if fcntl: + self.assertFalse(_is_fd_in_blocking_mode(self.serv)) + + self.serv.settimeout(None) + self.assertTrue(self.serv.getblocking()) + if fcntl: + self.assertTrue(_is_fd_in_blocking_mode(self.serv)) + + self.serv.settimeout(0) + self.assertFalse(self.serv.getblocking()) + self.assertEqual(self.serv.gettimeout(), 0) + if fcntl: + self.assertFalse(_is_fd_in_blocking_mode(self.serv)) + + self.serv.settimeout(10) + self.assertTrue(self.serv.getblocking()) + self.assertEqual(self.serv.gettimeout(), 10) + if fcntl: + # When a Python socket has a non-zero timeout, it's + # switched internally to a non-blocking mode. + # Later, sock.sendall(), sock.recv(), and other socket + # operations use a `select()` call and handle EWOULDBLOCK/EGAIN + # on all socket operations. That's how timeouts are + # enforced. + self.assertFalse(_is_fd_in_blocking_mode(self.serv)) + + self.serv.settimeout(0) + self.assertFalse(self.serv.getblocking()) + if fcntl: + self.assertFalse(_is_fd_in_blocking_mode(self.serv)) + + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error setting non-blocking mode.") + + def _testSetBlocking(self): + pass + + @support.cpython_only + def testSetBlocking_overflow(self): + # Issue 15989 + import _testcapi + if _testcapi.UINT_MAX >= _testcapi.ULONG_MAX: + self.skipTest('needs UINT_MAX < ULONG_MAX') + self.serv.setblocking(False) + self.assertEqual(self.serv.gettimeout(), 0.0) + self.serv.setblocking(_testcapi.UINT_MAX + 1) + self.assertIsNone(self.serv.gettimeout()) + + _testSetBlocking_overflow = support.cpython_only(_testSetBlocking) + + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), + 'test needs socket.SOCK_NONBLOCK') + @support.requires_linux_version(2, 6, 28) + def testInitNonBlocking(self): + # reinit server socket + self.serv.close() + self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM | + socket.SOCK_NONBLOCK) + self.assertFalse(self.serv.getblocking()) + self.assertEqual(self.serv.gettimeout(), 0) + self.port = support.bind_port(self.serv) + self.serv.listen() + # actual testing + start = time.time() + try: + self.serv.accept() + except OSError: + pass + end = time.time() + self.assertTrue((end - start) < 1.0, "Error creating with non-blocking mode.") + + def _testInitNonBlocking(self): + pass + + def testInheritFlags(self): + # Issue #7995: when calling accept() on a listening socket with a + # timeout, the resulting socket should not be non-blocking. + self.serv.settimeout(10) + try: + conn, addr = self.serv.accept() + message = conn.recv(len(MSG)) + finally: + conn.close() + self.serv.settimeout(None) + + def _testInheritFlags(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + time.sleep(0.5) + self.cli.send(MSG) + + def testAccept(self): + # Testing non-blocking accept + self.serv.setblocking(0) + try: + conn, addr = self.serv.accept() + except OSError: + pass + else: + self.fail("Error trying to do non-blocking accept.") + read, write, err = select.select([self.serv], [], []) + if self.serv in read: + conn, addr = self.serv.accept() + self.assertIsNone(conn.gettimeout()) + conn.close() + else: + self.fail("Error trying to do accept after select.") + + def _testAccept(self): + time.sleep(0.1) + self.cli.connect((HOST, self.port)) + + def testConnect(self): + # Testing non-blocking connect + conn, addr = self.serv.accept() + conn.close() + + def _testConnect(self): + self.cli.settimeout(10) + self.cli.connect((HOST, self.port)) + + def testRecv(self): + # Testing non-blocking recv + conn, addr = self.serv.accept() + conn.setblocking(0) + try: + msg = conn.recv(len(MSG)) + except OSError: + pass + else: + self.fail("Error trying to do non-blocking recv.") + read, write, err = select.select([conn], [], []) + if conn in read: + msg = conn.recv(len(MSG)) + conn.close() + self.assertEqual(msg, MSG) + else: + self.fail("Error during select call to non-blocking socket.") + + def _testRecv(self): + self.cli.connect((HOST, self.port)) + time.sleep(0.1) + self.cli.send(MSG) + + +class FileObjectClassTestCase(SocketConnectedTest): + """Unit tests for the object returned by socket.makefile() + + self.read_file is the io object returned by makefile() on + the client connection. You can read from this file to + get output from the server. + + self.write_file is the io object returned by makefile() on the + server connection. You can write to this file to send output + to the client. + """ + + bufsize = -1 # Use default buffer size + encoding = 'utf-8' + errors = 'strict' + newline = None + + read_mode = 'rb' + read_msg = MSG + write_mode = 'wb' + write_msg = MSG + + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def setUp(self): + self.evt1, self.evt2, self.serv_finished, self.cli_finished = [ + threading.Event() for i in range(4)] + SocketConnectedTest.setUp(self) + self.read_file = self.cli_conn.makefile( + self.read_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def tearDown(self): + self.serv_finished.set() + self.read_file.close() + self.assertTrue(self.read_file.closed) + self.read_file = None + SocketConnectedTest.tearDown(self) + + def clientSetUp(self): + SocketConnectedTest.clientSetUp(self) + self.write_file = self.serv_conn.makefile( + self.write_mode, self.bufsize, + encoding = self.encoding, + errors = self.errors, + newline = self.newline) + + def clientTearDown(self): + self.cli_finished.set() + self.write_file.close() + self.assertTrue(self.write_file.closed) + self.write_file = None + SocketConnectedTest.clientTearDown(self) + + def testReadAfterTimeout(self): + # Issue #7322: A file object must disallow further reads + # after a timeout has occurred. + self.cli_conn.settimeout(1) + self.read_file.read(3) + # First read raises a timeout + self.assertRaises(socket.timeout, self.read_file.read, 1) + # Second read is disallowed + with self.assertRaises(OSError) as ctx: + self.read_file.read(1) + self.assertIn("cannot read from timed out object", str(ctx.exception)) + + def _testReadAfterTimeout(self): + self.write_file.write(self.write_msg[0:3]) + self.write_file.flush() + self.serv_finished.wait() + + def testSmallRead(self): + # Performing small file read test + first_seg = self.read_file.read(len(self.read_msg)-3) + second_seg = self.read_file.read(3) + msg = first_seg + second_seg + self.assertEqual(msg, self.read_msg) + + def _testSmallRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testFullRead(self): + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testFullRead(self): + self.write_file.write(self.write_msg) + self.write_file.close() + + def testUnbufferedRead(self): + # Performing unbuffered file read test + buf = type(self.read_msg)() + while 1: + char = self.read_file.read(1) + if not char: + break + buf += char + self.assertEqual(buf, self.read_msg) + + def _testUnbufferedRead(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testReadline(self): + # Performing file readline test + line = self.read_file.readline() + self.assertEqual(line, self.read_msg) + + def _testReadline(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testCloseAfterMakefile(self): + # The file returned by makefile should keep the socket open. + self.cli_conn.close() + # read until EOF + msg = self.read_file.read() + self.assertEqual(msg, self.read_msg) + + def _testCloseAfterMakefile(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileAfterMakefileClose(self): + self.read_file.close() + msg = self.cli_conn.recv(len(MSG)) + if isinstance(self.read_msg, str): + msg = msg.decode() + self.assertEqual(msg, self.read_msg) + + def _testMakefileAfterMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testClosedAttr(self): + self.assertTrue(not self.read_file.closed) + + def _testClosedAttr(self): + self.assertTrue(not self.write_file.closed) + + def testAttributes(self): + self.assertEqual(self.read_file.mode, self.read_mode) + self.assertEqual(self.read_file.name, self.cli_conn.fileno()) + + def _testAttributes(self): + self.assertEqual(self.write_file.mode, self.write_mode) + self.assertEqual(self.write_file.name, self.serv_conn.fileno()) + + def testRealClose(self): + self.read_file.close() + self.assertRaises(ValueError, self.read_file.fileno) + self.cli_conn.close() + self.assertRaises(OSError, self.cli_conn.getsockname) + + def _testRealClose(self): + pass + + +class UnbufferedFileObjectClassTestCase(FileObjectClassTestCase): + + """Repeat the tests from FileObjectClassTestCase with bufsize==0. + + In this case (and in this case only), it should be possible to + create a file object, read a line from it, create another file + object, read another line from it, without loss of data in the + first file object's buffer. Note that http.client relies on this + when reading multiple requests from the same socket.""" + + bufsize = 0 # Use unbuffered mode + + def testUnbufferedReadline(self): + # Read a line, create a new file object, read another line with it + line = self.read_file.readline() # first line + self.assertEqual(line, b"A. " + self.write_msg) # first line + self.read_file = self.cli_conn.makefile('rb', 0) + line = self.read_file.readline() # second line + self.assertEqual(line, b"B. " + self.write_msg) # second line + + def _testUnbufferedReadline(self): + self.write_file.write(b"A. " + self.write_msg) + self.write_file.write(b"B. " + self.write_msg) + self.write_file.flush() + + def testMakefileClose(self): + # The file returned by makefile should keep the socket open... + self.cli_conn.close() + msg = self.cli_conn.recv(1024) + self.assertEqual(msg, self.read_msg) + # ...until the file is itself closed + self.read_file.close() + self.assertRaises(OSError, self.cli_conn.recv, 1024) + + def _testMakefileClose(self): + self.write_file.write(self.write_msg) + self.write_file.flush() + + def testMakefileCloseSocketDestroy(self): + refcount_before = sys.getrefcount(self.cli_conn) + self.read_file.close() + refcount_after = sys.getrefcount(self.cli_conn) + self.assertEqual(refcount_before - 1, refcount_after) + + def _testMakefileCloseSocketDestroy(self): + pass + + # Non-blocking ops + # NOTE: to set `read_file` as non-blocking, we must call + # `cli_conn.setblocking` and vice-versa (see setUp / clientSetUp). + + def testSmallReadNonBlocking(self): + self.cli_conn.setblocking(False) + self.assertEqual(self.read_file.readinto(bytearray(10)), None) + self.assertEqual(self.read_file.read(len(self.read_msg) - 3), None) + self.evt1.set() + self.evt2.wait(1.0) + first_seg = self.read_file.read(len(self.read_msg) - 3) + if first_seg is None: + # Data not arrived (can happen under Windows), wait a bit + time.sleep(0.5) + first_seg = self.read_file.read(len(self.read_msg) - 3) + buf = bytearray(10) + n = self.read_file.readinto(buf) + self.assertEqual(n, 3) + msg = first_seg + buf[:n] + self.assertEqual(msg, self.read_msg) + self.assertEqual(self.read_file.readinto(bytearray(16)), None) + self.assertEqual(self.read_file.read(1), None) + + def _testSmallReadNonBlocking(self): + self.evt1.wait(1.0) + self.write_file.write(self.write_msg) + self.write_file.flush() + self.evt2.set() + # Avoid closing the socket before the server test has finished, + # otherwise system recv() will return 0 instead of EWOULDBLOCK. + self.serv_finished.wait(5.0) + + def testWriteNonBlocking(self): + self.cli_finished.wait(5.0) + # The client thread can't skip directly - the SkipTest exception + # would appear as a failure. + if self.serv_skipped: + self.skipTest(self.serv_skipped) + + def _testWriteNonBlocking(self): + self.serv_skipped = None + self.serv_conn.setblocking(False) + # Try to saturate the socket buffer pipe with repeated large writes. + BIG = b"x" * support.SOCK_MAX_SIZE + LIMIT = 10 + # The first write() succeeds since a chunk of data can be buffered + n = self.write_file.write(BIG) + self.assertGreater(n, 0) + for i in range(LIMIT): + n = self.write_file.write(BIG) + if n is None: + # Succeeded + break + self.assertGreater(n, 0) + else: + # Let us know that this test didn't manage to establish + # the expected conditions. This is not a failure in itself but, + # if it happens repeatedly, the test should be fixed. + self.serv_skipped = "failed to saturate the socket buffer" + + +class LineBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 1 # Default-buffered for reading; line-buffered for writing + + +class SmallBufferedFileObjectClassTestCase(FileObjectClassTestCase): + + bufsize = 2 # Exercise the buffering code + + +class UnicodeReadFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'wb' + write_msg = MSG + newline = '' + + +class UnicodeWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'rb' + read_msg = MSG + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class UnicodeReadWriteFileObjectClassTestCase(FileObjectClassTestCase): + """Tests for socket.makefile() in text mode (rather than binary)""" + + read_mode = 'r' + read_msg = MSG.decode('utf-8') + write_mode = 'w' + write_msg = MSG.decode('utf-8') + newline = '' + + +class NetworkConnectionTest(object): + """Prove network connection.""" + + def clientSetUp(self): + # We're inherited below by BasicTCPTest2, which also inherits + # BasicTCPTest, which defines self.port referenced below. + self.cli = socket.create_connection((HOST, self.port)) + self.serv_conn = self.cli + +class BasicTCPTest2(NetworkConnectionTest, BasicTCPTest): + """Tests that NetworkConnection does not break existing TCP functionality. + """ + +class NetworkConnectionNoServer(unittest.TestCase): + + class MockSocket(socket.socket): + def connect(self, *args): + raise socket.timeout('timed out') + + @contextlib.contextmanager + def mocked_socket_module(self): + """Return a socket which times out on connect""" + old_socket = socket.socket + socket.socket = self.MockSocket + try: + yield + finally: + socket.socket = old_socket + + def test_connect(self): + port = support.find_unused_port() + cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(cli.close) + with self.assertRaises(OSError) as cm: + cli.connect((HOST, port)) + self.assertEqual(cm.exception.errno, errno.ECONNREFUSED) + + def test_create_connection(self): + # Issue #9792: errors raised by create_connection() should have + # a proper errno attribute. + port = support.find_unused_port() + with self.assertRaises(OSError) as cm: + socket.create_connection((HOST, port)) + + # Issue #16257: create_connection() calls getaddrinfo() against + # 'localhost'. This may result in an IPV6 addr being returned + # as well as an IPV4 one: + # >>> socket.getaddrinfo('localhost', port, 0, SOCK_STREAM) + # >>> [(2, 2, 0, '', ('127.0.0.1', 41230)), + # (26, 2, 0, '', ('::1', 41230, 0, 0))] + # + # create_connection() enumerates through all the addresses returned + # and if it doesn't successfully bind to any of them, it propagates + # the last exception it encountered. + # + # On Solaris, ENETUNREACH is returned in this circumstance instead + # of ECONNREFUSED. So, if that errno exists, add it to our list of + # expected errnos. + expected_errnos = [ errno.ECONNREFUSED, ] + if hasattr(errno, 'ENETUNREACH'): + expected_errnos.append(errno.ENETUNREACH) + if hasattr(errno, 'EADDRNOTAVAIL'): + # bpo-31910: socket.create_connection() fails randomly + # with EADDRNOTAVAIL on Travis CI + expected_errnos.append(errno.EADDRNOTAVAIL) + + self.assertIn(cm.exception.errno, expected_errnos) + + def test_create_connection_timeout(self): + # Issue #9792: create_connection() should not recast timeout errors + # as generic socket errors. + with self.mocked_socket_module(): + with self.assertRaises(socket.timeout): + socket.create_connection((HOST, 1234)) + + +class NetworkConnectionAttributesTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.source_port = support.find_unused_port() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def _justAccept(self): + conn, addr = self.serv.accept() + conn.close() + + testFamily = _justAccept + def _testFamily(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.family, 2) + + testSourceAddress = _justAccept + def _testSourceAddress(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30, + source_address=('', self.source_port)) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.getsockname()[1], self.source_port) + # The port number being used is sufficient to show that the bind() + # call happened. + + testTimeoutDefault = _justAccept + def _testTimeoutDefault(self): + # passing no explicit timeout uses socket's global default + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(42) + try: + self.cli = socket.create_connection((HOST, self.port)) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), 42) + + testTimeoutNone = _justAccept + def _testTimeoutNone(self): + # None timeout means the same as sock.settimeout(None) + self.assertTrue(socket.getdefaulttimeout() is None) + socket.setdefaulttimeout(30) + try: + self.cli = socket.create_connection((HOST, self.port), timeout=None) + self.addCleanup(self.cli.close) + finally: + socket.setdefaulttimeout(None) + self.assertEqual(self.cli.gettimeout(), None) + + testTimeoutValueNamed = _justAccept + def _testTimeoutValueNamed(self): + self.cli = socket.create_connection((HOST, self.port), timeout=30) + self.assertEqual(self.cli.gettimeout(), 30) + + testTimeoutValueNonamed = _justAccept + def _testTimeoutValueNonamed(self): + self.cli = socket.create_connection((HOST, self.port), 30) + self.addCleanup(self.cli.close) + self.assertEqual(self.cli.gettimeout(), 30) + + +class NetworkConnectionBehaviourTest(SocketTCPTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketTCPTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + def testInsideTimeout(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + time.sleep(3) + conn.send(b"done!") + testOutsideTimeout = testInsideTimeout + + def _testInsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port)) + data = sock.recv(5) + self.assertEqual(data, b"done!") + + def _testOutsideTimeout(self): + self.cli = sock = socket.create_connection((HOST, self.port), timeout=1) + self.assertRaises(socket.timeout, lambda: sock.recv(5)) + + +class TCPTimeoutTest(SocketTCPTest): + + def testTCPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.accept() + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (TCP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of error (TCP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (TCP)") + if not ok: + self.fail("accept() returned success when we did not expect it") + + @unittest.skipUnless(hasattr(signal, 'alarm'), + 'test needs signal.alarm()') + def testInterruptedTimeout(self): + # XXX I don't know how to do this test on MSWindows or any other + # platform that doesn't support signal.alarm() or os.kill(), though + # the bug should have existed on all platforms. + self.serv.settimeout(5.0) # must be longer than alarm + class Alarm(Exception): + pass + def alarm_handler(signal, frame): + raise Alarm + old_alarm = signal.signal(signal.SIGALRM, alarm_handler) + try: + try: + signal.alarm(2) # POSIX allows alarm to be up to 1 second early + foo = self.serv.accept() + except socket.timeout: + self.fail("caught timeout instead of Alarm") + except Alarm: + pass + except: + self.fail("caught other exception instead of Alarm:" + " %s(%s):\n%s" % + (sys.exc_info()[:2] + (traceback.format_exc(),))) + else: + self.fail("nothing caught") + finally: + signal.alarm(0) # shut off alarm + except Alarm: + self.fail("got Alarm in wrong place") + finally: + # no alarm can be pending. Safe to restore old handler. + signal.signal(signal.SIGALRM, old_alarm) + +class UDPTimeoutTest(SocketUDPTest): + + def testUDPTimeout(self): + def raise_timeout(*args, **kwargs): + self.serv.settimeout(1.0) + self.serv.recv(1024) + self.assertRaises(socket.timeout, raise_timeout, + "Error generating a timeout exception (UDP)") + + def testTimeoutZero(self): + ok = False + try: + self.serv.settimeout(0.0) + foo = self.serv.recv(1024) + except socket.timeout: + self.fail("caught timeout instead of error (UDP)") + except OSError: + ok = True + except: + self.fail("caught unexpected exception (UDP)") + if not ok: + self.fail("recv() returned success when we did not expect it") + +class TestExceptions(unittest.TestCase): + + def testExceptionTree(self): + self.assertTrue(issubclass(OSError, Exception)) + self.assertTrue(issubclass(socket.herror, OSError)) + self.assertTrue(issubclass(socket.gaierror, OSError)) + self.assertTrue(issubclass(socket.timeout, OSError)) + + def test_setblocking_invalidfd(self): + # Regression test for issue #28471 + + sock0 = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + sock = socket.socket( + socket.AF_INET, socket.SOCK_STREAM, 0, sock0.fileno()) + sock0.close() + self.addCleanup(sock.detach) + + with self.assertRaises(OSError): + sock.setblocking(False) + + +@unittest.skipUnless(sys.platform == 'linux', 'Linux specific test') +class TestLinuxAbstractNamespace(unittest.TestCase): + + UNIX_PATH_MAX = 108 + + def testLinuxAbstractNamespace(self): + address = b"\x00python-test-hello\x00\xff" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s1: + s1.bind(address) + s1.listen() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s2: + s2.connect(s1.getsockname()) + with s1.accept()[0] as s3: + self.assertEqual(s1.getsockname(), address) + self.assertEqual(s2.getpeername(), address) + + def testMaxName(self): + address = b"\x00" + b"h" * (self.UNIX_PATH_MAX - 1) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(address) + self.assertEqual(s.getsockname(), address) + + def testNameOverflow(self): + address = "\x00" + "h" * self.UNIX_PATH_MAX + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + self.assertRaises(OSError, s.bind, address) + + def testStrName(self): + # Check that an abstract name can be passed as a string. + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s.bind("\x00python\x00test\x00") + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + finally: + s.close() + + def testBytearrayName(self): + # Check that an abstract name can be passed as a bytearray. + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(bytearray(b"\x00python\x00test\x00")) + self.assertEqual(s.getsockname(), b"\x00python\x00test\x00") + +@unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'test needs socket.AF_UNIX') +class TestUnixDomain(unittest.TestCase): + + def setUp(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + def tearDown(self): + self.sock.close() + + def encoded(self, path): + # Return the given path encoded in the file system encoding, + # or skip the test if this is not possible. + try: + return os.fsencode(path) + except UnicodeEncodeError: + self.skipTest( + "Pathname {0!a} cannot be represented in file " + "system encoding {1!r}".format( + path, sys.getfilesystemencoding())) + + def bind(self, sock, path): + # Bind the socket + try: + support.bind_unix_socket(sock, path) + except OSError as e: + if str(e) == "AF_UNIX path too long": + self.skipTest( + "Pathname {0!a} is too long to serve as an AF_UNIX path" + .format(path)) + else: + raise + + def testUnbound(self): + # Issue #30205 (note getsockname() can return None on OS X) + self.assertIn(self.sock.getsockname(), ('', None)) + + def testStrAddr(self): + # Test binding to and retrieving a normal string pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testBytesAddr(self): + # Test binding to a bytes pathname. + path = os.path.abspath(support.TESTFN) + self.bind(self.sock, self.encoded(path)) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testSurrogateescapeBind(self): + # Test binding to a valid non-ASCII pathname, with the + # non-ASCII bytes supplied using surrogateescape encoding. + path = os.path.abspath(support.TESTFN_UNICODE) + b = self.encoded(path) + self.bind(self.sock, b.decode("ascii", "surrogateescape")) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + def testUnencodableAddr(self): + # Test binding to a pathname that cannot be encoded in the + # file system encoding. + if support.TESTFN_UNENCODABLE is None: + self.skipTest("No unencodable filename available") + path = os.path.abspath(support.TESTFN_UNENCODABLE) + self.bind(self.sock, path) + self.addCleanup(support.unlink, path) + self.assertEqual(self.sock.getsockname(), path) + + +class BufferIOTest(SocketConnectedTest): + """ + Test the buffer versions of socket.recv() and socket.send(). + """ + def __init__(self, methodName='runTest'): + SocketConnectedTest.__init__(self, methodName=methodName) + + def testRecvIntoArray(self): + buf = array.array("B", [0] * len(MSG)) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + buf = buf.tobytes() + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvIntoBytearray(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoBytearray = _testRecvIntoArray + + def testRecvIntoMemoryview(self): + buf = bytearray(1024) + nbytes = self.cli_conn.recv_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvIntoMemoryview = _testRecvIntoArray + + def testRecvFromIntoArray(self): + buf = array.array("B", [0] * len(MSG)) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + buf = buf.tobytes() + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + def _testRecvFromIntoArray(self): + buf = bytes(MSG) + self.serv_conn.send(buf) + + def testRecvFromIntoBytearray(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(buf) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoBytearray = _testRecvFromIntoArray + + def testRecvFromIntoMemoryview(self): + buf = bytearray(1024) + nbytes, addr = self.cli_conn.recvfrom_into(memoryview(buf)) + self.assertEqual(nbytes, len(MSG)) + msg = buf[:len(MSG)] + self.assertEqual(msg, MSG) + + _testRecvFromIntoMemoryview = _testRecvFromIntoArray + + def testRecvFromIntoSmallBuffer(self): + # See issue #20246. + buf = bytearray(8) + self.assertRaises(ValueError, self.cli_conn.recvfrom_into, buf, 1024) + + def _testRecvFromIntoSmallBuffer(self): + self.serv_conn.send(MSG) + + def testRecvFromIntoEmptyBuffer(self): + buf = bytearray() + self.cli_conn.recvfrom_into(buf) + self.cli_conn.recvfrom_into(buf, 0) + + _testRecvFromIntoEmptyBuffer = _testRecvFromIntoArray + + +TIPC_STYPE = 2000 +TIPC_LOWER = 200 +TIPC_UPPER = 210 + +def isTipcAvailable(): + """Check if the TIPC module is loaded + + The TIPC module is not loaded automatically on Ubuntu and probably + other Linux distros. + """ + if not hasattr(socket, "AF_TIPC"): + return False + try: + f = open("/proc/modules") + except (FileNotFoundError, IsADirectoryError, PermissionError): + # It's ok if the file does not exist, is a directory or if we + # have not the permission to read it. + return False + with f: + for line in f: + if line.startswith("tipc "): + return True + return False + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCTest(unittest.TestCase): + def testRDM(self): + srv = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + cli = socket.socket(socket.AF_TIPC, socket.SOCK_RDM) + self.addCleanup(srv.close) + self.addCleanup(cli.close) + + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + srv.bind(srvaddr) + + sendaddr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + cli.sendto(MSG, sendaddr) + + msg, recvaddr = srv.recvfrom(1024) + + self.assertEqual(cli.getsockname(), recvaddr) + self.assertEqual(msg, MSG) + + +@unittest.skipUnless(isTipcAvailable(), + "TIPC module is not loaded, please 'sudo modprobe tipc'") +class TIPCThreadableTest(unittest.TestCase, ThreadableTest): + def __init__(self, methodName = 'runTest'): + unittest.TestCase.__init__(self, methodName = methodName) + ThreadableTest.__init__(self) + + def setUp(self): + self.srv = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.srv.close) + self.srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srvaddr = (socket.TIPC_ADDR_NAMESEQ, TIPC_STYPE, + TIPC_LOWER, TIPC_UPPER) + self.srv.bind(srvaddr) + self.srv.listen() + self.serverExplicitReady() + self.conn, self.connaddr = self.srv.accept() + self.addCleanup(self.conn.close) + + def clientSetUp(self): + # There is a hittable race between serverExplicitReady() and the + # accept() call; sleep a little while to avoid it, otherwise + # we could get an exception + time.sleep(0.1) + self.cli = socket.socket(socket.AF_TIPC, socket.SOCK_STREAM) + self.addCleanup(self.cli.close) + addr = (socket.TIPC_ADDR_NAME, TIPC_STYPE, + TIPC_LOWER + int((TIPC_UPPER - TIPC_LOWER) / 2), 0) + self.cli.connect(addr) + self.cliaddr = self.cli.getsockname() + + def testStream(self): + msg = self.conn.recv(1024) + self.assertEqual(msg, MSG) + self.assertEqual(self.cliaddr, self.connaddr) + + def _testStream(self): + self.cli.send(MSG) + self.cli.close() + + +class ContextManagersTest(ThreadedTCPSocketTest): + + def _testSocketClass(self): + # base test + with socket.socket() as sock: + self.assertFalse(sock._closed) + self.assertTrue(sock._closed) + # close inside with block + with socket.socket() as sock: + sock.close() + self.assertTrue(sock._closed) + # exception inside with block + with socket.socket() as sock: + self.assertRaises(OSError, sock.sendall, b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionBase(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionBase(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + self.assertFalse(sock._closed) + sock.sendall(b'foo') + self.assertEqual(sock.recv(1024), b'foo') + self.assertTrue(sock._closed) + + def testCreateConnectionClose(self): + conn, addr = self.serv.accept() + self.addCleanup(conn.close) + data = conn.recv(1024) + conn.sendall(data) + + def _testCreateConnectionClose(self): + address = self.serv.getsockname() + with socket.create_connection(address) as sock: + sock.close() + self.assertTrue(sock._closed) + self.assertRaises(OSError, sock.sendall, b'foo') + + +class InheritanceTest(unittest.TestCase): + @unittest.skipUnless(hasattr(socket, "SOCK_CLOEXEC"), + "SOCK_CLOEXEC not defined") + @support.requires_linux_version(2, 6, 28) + def test_SOCK_CLOEXEC(self): + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: + self.assertEqual(s.type, socket.SOCK_STREAM) + self.assertFalse(s.get_inheritable()) + + def test_default_inheritable(self): + sock = socket.socket() + with sock: + self.assertEqual(sock.get_inheritable(), False) + + def test_dup(self): + sock = socket.socket() + with sock: + newsock = sock.dup() + sock.close() + with newsock: + self.assertEqual(newsock.get_inheritable(), False) + + def test_set_inheritable(self): + sock = socket.socket() + with sock: + sock.set_inheritable(True) + self.assertEqual(sock.get_inheritable(), True) + + sock.set_inheritable(False) + self.assertEqual(sock.get_inheritable(), False) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(sock.get_inheritable(), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(sock.get_inheritable(), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + sock = socket.socket() + with sock: + fd = sock.fileno() + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + sock.set_inheritable(True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + + def test_socketpair(self): + s1, s2 = socket.socketpair() + self.addCleanup(s1.close) + self.addCleanup(s2.close) + self.assertEqual(s1.get_inheritable(), False) + self.assertEqual(s2.get_inheritable(), False) + + +@unittest.skipUnless(hasattr(socket, "SOCK_NONBLOCK"), + "SOCK_NONBLOCK not defined") +class NonblockConstantTest(unittest.TestCase): + def checkNonblock(self, s, nonblock=True, timeout=0.0): + if nonblock: + self.assertEqual(s.type, socket.SOCK_STREAM) + self.assertEqual(s.gettimeout(), timeout) + self.assertTrue( + fcntl.fcntl(s, fcntl.F_GETFL, os.O_NONBLOCK) & os.O_NONBLOCK) + if timeout == 0: + # timeout == 0: means that getblocking() must be False. + self.assertFalse(s.getblocking()) + else: + # If timeout > 0, the socket will be in a "blocking" mode + # from the standpoint of the Python API. For Python socket + # object, "blocking" means that operations like 'sock.recv()' + # will block. Internally, file descriptors for + # "blocking" Python sockets *with timeouts* are in a + # *non-blocking* mode, and 'sock.recv()' uses 'select()' + # and handles EWOULDBLOCK/EAGAIN to enforce the timeout. + self.assertTrue(s.getblocking()) + else: + self.assertEqual(s.type, socket.SOCK_STREAM) + self.assertEqual(s.gettimeout(), None) + self.assertFalse( + fcntl.fcntl(s, fcntl.F_GETFL, os.O_NONBLOCK) & os.O_NONBLOCK) + self.assertTrue(s.getblocking()) + + @support.requires_linux_version(2, 6, 28) + def test_SOCK_NONBLOCK(self): + # a lot of it seems silly and redundant, but I wanted to test that + # changing back and forth worked ok + with socket.socket(socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) as s: + self.checkNonblock(s) + s.setblocking(1) + self.checkNonblock(s, nonblock=False) + s.setblocking(0) + self.checkNonblock(s) + s.settimeout(None) + self.checkNonblock(s, nonblock=False) + s.settimeout(2.0) + self.checkNonblock(s, timeout=2.0) + s.setblocking(1) + self.checkNonblock(s, nonblock=False) + # defaulttimeout + t = socket.getdefaulttimeout() + socket.setdefaulttimeout(0.0) + with socket.socket() as s: + self.checkNonblock(s) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(2.0) + with socket.socket() as s: + self.checkNonblock(s, timeout=2.0) + socket.setdefaulttimeout(None) + with socket.socket() as s: + self.checkNonblock(s, False) + socket.setdefaulttimeout(t) + + +@unittest.skipUnless(os.name == "nt", "Windows specific") +@unittest.skipUnless(multiprocessing, "need multiprocessing") +class TestSocketSharing(SocketTCPTest): + # This must be classmethod and not staticmethod or multiprocessing + # won't be able to bootstrap it. + @classmethod + def remoteProcessServer(cls, q): + # Recreate socket from shared data + sdata = q.get() + message = q.get() + + s = socket.fromshare(sdata) + s2, c = s.accept() + + # Send the message + s2.sendall(message) + s2.close() + s.close() + + def testShare(self): + # Transfer the listening server socket to another process + # and service it from there. + + # Create process: + q = multiprocessing.Queue() + p = multiprocessing.Process(target=self.remoteProcessServer, args=(q,)) + p.start() + + # Get the shared socket data + data = self.serv.share(p.pid) + + # Pass the shared socket to the other process + addr = self.serv.getsockname() + self.serv.close() + q.put(data) + + # The data that the server will send us + message = b"slapmahfro" + q.put(message) + + # Connect + s = socket.create_connection(addr) + # listen for the data + m = [] + while True: + data = s.recv(100) + if not data: + break + m.append(data) + s.close() + received = b"".join(m) + self.assertEqual(received, message) + p.join() + + def testShareLength(self): + data = self.serv.share(os.getpid()) + self.assertRaises(ValueError, socket.fromshare, data[:-1]) + self.assertRaises(ValueError, socket.fromshare, data+b"foo") + + def compareSockets(self, org, other): + # socket sharing is expected to work only for blocking socket + # since the internal python timeout value isn't transferred. + self.assertEqual(org.gettimeout(), None) + self.assertEqual(org.gettimeout(), other.gettimeout()) + + self.assertEqual(org.family, other.family) + self.assertEqual(org.type, other.type) + # If the user specified "0" for proto, then + # internally windows will have picked the correct value. + # Python introspection on the socket however will still return + # 0. For the shared socket, the python value is recreated + # from the actual value, so it may not compare correctly. + if org.proto != 0: + self.assertEqual(org.proto, other.proto) + + def testShareLocal(self): + data = self.serv.share(os.getpid()) + s = socket.fromshare(data) + try: + self.compareSockets(self.serv, s) + finally: + s.close() + + def testTypes(self): + families = [socket.AF_INET, socket.AF_INET6] + types = [socket.SOCK_STREAM, socket.SOCK_DGRAM] + for f in families: + for t in types: + try: + source = socket.socket(f, t) + except OSError: + continue # This combination is not supported + try: + data = source.share(os.getpid()) + shared = socket.fromshare(data) + try: + self.compareSockets(source, shared) + finally: + shared.close() + finally: + source.close() + + +class SendfileUsingSendTest(ThreadedTCPSocketTest): + """ + Test the send() implementation of socket.sendfile(). + """ + + FILESIZE = (10 * 1024 * 1024) # 10 MiB + BUFSIZE = 8192 + FILEDATA = b"" + TIMEOUT = 2 + + @classmethod + def setUpClass(cls): + def chunks(total, step): + assert total >= step + while total > step: + yield step + total -= step + if total: + yield total + + chunk = b"".join([random.choice(string.ascii_letters).encode() + for i in range(cls.BUFSIZE)]) + with open(support.TESTFN, 'wb') as f: + for csize in chunks(cls.FILESIZE, cls.BUFSIZE): + f.write(chunk) + with open(support.TESTFN, 'rb') as f: + cls.FILEDATA = f.read() + assert len(cls.FILEDATA) == cls.FILESIZE + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + + def accept_conn(self): + self.serv.settimeout(self.TIMEOUT) + conn, addr = self.serv.accept() + conn.settimeout(self.TIMEOUT) + self.addCleanup(conn.close) + return conn + + def recv_data(self, conn): + received = [] + while True: + chunk = conn.recv(self.BUFSIZE) + if not chunk: + break + received.append(chunk) + return b''.join(received) + + def meth_from_sock(self, sock): + # Depending on the mixin class being run return either send() + # or sendfile() method implementation. + return getattr(sock, "_sendfile_use_send") + + # regular file + + def _testRegularFile(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, self.FILESIZE) + self.assertEqual(file.tell(), self.FILESIZE) + + def testRegularFile(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # non regular file + + def _testNonRegularFile(self): + address = self.serv.getsockname() + file = io.BytesIO(self.FILEDATA) + with socket.create_connection(address) as sock, file as file: + sent = sock.sendfile(file) + self.assertEqual(sent, self.FILESIZE) + self.assertEqual(file.tell(), self.FILESIZE) + self.assertRaises(socket._GiveupOnSendfile, + sock._sendfile_use_sendfile, file) + + def testNonRegularFile(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # empty file + + def _testEmptyFileSend(self): + address = self.serv.getsockname() + filename = support.TESTFN + "2" + with open(filename, 'wb'): + self.addCleanup(support.unlink, filename) + file = open(filename, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, 0) + self.assertEqual(file.tell(), 0) + + def testEmptyFileSend(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(data, b"") + + # offset + + def _testOffset(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file, offset=5000) + self.assertEqual(sent, self.FILESIZE - 5000) + self.assertEqual(file.tell(), self.FILESIZE) + + def testOffset(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE - 5000) + self.assertEqual(data, self.FILEDATA[5000:]) + + # count + + def _testCount(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 5000007 + meth = self.meth_from_sock(sock) + sent = meth(file, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count) + + def testCount(self): + count = 5000007 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[:count]) + + # count small + + def _testCountSmall(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 1 + meth = self.meth_from_sock(sock) + sent = meth(file, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count) + + def testCountSmall(self): + count = 1 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[:count]) + + # count + offset + + def _testCountWithOffset(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + count = 100007 + meth = self.meth_from_sock(sock) + sent = meth(file, offset=2007, count=count) + self.assertEqual(sent, count) + self.assertEqual(file.tell(), count + 2007) + + def testCountWithOffset(self): + count = 100007 + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), count) + self.assertEqual(data, self.FILEDATA[2007:count+2007]) + + # non blocking sockets are not supposed to work + + def _testNonBlocking(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address) as sock, file as file: + sock.setblocking(False) + meth = self.meth_from_sock(sock) + self.assertRaises(ValueError, meth, file) + self.assertRaises(ValueError, sock.sendfile, file) + + def testNonBlocking(self): + conn = self.accept_conn() + if conn.recv(8192): + self.fail('was not supposed to receive any data') + + # timeout (non-triggered) + + def _testWithTimeout(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=2) as sock, file as file: + meth = self.meth_from_sock(sock) + sent = meth(file) + self.assertEqual(sent, self.FILESIZE) + + def testWithTimeout(self): + conn = self.accept_conn() + data = self.recv_data(conn) + self.assertEqual(len(data), self.FILESIZE) + self.assertEqual(data, self.FILEDATA) + + # timeout (triggered) + + def _testWithTimeoutTriggeredSend(self): + address = self.serv.getsockname() + file = open(support.TESTFN, 'rb') + with socket.create_connection(address, timeout=0.01) as sock, \ + file as file: + meth = self.meth_from_sock(sock) + self.assertRaises(socket.timeout, meth, file) + + def testWithTimeoutTriggeredSend(self): + conn = self.accept_conn() + conn.recv(88192) + + # errors + + def _test_errors(self): + pass + + def test_errors(self): + with open(support.TESTFN, 'rb') as file: + with socket.socket(type=socket.SOCK_DGRAM) as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex( + ValueError, "SOCK_STREAM", meth, file) + with open(support.TESTFN, 'rt') as file: + with socket.socket() as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex( + ValueError, "binary mode", meth, file) + with open(support.TESTFN, 'rb') as file: + with socket.socket() as s: + meth = self.meth_from_sock(s) + self.assertRaisesRegex(TypeError, "positive integer", + meth, file, count='2') + self.assertRaisesRegex(TypeError, "positive integer", + meth, file, count=0.1) + self.assertRaisesRegex(ValueError, "positive integer", + meth, file, count=0) + self.assertRaisesRegex(ValueError, "positive integer", + meth, file, count=-1) + + +@unittest.skipUnless(hasattr(os, "sendfile"), + 'os.sendfile() required for this test.') +class SendfileUsingSendfileTest(SendfileUsingSendTest): + """ + Test the sendfile() implementation of socket.sendfile(). + """ + def meth_from_sock(self, sock): + return getattr(sock, "_sendfile_use_sendfile") + + +@unittest.skipUnless(HAVE_SOCKET_ALG, 'AF_ALG required') +class LinuxKernelCryptoAPI(unittest.TestCase): + # tests for AF_ALG + def create_alg(self, typ, name): + sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + try: + sock.bind((typ, name)) + except FileNotFoundError as e: + # type / algorithm is not available + sock.close() + raise unittest.SkipTest(str(e), typ, name) + else: + return sock + + # bpo-31705: On kernel older than 4.5, sendto() failed with ENOKEY, + # at least on ppc64le architecture + @support.requires_linux_version(4, 5) + def test_sha256(self): + expected = bytes.fromhex("ba7816bf8f01cfea414140de5dae2223b00361a396" + "177a9cb410ff61f20015ad") + with self.create_alg('hash', 'sha256') as algo: + op, _ = algo.accept() + with op: + op.sendall(b"abc") + self.assertEqual(op.recv(512), expected) + + op, _ = algo.accept() + with op: + op.send(b'a', socket.MSG_MORE) + op.send(b'b', socket.MSG_MORE) + op.send(b'c', socket.MSG_MORE) + op.send(b'') + self.assertEqual(op.recv(512), expected) + + def test_hmac_sha1(self): + expected = bytes.fromhex("effcdf6ae5eb2fa2d27416d5f184df9c259a7c79") + with self.create_alg('hash', 'hmac(sha1)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, b"Jefe") + op, _ = algo.accept() + with op: + op.sendall(b"what do ya want for nothing?") + self.assertEqual(op.recv(512), expected) + + # Although it should work with 3.19 and newer the test blocks on + # Ubuntu 15.10 with Kernel 4.2.0-19. + @support.requires_linux_version(4, 3) + def test_aes_cbc(self): + key = bytes.fromhex('06a9214036b8a15b512e03d534120006') + iv = bytes.fromhex('3dafba429d9eb430b422da802c9fac41') + msg = b"Single block msg" + ciphertext = bytes.fromhex('e353779c1079aeb82708942dbe77181a') + msglen = len(msg) + with self.create_alg('skcipher', 'cbc(aes)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) + op, _ = algo.accept() + with op: + op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, + flags=socket.MSG_MORE) + op.sendall(msg) + self.assertEqual(op.recv(msglen), ciphertext) + + op, _ = algo.accept() + with op: + op.sendmsg_afalg([ciphertext], + op=socket.ALG_OP_DECRYPT, iv=iv) + self.assertEqual(op.recv(msglen), msg) + + # long message + multiplier = 1024 + longmsg = [msg] * multiplier + op, _ = algo.accept() + with op: + op.sendmsg_afalg(longmsg, + op=socket.ALG_OP_ENCRYPT, iv=iv) + enc = op.recv(msglen * multiplier) + self.assertEqual(len(enc), msglen * multiplier) + self.assertTrue(enc[:msglen], ciphertext) + + op, _ = algo.accept() + with op: + op.sendmsg_afalg([enc], + op=socket.ALG_OP_DECRYPT, iv=iv) + dec = op.recv(msglen * multiplier) + self.assertEqual(len(dec), msglen * multiplier) + self.assertEqual(dec, msg * multiplier) + + @support.requires_linux_version(4, 9) # see issue29324 + def test_aead_aes_gcm(self): + key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') + iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') + plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') + assoc = bytes.fromhex('24825602bd12a984e0092d3e448eda5f') + expected_ct = bytes.fromhex('93fe7d9e9bfd10348a5606e5cafa7354') + expected_tag = bytes.fromhex('0032a1dc85f1c9786925a2e71d8272dd') + + taglen = len(expected_tag) + assoclen = len(assoc) + + with self.create_alg('aead', 'gcm(aes)') as algo: + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, key) + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_AEAD_AUTHSIZE, + None, taglen) + + # send assoc, plain and tag buffer in separate steps + op, _ = algo.accept() + with op: + op.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, iv=iv, + assoclen=assoclen, flags=socket.MSG_MORE) + op.sendall(assoc, socket.MSG_MORE) + op.sendall(plain) + res = op.recv(assoclen + len(plain) + taglen) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # now with msg + op, _ = algo.accept() + with op: + msg = assoc + plain + op.sendmsg_afalg([msg], op=socket.ALG_OP_ENCRYPT, iv=iv, + assoclen=assoclen) + res = op.recv(assoclen + len(plain) + taglen) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # create anc data manually + pack_uint32 = struct.Struct('I').pack + op, _ = algo.accept() + with op: + msg = assoc + plain + op.sendmsg( + [msg], + ([socket.SOL_ALG, socket.ALG_SET_OP, pack_uint32(socket.ALG_OP_ENCRYPT)], + [socket.SOL_ALG, socket.ALG_SET_IV, pack_uint32(len(iv)) + iv], + [socket.SOL_ALG, socket.ALG_SET_AEAD_ASSOCLEN, pack_uint32(assoclen)], + ) + ) + res = op.recv(len(msg) + taglen) + self.assertEqual(expected_ct, res[assoclen:-taglen]) + self.assertEqual(expected_tag, res[-taglen:]) + + # decrypt and verify + op, _ = algo.accept() + with op: + msg = assoc + expected_ct + expected_tag + op.sendmsg_afalg([msg], op=socket.ALG_OP_DECRYPT, iv=iv, + assoclen=assoclen) + res = op.recv(len(msg) - taglen) + self.assertEqual(plain, res[assoclen:]) + + @support.requires_linux_version(4, 3) # see test_aes_cbc + def test_drbg_pr_sha256(self): + # deterministic random bit generator, prediction resistance, sha256 + with self.create_alg('rng', 'drbg_pr_sha256') as algo: + extra_seed = os.urandom(32) + algo.setsockopt(socket.SOL_ALG, socket.ALG_SET_KEY, extra_seed) + op, _ = algo.accept() + with op: + rn = op.recv(32) + self.assertEqual(len(rn), 32) + + def test_sendmsg_afalg_args(self): + sock = socket.socket(socket.AF_ALG, socket.SOCK_SEQPACKET, 0) + with sock: + with self.assertRaises(TypeError): + sock.sendmsg_afalg() + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=None) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(1) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, assoclen=None) + + with self.assertRaises(TypeError): + sock.sendmsg_afalg(op=socket.ALG_OP_ENCRYPT, assoclen=-1) + +@unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") +class TestMSWindowsTCPFlags(unittest.TestCase): + knownTCPFlags = { + # available since long time ago + 'TCP_MAXSEG', + 'TCP_NODELAY', + # available starting with Windows 10 1607 + 'TCP_FASTOPEN', + # available starting with Windows 10 1703 + 'TCP_KEEPCNT', + # available starting with Windows 10 1709 + 'TCP_KEEPIDLE', + 'TCP_KEEPINTVL' + } + + def test_new_tcp_flags(self): + provided = [s for s in dir(socket) if s.startswith('TCP')] + unknown = [s for s in provided if s not in self.knownTCPFlags] + + self.assertEqual([], unknown, + "New TCP flags were discovered. See bpo-32394 for more information") + +def test_main(): + tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest, + TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ] + + tests.extend([ + NonBlockingTCPTests, + FileObjectClassTestCase, + UnbufferedFileObjectClassTestCase, + LineBufferedFileObjectClassTestCase, + SmallBufferedFileObjectClassTestCase, + UnicodeReadFileObjectClassTestCase, + UnicodeWriteFileObjectClassTestCase, + UnicodeReadWriteFileObjectClassTestCase, + NetworkConnectionNoServer, + NetworkConnectionAttributesTest, + NetworkConnectionBehaviourTest, + ContextManagersTest, + InheritanceTest, + NonblockConstantTest + ]) + tests.append(BasicSocketPairTest) + tests.append(TestUnixDomain) + tests.append(TestLinuxAbstractNamespace) + tests.extend([TIPCTest, TIPCThreadableTest]) + tests.extend([BasicCANTest, CANTest]) + tests.extend([BasicRDSTest, RDSTest]) + tests.append(LinuxKernelCryptoAPI) + tests.extend([ + BasicVSOCKTest, + ThreadedVSOCKSocketStreamTest, + ]) + tests.extend([ + CmsgMacroTests, + SendmsgUDPTest, + RecvmsgUDPTest, + RecvmsgIntoUDPTest, + SendmsgUDP6Test, + RecvmsgUDP6Test, + RecvmsgRFC3542AncillaryUDP6Test, + RecvmsgIntoRFC3542AncillaryUDP6Test, + RecvmsgIntoUDP6Test, + SendmsgTCPTest, + RecvmsgTCPTest, + RecvmsgIntoTCPTest, + SendmsgSCTPStreamTest, + RecvmsgSCTPStreamTest, + RecvmsgIntoSCTPStreamTest, + SendmsgUnixStreamTest, + RecvmsgUnixStreamTest, + RecvmsgIntoUnixStreamTest, + RecvmsgSCMRightsStreamTest, + RecvmsgIntoSCMRightsStreamTest, + # These are slow when setitimer() is not available + InterruptedRecvTimeoutTest, + InterruptedSendTimeoutTest, + TestSocketSharing, + SendfileUsingSendTest, + SendfileUsingSendfileTest, + ]) + tests.append(TestMSWindowsTCPFlags) + + thread_info = support.threading_setup() + support.run_unittest(*tests) + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.7/test_ssl.py b/src/greentest/3.7/test_ssl.py new file mode 100644 index 0000000..7bbaa9f --- /dev/null +++ b/src/greentest/3.7/test_ssl.py @@ -0,0 +1,4190 @@ +# Test the support for SSL and sockets + +import sys +import unittest +from test import support +import socket +import select +import time +import datetime +import gc +import os +import errno +import pprint +import urllib.request +import threading +import traceback +import asyncore +import weakref +import platform +import functools +import sysconfig +try: + import ctypes +except ImportError: + ctypes = None + +ssl = support.import_module("ssl") + + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = support.HOST +IS_LIBRESSL = ssl.OPENSSL_VERSION.startswith('LibreSSL') +IS_OPENSSL_1_1_0 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 0) +IS_OPENSSL_1_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 1) +PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/make_ssl_certs.py. +# Other certificates are simply fetched from the Internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = os.fsencode(CERTFILE) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = os.fsencode(ONLYCERT) +BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = os.fsencode(CAPATH) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") +WRONG_CERT = data_file("wrongcert.pem") + +CERTFILE_INFO = { + 'issuer': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'notAfter': 'Jan 17 19:09:06 2028 GMT', + 'notBefore': 'Jan 19 19:09:06 2018 GMT', + 'serialNumber': 'F9BA076D5B6ABD9B', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3 +} + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SIGNED_CERTFILE_HOSTNAME = 'localhost' + +SIGNED_CERTFILE_INFO = { + 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Nov 28 19:09:06 2027 GMT', + 'notBefore': 'Jan 19 19:09:06 2018 GMT', + 'serialNumber': '82EDBF41C880919C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3 +} + +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNED_CERTFILE2_HOSTNAME = 'fakehostname' +SIGNED_CERTFILE_ECC = data_file("keycertecc.pem") +SIGNED_CERTFILE_ECC_HOSTNAME = 'localhost-ecc' + +# Same certificate as pycacert.pem, but without extra text in file +SIGNING_CA = data_file("capath", "ceff1710.0") +# cert with all kinds of subject alt names +ALLSANFILE = data_file("allsans.pem") +IDNSANSFILE = data_file("idnsans.pem") + +REMOTE_HOST = "self-signed.pythontest.net" + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +NONEXISTINGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") + +DHFILE = data_file("dh1024.pem") +BYTES_DHFILE = os.fsencode(DHFILE) + +# Not defined in all versions of OpenSSL +OP_NO_COMPRESSION = getattr(ssl, "OP_NO_COMPRESSION", 0) +OP_SINGLE_DH_USE = getattr(ssl, "OP_SINGLE_DH_USE", 0) +OP_SINGLE_ECDH_USE = getattr(ssl, "OP_SINGLE_ECDH_USE", 0) +OP_CIPHER_SERVER_PREFERENCE = getattr(ssl, "OP_CIPHER_SERVER_PREFERENCE", 0) +OP_ENABLE_MIDDLEBOX_COMPAT = getattr(ssl, "OP_ENABLE_MIDDLEBOX_COMPAT", 0) + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + +def can_clear_options(): + # 0.9.8m or higher + return ssl._OPENSSL_API_VERSION >= (0, 9, 8, 13, 15) + +def no_sslv2_implies_sslv3_hello(): + # 0.9.7h or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) + +def have_verify_flags(): + # 0.9.8 or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 8, 0, 15) + +def _have_secp_curves(): + if not ssl.HAS_ECDH: + return False + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + try: + ctx.set_ecdh_curve("secp384r1") + except ValueError: + return False + else: + return True + + +HAVE_SECP_CURVES = _have_secp_curves() + + +def utc_offset(): #NOTE: ignore issues like #1647654 + # local time = utc time + utc offset + if time.daylight and time.localtime().tm_isdst > 0: + return -time.altzone # seconds + return -time.timezone + +def asn1time(cert_time): + # Some versions of OpenSSL ignore seconds, see #18207 + # 0.9.8.i + if ssl._OPENSSL_API_VERSION == (0, 9, 8, 9, 15): + fmt = "%b %d %H:%M:%S %Y GMT" + dt = datetime.datetime.strptime(cert_time, fmt) + dt = dt.replace(second=0) + cert_time = dt.strftime(fmt) + # %d adds leading zero but ASN1_TIME_print() uses leading space + if cert_time[4] == "0": + cert_time = cert_time[:4] + " " + cert_time[5:] + + return cert_time + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): + if hasattr(ssl, 'PROTOCOL_SSLv2'): + @functools.wraps(func) + def f(*args, **kwargs): + try: + ssl.SSLContext(ssl.PROTOCOL_SSLv2) + except ssl.SSLError: + if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and + platform.linux_distribution() == ('debian', 'squeeze/sid', '')): + raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") + return func(*args, **kwargs) + return f + else: + return func + +needs_sni = unittest.skipUnless(ssl.HAS_SNI, "SNI support needed for this test") + + +def test_wrap_socket(sock, ssl_version=ssl.PROTOCOL_TLS, *, + cert_reqs=ssl.CERT_NONE, ca_certs=None, + ciphers=None, certfile=None, keyfile=None, + **kwargs): + context = ssl.SSLContext(ssl_version) + if cert_reqs is not None: + if cert_reqs == ssl.CERT_NONE: + context.check_hostname = False + context.verify_mode = cert_reqs + if ca_certs is not None: + context.load_verify_locations(ca_certs) + if certfile is not None or keyfile is not None: + context.load_cert_chain(certfile, keyfile) + if ciphers is not None: + context.set_ciphers(ciphers) + return context.wrap_socket(sock, **kwargs) + + +def testing_context(server_cert=SIGNED_CERTFILE): + """Create context + + client_context, server_context, hostname = testing_context() + """ + if server_cert == SIGNED_CERTFILE: + hostname = SIGNED_CERTFILE_HOSTNAME + elif server_cert == SIGNED_CERTFILE2: + hostname = SIGNED_CERTFILE2_HOSTNAME + else: + raise ValueError(server_cert) + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(server_cert) + client_context.load_verify_locations(SIGNING_CA) + + return client_context, server_context, hostname + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + if ssl.HAS_ECDH: + ssl.OP_SINGLE_ECDH_USE + if ssl.OPENSSL_VERSION_INFO >= (1, 0): + ssl.OP_NO_COMPRESSION + self.assertIn(ssl.HAS_SNI, {True, False}) + self.assertIn(ssl.HAS_ECDH, {True, False}) + ssl.OP_NO_SSLv2 + ssl.OP_NO_SSLv3 + ssl.OP_NO_TLSv1 + ssl.OP_NO_TLSv1_3 + if ssl.OPENSSL_VERSION_INFO >= (1, 0, 1): + ssl.OP_NO_TLSv1_1 + ssl.OP_NO_TLSv1_2 + self.assertEqual(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv23) + + def test_private_init(self): + with self.assertRaisesRegex(TypeError, "public constructor"): + with socket.socket() as s: + ssl.SSLSocket(s) + + def test_str_for_enums(self): + # Make sure that the PROTOCOL_* constants have enum-like string + # reprs. + proto = ssl.PROTOCOL_TLS + self.assertEqual(str(proto), '_SSLMethod.PROTOCOL_TLS') + ctx = ssl.SSLContext(proto) + self.assertIs(ctx.protocol, proto) + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + + data, is_cryptographic = ssl.RAND_pseudo_bytes(16) + self.assertEqual(len(data), 16) + self.assertEqual(is_cryptographic, v == 1) + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + + # negative num is invalid + self.assertRaises(ValueError, ssl.RAND_bytes, -5) + self.assertRaises(ValueError, ssl.RAND_pseudo_bytes, -5) + + if hasattr(ssl, 'RAND_egd'): + self.assertRaises(TypeError, ssl.RAND_egd, 1) + self.assertRaises(TypeError, ssl.RAND_egd, 'foo', 1) + ssl.RAND_add("this is a random string", 75.0) + ssl.RAND_add(b"this is a random bytes object", 75.0) + ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) + + @unittest.skipUnless(os.name == 'posix', 'requires posix') + def test_random_fork(self): + status = ssl.RAND_status() + if not status: + self.fail("OpenSSL's PRNG has insufficient randomness") + + rfd, wfd = os.pipe() + pid = os.fork() + if pid == 0: + try: + os.close(rfd) + child_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(child_random), 16) + os.write(wfd, child_random) + os.close(wfd) + except BaseException: + os._exit(1) + else: + os._exit(0) + else: + os.close(wfd) + self.addCleanup(os.close, rfd) + _, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + child_random = os.read(rfd, 16) + self.assertEqual(len(child_random), 16) + parent_random = ssl.RAND_pseudo_bytes(16)[0] + self.assertEqual(len(parent_random), 16) + + self.assertNotEqual(child_random, parent_random) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + self.assertEqual( + ssl._ssl._test_decode_cert(CERTFILE), + CERTFILE_INFO + ) + self.assertEqual( + ssl._ssl._test_decode_cert(SIGNED_CERTFILE), + SIGNED_CERTFILE_INFO + ) + + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + + def test_parse_all_sans(self): + p = ssl._ssl._test_decode_cert(ALLSANFILE) + self.assertEqual(p['subjectAltName'], + ( + ('DNS', 'allsans'), + ('othername', ''), + ('othername', ''), + ('email', 'user@example.org'), + ('DNS', 'www.example.org'), + ('DirName', + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'dirname example'),))), + ('URI', 'https://www.python.org/'), + ('IP Address', '127.0.0.1'), + ('IP Address', '0:0:0:0:0:0:0:1\n'), + ('Registered ID', '1.2.3.4.5') + ) + ) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, int) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 0.9 + self.assertGreaterEqual(n, 0x900000) + # < 3.0 + self.assertLess(n, 0x30000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 0) + self.assertLess(major, 3) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + # Version string as returned by {Open,Libre}SSL, the format might change + if IS_LIBRESSL: + self.assertTrue(s.startswith("LibreSSL {:d}".format(major)), + (s, t, hex(n))) + else: + self.assertTrue(s.startswith("OpenSSL {:d}.{:d}.{:d}".format(major, minor, fix)), + (s, t, hex(n))) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = test_wrap_socket(s) + wr = weakref.ref(ss) + with support.check_warnings(("", ResourceWarning)): + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # OSError raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s) as ss: + self.assertRaises(OSError, ss.recv, 1) + self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) + self.assertRaises(OSError, ss.recvfrom, 1) + self.assertRaises(OSError, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(OSError, ss.send, b'x') + self.assertRaises(OSError, ss.sendto, b'x', ('0.0.0.0', 0)) + self.assertRaises(NotImplementedError, ss.sendmsg, + [b'x'], (), 0, ('0.0.0.0', 0)) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with test_wrap_socket(s) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + def test_errors_sslwrap(self): + sock = socket.socket() + self.assertRaisesRegex(ValueError, + "certfile must be specified", + ssl.wrap_socket, sock, keyfile=CERTFILE) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True, certfile="") + with ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) as s: + self.assertRaisesRegex(ValueError, "can't connect in server-side mode", + s.connect, (HOST, 8080)) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, + certfile=CERTFILE, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(OSError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, + certfile=NONEXISTINGCERT, keyfile=NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def bad_cert_test(self, certfile): + """Check that trying to use the given client certificate fails""" + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + certfile) + sock = socket.socket() + self.addCleanup(sock.close) + with self.assertRaises(ssl.SSLError): + test_wrap_socket(sock, + certfile=certfile) + + def test_empty_cert(self): + """Wrapping with an empty cert file""" + self.bad_cert_test("nullcert.pem") + + def test_malformed_cert(self): + """Wrapping with a badly formatted certificate (syntax error)""" + self.bad_cert_test("badcert.pem") + + def test_malformed_key(self): + """Wrapping with a badly formatted key (syntax error)""" + self.bad_cert_test("badkey.pem") + + def test_match_hostname(self): + def ok(cert, hostname): + ssl.match_hostname(cert, hostname) + def fail(cert, hostname): + self.assertRaises(ssl.CertificateError, + ssl.match_hostname, cert, hostname) + + # -- Hostname matching -- + + cert = {'subject': ((('commonName', 'example.com'),),)} + ok(cert, 'example.com') + ok(cert, 'ExAmple.cOm') + fail(cert, 'www.example.com') + fail(cert, '.example.com') + fail(cert, 'example.org') + fail(cert, 'exampleXcom') + + cert = {'subject': ((('commonName', '*.a.com'),),)} + ok(cert, 'foo.a.com') + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + # only match wildcards when they are the only thing + # in left-most segment + cert = {'subject': ((('commonName', 'f*.com'),),)} + fail(cert, 'foo.com') + fail(cert, 'f.com') + fail(cert, 'bar.com') + fail(cert, 'foo.a.com') + fail(cert, 'bar.foo.com') + + # NULL bytes are bad, CVE-2013-4073 + cert = {'subject': ((('commonName', + 'null.python.org\x00example.org'),),)} + ok(cert, 'null.python.org\x00example.org') # or raise an error? + fail(cert, 'example.org') + fail(cert, 'null.python.org') + + # error cases with wildcards + cert = {'subject': ((('commonName', '*.*.a.com'),),)} + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + cert = {'subject': ((('commonName', 'a.*.com'),),)} + fail(cert, 'a.foo.com') + fail(cert, 'a..com') + fail(cert, 'a.com') + + # wildcard doesn't match IDNA prefix 'xn--' + idna = 'püthon.python.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + ok(cert, idna) + cert = {'subject': ((('commonName', 'x*.python.org'),),)} + fail(cert, idna) + cert = {'subject': ((('commonName', 'xn--p*.python.org'),),)} + fail(cert, idna) + + # wildcard in first fragment and IDNA A-labels in sequent fragments + # are supported. + idna = 'www*.pythön.org'.encode("idna").decode("ascii") + cert = {'subject': ((('commonName', idna),),)} + fail(cert, 'www.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'ftp.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'pythön.org'.encode("idna").decode("ascii")) + + # Slightly fake real-world example + cert = {'notAfter': 'Jun 26 21:41:46 2011 GMT', + 'subject': ((('commonName', 'linuxfrz.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('DNS', 'linuxfr.com'), + ('othername', ''))} + ok(cert, 'linuxfr.org') + ok(cert, 'linuxfr.com') + # Not a "DNS" entry + fail(cert, '') + # When there is a subjectAltName, commonName isn't used + fail(cert, 'linuxfrz.org') + + # A pristine real-world example + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),), + (('commonName', 'mail.google.com'),))} + ok(cert, 'mail.google.com') + fail(cert, 'gmail.com') + # Only commonName is considered + fail(cert, 'California') + + # -- IPv4 matching -- + cert = {'subject': ((('commonName', 'example.com'),),), + 'subjectAltName': (('DNS', 'example.com'), + ('IP Address', '10.11.12.13'), + ('IP Address', '14.15.16.17'))} + ok(cert, '10.11.12.13') + ok(cert, '14.15.16.17') + fail(cert, '14.15.16.18') + fail(cert, 'example.net') + + # -- IPv6 matching -- + if hasattr(socket, 'AF_INET6'): + cert = {'subject': ((('commonName', 'example.com'),),), + 'subjectAltName': ( + ('DNS', 'example.com'), + ('IP Address', '2001:0:0:0:0:0:0:CAFE\n'), + ('IP Address', '2003:0:0:0:0:0:0:BABA\n'))} + ok(cert, '2001::cafe') + ok(cert, '2003::baba') + fail(cert, '2003::bebe') + fail(cert, 'example.net') + + # -- Miscellaneous -- + + # Neither commonName nor subjectAltName + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),))} + fail(cert, 'mail.google.com') + + # No DNS entry in subjectAltName but a commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('commonName', 'mail.google.com'),)), + 'subjectAltName': (('othername', 'blabla'), )} + ok(cert, 'mail.google.com') + + # No DNS entry subjectAltName and no commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),)), + 'subjectAltName': (('othername', 'blabla'),)} + fail(cert, 'google.com') + + # Empty cert / no cert + self.assertRaises(ValueError, ssl.match_hostname, None, 'example.com') + self.assertRaises(ValueError, ssl.match_hostname, {}, 'example.com') + + # Issue #17980: avoid denials of service by refusing more than one + # wildcard per fragment. + cert = {'subject': ((('commonName', 'a*b.example.com'),),)} + with self.assertRaisesRegex( + ssl.CertificateError, + "partial wildcards in leftmost label are not supported"): + ssl.match_hostname(cert, 'axxb.example.com') + + cert = {'subject': ((('commonName', 'www.*.example.com'),),)} + with self.assertRaisesRegex( + ssl.CertificateError, + "wildcard can only be present in the leftmost label"): + ssl.match_hostname(cert, 'www.sub.example.com') + + cert = {'subject': ((('commonName', 'a*b*.example.com'),),)} + with self.assertRaisesRegex( + ssl.CertificateError, + "too many wildcards"): + ssl.match_hostname(cert, 'axxbxxc.example.com') + + cert = {'subject': ((('commonName', '*'),),)} + with self.assertRaisesRegex( + ssl.CertificateError, + "sole wildcard without additional labels are not support"): + ssl.match_hostname(cert, 'host') + + cert = {'subject': ((('commonName', '*.com'),),)} + with self.assertRaisesRegex( + ssl.CertificateError, + r"hostname 'com' doesn't match '\*.com'"): + ssl.match_hostname(cert, 'com') + + # extra checks for _inet_paton() + for invalid in ['1', '', '1.2.3', '256.0.0.1', '127.0.0.1/24']: + with self.assertRaises(ValueError): + ssl._inet_paton(invalid) + for ipaddr in ['127.0.0.1', '192.168.0.1']: + self.assertTrue(ssl._inet_paton(ipaddr)) + if hasattr(socket, 'AF_INET6'): + for ipaddr in ['::1', '2001:db8:85a3::8a2e:370:7334']: + self.assertTrue(ssl._inet_paton(ipaddr)) + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + with socket.socket() as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + s.bind(('127.0.0.1', 0)) + s.listen() + c = socket.socket(socket.AF_INET) + c.connect(s.getsockname()) + with test_wrap_socket(c, do_handshake_on_connect=False) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + s.close() + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + def test_dealloc_warn(self): + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + r = repr(ss) + with self.assertWarns(ResourceWarning) as cm: + ss = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (set, bool)) + if isinstance(trust, set): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegex(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegex(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + test_wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + def cert_time_ok(self, timestring, timestamp): + self.assertEqual(ssl.cert_time_to_seconds(timestring), timestamp) + + def cert_time_fail(self, timestring): + with self.assertRaises(ValueError): + ssl.cert_time_to_seconds(timestring) + + @unittest.skipUnless(utc_offset(), + 'local time needs to be different from UTC') + def test_cert_time_to_seconds_timezone(self): + # Issue #19940: ssl.cert_time_to_seconds() returns wrong + # results if local timezone is not UTC + self.cert_time_ok("May 9 00:00:00 2007 GMT", 1178668800.0) + self.cert_time_ok("Jan 5 09:34:43 2018 GMT", 1515144883.0) + + def test_cert_time_to_seconds(self): + timestring = "Jan 5 09:34:43 2018 GMT" + ts = 1515144883.0 + self.cert_time_ok(timestring, ts) + # accept keyword parameter, assert its name + self.assertEqual(ssl.cert_time_to_seconds(cert_time=timestring), ts) + # accept both %e and %d (space or zero generated by strftime) + self.cert_time_ok("Jan 05 09:34:43 2018 GMT", ts) + # case-insensitive + self.cert_time_ok("JaN 5 09:34:43 2018 GmT", ts) + self.cert_time_fail("Jan 5 09:34 2018 GMT") # no seconds + self.cert_time_fail("Jan 5 09:34:43 2018") # no GMT + self.cert_time_fail("Jan 5 09:34:43 2018 UTC") # not GMT timezone + self.cert_time_fail("Jan 35 09:34:43 2018 GMT") # invalid day + self.cert_time_fail("Jon 5 09:34:43 2018 GMT") # invalid month + self.cert_time_fail("Jan 5 24:00:00 2018 GMT") # invalid hour + self.cert_time_fail("Jan 5 09:60:43 2018 GMT") # invalid minute + + newyear_ts = 1230768000.0 + # leap seconds + self.cert_time_ok("Dec 31 23:59:60 2008 GMT", newyear_ts) + # same timestamp + self.cert_time_ok("Jan 1 00:00:00 2009 GMT", newyear_ts) + + self.cert_time_ok("Jan 5 09:34:59 2018 GMT", 1515144899) + # allow 60th second (even if it is not a leap second) + self.cert_time_ok("Jan 5 09:34:60 2018 GMT", 1515144900) + # allow 2nd leap second for compatibility with time.strptime() + self.cert_time_ok("Jan 5 09:34:61 2018 GMT", 1515144901) + self.cert_time_fail("Jan 5 09:34:62 2018 GMT") # invalid seconds + + # no special treatment for the special value: + # 99991231235959Z (rfc 5280) + self.cert_time_ok("Dec 31 23:59:59 9999 GMT", 253402300799.0) + + @support.run_with_locale('LC_ALL', '') + def test_cert_time_to_seconds_locale(self): + # `cert_time_to_seconds()` should be locale independent + + def local_february_name(): + return time.strftime('%b', (1, 2, 3, 4, 5, 6, 0, 0, 0)) + + if local_february_name().lower() == 'feb': + self.skipTest("locale-specific month name needs to be " + "different from C locale") + + # locale-independent + self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) + self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + + def test_connect_ex_error(self): + server = socket.socket(socket.AF_INET) + self.addCleanup(server.close) + port = support.bind_port(server) # Reserve port but don't listen + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + rc = s.connect_ex((HOST, port)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + + +class ContextTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_constructor(self): + for protocol in PROTOCOLS: + ssl.SSLContext(protocol) + ctx = ssl.SSLContext() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + @skip_if_broken_ubuntu_ssl + def test_protocol(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.protocol, proto) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @unittest.skipUnless(PY_SSL_DEFAULT_CIPHERS == 1, + "Test applies only to Python default ciphers") + def test_python_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ciphers = ctx.get_ciphers() + for suite in ciphers: + name = suite['name'] + self.assertNotIn("PSK", name) + self.assertNotIn("SRP", name) + self.assertNotIn("MD5", name) + self.assertNotIn("RC4", name) + self.assertNotIn("3DES", name) + + @unittest.skipIf(ssl.OPENSSL_VERSION_INFO < (1, 0, 2, 0, 0), 'OpenSSL too old') + def test_get_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_ciphers('AESGCM') + names = set(d['name'] for d in ctx.get_ciphers()) + self.assertIn('AES256-GCM-SHA384', names) + self.assertIn('AES128-GCM-SHA256', names) + + @skip_if_broken_ubuntu_ssl + def test_options(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + # SSLContext also enables these by default + default |= (OP_NO_COMPRESSION | OP_CIPHER_SERVER_PREFERENCE | + OP_SINGLE_DH_USE | OP_SINGLE_ECDH_USE | + OP_ENABLE_MIDDLEBOX_COMPAT) + self.assertEqual(default, ctx.options) + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + if can_clear_options(): + ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) + self.assertEqual(default, ctx.options) + ctx.options = 0 + # Ubuntu has OP_NO_SSLv3 forced on by default + self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + else: + with self.assertRaises(ValueError): + ctx.options = 0 + + def test_verify_mode_protocol(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + + def test_hostname_checks_common_name(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.hostname_checks_common_name) + if ssl.HAS_NEVER_CHECK_COMMON_NAME: + ctx.hostname_checks_common_name = True + self.assertTrue(ctx.hostname_checks_common_name) + ctx.hostname_checks_common_name = False + self.assertFalse(ctx.hostname_checks_common_name) + ctx.hostname_checks_common_name = True + self.assertTrue(ctx.hostname_checks_common_name) + else: + with self.assertRaises(AttributeError): + ctx.hostname_checks_common_name = True + + @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'), + "required OpenSSL 1.1.0g") + def test_min_max_version(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertEqual( + ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED + ) + + ctx.minimum_version = ssl.TLSVersion.TLSv1_1 + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + self.assertEqual( + ctx.minimum_version, ssl.TLSVersion.TLSv1_1 + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.TLSv1_2 + ) + + ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + ctx.maximum_version = ssl.TLSVersion.TLSv1 + self.assertEqual( + ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.TLSv1 + ) + + ctx.maximum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED + ) + + ctx.maximum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + self.assertIn( + ctx.maximum_version, + {ssl.TLSVersion.TLSv1, ssl.TLSVersion.SSLv3} + ) + + ctx.minimum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED + self.assertIn( + ctx.minimum_version, + {ssl.TLSVersion.TLSv1_2, ssl.TLSVersion.TLSv1_3} + ) + + with self.assertRaises(ValueError): + ctx.minimum_version = 42 + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) + + self.assertEqual( + ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED + ) + with self.assertRaises(ValueError): + ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + with self.assertRaises(ValueError): + ctx.maximum_version = ssl.TLSVersion.TLSv1 + + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read() + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read() + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegex(ssl.SSLError, "no start line"): + ctx.load_verify_locations(cadata="broken") + with self.assertRaisesRegex(ssl.SSLError, "not enough data"): + ctx.load_verify_locations(cadata=b"broken") + + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_dh_params(DHFILE) + if os.name != 'nt': + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(FileNotFoundError) as cm: + ctx.load_dh_params(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + @skip_if_broken_ubuntu_ssl + def test_session_stats(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + @needs_sni + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @needs_sni + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': asn1time('Mar 29 12:29:49 2033 GMT'), + 'notBefore': asn1time('Mar 30 12:29:49 2003 GMT'), + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + @unittest.skipIf(hasattr(sys, "gettotalrefcount"), "Debug build does not share environment between CRTs") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with support.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def _assert_context_options(self, ctx): + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + if OP_NO_COMPRESSION != 0: + self.assertEqual(ctx.options & OP_NO_COMPRESSION, + OP_NO_COMPRESSION) + if OP_SINGLE_DH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_DH_USE, + OP_SINGLE_DH_USE) + if OP_SINGLE_ECDH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_ECDH_USE, + OP_SINGLE_ECDH_USE) + if OP_CIPHER_SERVER_PREFERENCE != 0: + self.assertEqual(ctx.options & OP_CIPHER_SERVER_PREFERENCE, + OP_CIPHER_SERVER_PREFERENCE) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) + + with open(SIGNING_CA) as f: + cadata = f.read() + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self._assert_context_options(ctx) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self._assert_context_options(ctx) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + def test_check_hostname(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + # Auto set CERT_REQUIRED + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + # Changing verify_mode does not affect check_hostname + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + # Auto set + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + # keep CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + def test_context_client_server(self): + # PROTOCOL_TLS_CLIENT has sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + # PROTOCOL_TLS_SERVER has different but also sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + def test_context_custom_class(self): + class MySSLSocket(ssl.SSLSocket): + pass + + class MySSLObject(ssl.SSLObject): + pass + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.sslsocket_class = MySSLSocket + ctx.sslobject_class = MySSLObject + + with ctx.wrap_socket(socket.socket(), server_side=True) as sock: + self.assertIsInstance(sock, MySSLSocket) + obj = ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO()) + self.assertIsInstance(obj, MySSLObject) + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + self.assertEqual(cm.exception.library, 'PEM') + self.assertEqual(cm.exception.reason, 'NO_START_LINE') + s = str(cm.exception) + self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen() + c = socket.socket() + c.connect(s.getsockname()) + c.setblocking(False) + with ctx.wrap_socket(c, False, do_handshake_on_connect=False) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertTrue(s.startswith("The operation did not complete (read)"), s) + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + + def test_bad_server_hostname(self): + ctx = ssl.create_default_context() + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="") + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname=".example.org") + with self.assertRaises(TypeError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="example.org\x00evil.com") + + +class MemoryBIOTests(unittest.TestCase): + + def test_read_write(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + self.assertEqual(bio.read(), b'') + bio.write(b'foo') + bio.write(b'bar') + self.assertEqual(bio.read(), b'foobar') + self.assertEqual(bio.read(), b'') + bio.write(b'baz') + self.assertEqual(bio.read(2), b'ba') + self.assertEqual(bio.read(1), b'z') + self.assertEqual(bio.read(1), b'') + + def test_eof(self): + bio = ssl.MemoryBIO() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertFalse(bio.eof) + bio.write(b'foo') + self.assertFalse(bio.eof) + bio.write_eof() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(2), b'fo') + self.assertFalse(bio.eof) + self.assertEqual(bio.read(1), b'o') + self.assertTrue(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertTrue(bio.eof) + + def test_pending(self): + bio = ssl.MemoryBIO() + self.assertEqual(bio.pending, 0) + bio.write(b'foo') + self.assertEqual(bio.pending, 3) + for i in range(3): + bio.read(1) + self.assertEqual(bio.pending, 3-i-1) + for i in range(3): + bio.write(b'x') + self.assertEqual(bio.pending, i+1) + bio.read() + self.assertEqual(bio.pending, 0) + + def test_buffer_types(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + bio.write(bytearray(b'bar')) + self.assertEqual(bio.read(), b'bar') + bio.write(memoryview(b'baz')) + self.assertEqual(bio.read(), b'baz') + + def test_error_types(self): + bio = ssl.MemoryBIO() + self.assertRaises(TypeError, bio.write, 'foo') + self.assertRaises(TypeError, bio.write, None) + self.assertRaises(TypeError, bio.write, True) + self.assertRaises(TypeError, bio.write, 1) + + +class SSLObjectTests(unittest.TestCase): + def test_private_init(self): + bio = ssl.MemoryBIO() + with self.assertRaisesRegex(TypeError, "public constructor"): + ssl.SSLObject(bio, bio) + + +class SimpleBackgroundTests(unittest.TestCase): + """Tests that connect to a simple server running in the background""" + + def setUp(self): + server = ThreadedEchoServer(SIGNED_CERTFILE) + self.server_addr = (HOST, server.port) + server.__enter__() + self.addCleanup(server.__exit__, None, None, None) + + def test_connect(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + self.assertFalse(s.server_side) + + # this should succeed because we specify the root cert + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) as s: + s.connect(self.server_addr) + self.assertTrue(s.getpeercert()) + self.assertFalse(s.server_side) + + def test_connect_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) + self.addCleanup(s.close) + self.assertEqual(0, s.connect_ex(self.server_addr)) + self.assertTrue(s.getpeercert()) + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.setblocking(False) + rc = s.connect_ex(self.server_addr) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + + def test_connect_with_context(self): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + # Same with a server hostname + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="dummy") as s: + s.connect(self.server_addr) + ctx.verify_mode = ssl.CERT_REQUIRED + # This should succeed because we specify the root cert + ctx.load_verify_locations(SIGNING_CA) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_with_context_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_cadata(self): + with open(SIGNING_CA) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + ss.connect(self.server_addr) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + s = socket.socket(socket.AF_INET) + s.connect(self.server_addr) + s.setblocking(False) + s = test_wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + self.addCleanup(s.close) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + _test_get_server_certificate(self, *self.server_addr, cert=SIGNING_CA) + + def test_get_server_certificate_fail(self): + # Connection failure crashes ThreadedEchoServer, so run this in an + # independent test method + _test_get_server_certificate_fail(self, *self.server_addr) + + def test_ciphers(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(self.server_addr) + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(self.server_addr) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = test_wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(self.server_addr) + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname='localhost') as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + self.assertEqual(len(ctx.get_ca_certs()), 1) + + @needs_sni + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx1.load_verify_locations(capath=CAPATH) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx2.load_verify_locations(capath=CAPATH) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s, server_hostname='localhost') as ss: + ss.connect(self.server_addr) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + + def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): + # A simple IO loop. Call func(*args) depending on the error we get + # (WANT_READ or WANT_WRITE) move data between the socket and the BIOs. + timeout = kwargs.get('timeout', 10) + deadline = time.monotonic() + timeout + count = 0 + while True: + if time.monotonic() > deadline: + self.fail("timeout") + errno = None + count += 1 + try: + ret = func(*args) + except ssl.SSLError as e: + if e.errno not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + raise + errno = e.errno + # Get any data from the outgoing BIO irrespective of any error, and + # send it to the socket. + buf = outgoing.read() + sock.sendall(buf) + # If there's no error, we're done. For WANT_READ, we need to get + # data from the socket and put it in the incoming BIO. + if errno is None: + break + elif errno == ssl.SSL_ERROR_WANT_READ: + buf = sock.recv(32768) + if buf: + incoming.write(buf) + else: + incoming.write_eof() + if support.verbose: + sys.stdout.write("Needed %d calls to complete %s().\n" + % (count, func.__name__)) + return ret + + def test_bio_handshake(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.load_verify_locations(SIGNING_CA) + sslobj = ctx.wrap_bio(incoming, outgoing, False, + SIGNED_CERTFILE_HOSTNAME) + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNone(sslobj.version()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNotNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # If the server shuts down the TCP connection without sending a + # secure shutdown message, this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + + def test_bio_read_write_data(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'FOO\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf, b'foo\n') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + + +class NetworkedTests(unittest.TestCase): + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with support.transient_internet(REMOTE_HOST): + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'Needs IPv6') + def test_get_server_certificate_ipv6(self): + with support.transient_internet('ipv6.google.com'): + _test_get_server_certificate(self, 'ipv6.google.com', 443) + _test_get_server_certificate_fail(self, 'ipv6.google.com', 443) + + +def _test_get_server_certificate(test, host, port, cert=None): + pem = ssl.get_server_certificate((host, port)) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + +def _test_get_server_certificate_fail(test, host, port): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + +from test.ssl_servers import make_https_server + +class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(1) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol()) + self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) + except (ConnectionResetError, BrokenPipeError) as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL + # tries to send session tickets after handshake. + # https://github.com/openssl/openssl/issues/6342 + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.close() + return False + except (ssl.SSLError, OSError) as e: + # OSError may occur with wrong protocols, e.g. both + # sides use PROTOCOL_TLS_SERVER. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + # + # bpo-31323: Store the exception as string to prevent + # a reference leak: server -> conn_errors -> exception + # -> traceback -> self (ConnectionHandler) -> server + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.server.stop() + self.close() + return False + else: + self.server.shared_ciphers.append(self.sslconn.shared_ciphers()) + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + sys.stdout.write(" cert binary is " + str(len(cert_binary)) + " bytes\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + sys.stdout.write(" server: selected protocol is now " + + str(self.sslconn.selected_npn_protocol()) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + try: + self.sock = self.sslconn.unwrap() + except OSError: + # Many tests shut the TCP connection down + # without an SSL shutdown. This causes + # unwrap() to raise OSError with errno=0! + pass + else: + self.sslconn = None + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except ConnectionResetError: + # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError + # when connection is not shut down gracefully. + if self.server.chatty and support.verbose: + sys.stdout.write( + " Connection reset by peer: {}\n".format( + self.addr) + ) + self.close() + self.running = False + except OSError: + if self.server.chatty: + handle_error("Test server failure:\n") + self.close() + self.running = False + + # normally, we'd just stop here, but for the test + # harness, we want to stop the server + self.server.stop() + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + npn_protocols=None, alpn_protocols=None, + ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLS_SERVER) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if npn_protocols: + self.context.set_npn_protocols(npn_protocols) + if alpn_protocols: + self.context.set_alpn_protocols(alpn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = support.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_npn_protocols = [] + self.selected_alpn_protocols = [] + self.shared_ciphers = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.sock.settimeout(0.05) + self.sock.listen() + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except socket.timeout: + pass + except KeyboardInterrupt: + self.stop() + except BaseException as e: + if support.verbose and self.chatty: + sys.stdout.write( + ' connection handling failed: ' + repr(e) + '\n') + + self.sock.close() + + def stop(self): + self.active = False + +class AsyncoreEchoServer(threading.Thread): + + # this one's based on asyncore.dispatcher + + class EchoServer (asyncore.dispatcher): + + class ConnectionHandler(asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = test_wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = support.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accepted(self, sock_obj, addr): + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + # make sure that ConnectionHandler is removed from socket_map + asyncore.close_all(ignore_all=True) + + def start (self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + +def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None, + session=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=sni_name, session=session) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_alpn_protocol': s.selected_alpn_protocol(), + 'client_npn_protocol': s.selected_npn_protocol(), + 'version': s.version(), + 'session_reused': s.session_reused, + 'session': s.session, + }) + s.close() + stats['server_alpn_protocols'] = server.selected_alpn_protocols + stats['server_npn_protocols'] = server.selected_npn_protocols + stats['server_shared_ciphers'] = server.shared_ciphers + return stats + +def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_TLS: + client_context.set_ciphers("ALL") + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(SIGNED_CERTFILE) + ctx.load_verify_locations(SIGNING_CA) + try: + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except OSError as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) + + +class ThreadedTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + for protocol in PROTOCOLS: + if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}: + continue + with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): + context = ssl.SSLContext(protocol) + context.load_cert_chain(CERTFILE) + server_params_test(context, context, + chatty=True, connectionchatty=True) + + client_context, server_context, hostname = testing_context() + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_SERVER): + server_params_test(client_context=client_context, + server_context=server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + client_context.check_hostname = False + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIn('called a function you should not call', + str(e.exception)) + + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_SERVER): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=server_context, + chatty=True, connectionchatty=True) + self.assertIn('called a function you should not call', + str(e.exception)) + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True) + self.assertIn('called a function you should not call', + str(e.exception)) + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + do_handshake_on_connect=False, + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + + @unittest.skipUnless(have_verify_flags(), + "verify_flags need OpenSSL > 0.9.8") + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(client_context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaisesRegex(ssl.SSLError, + "certificate verify failed"): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + client_context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: + with self.assertRaisesRegex( + ssl.CertificateError, + "Hostname mismatch, certificate is not valid for 'invalid'."): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with socket.socket() as s: + with self.assertRaisesRegex(ValueError, + "check_hostname requires server_hostname"): + client_context.wrap_socket(s) + + def test_ecc_cert(self): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + client_context.set_ciphers('ECDHE:ECDSA:!NULL:!aRSA') + hostname = SIGNED_CERTFILE_ECC_HOSTNAME + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # load ECC cert + server_context.load_cert_chain(SIGNED_CERTFILE_ECC) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher()[0].split('-') + self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + + def test_dual_rsa_ecc(self): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + # TODO: fix TLSv1.3 once SSLContext can restrict signature + # algorithms. + client_context.options |= ssl.OP_NO_TLSv1_3 + # only ECDSA certs + client_context.set_ciphers('ECDHE:ECDSA:!NULL:!aRSA') + hostname = SIGNED_CERTFILE_ECC_HOSTNAME + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # load ECC and RSA key/cert pairs + server_context.load_cert_chain(SIGNED_CERTFILE_ECC) + server_context.load_cert_chain(SIGNED_CERTFILE) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher()[0].split('-') + self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + + def test_check_hostname_idn(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(IDNSANSFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify, when specified in several + # different ways + idn_hostnames = [ + ('könig.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + ('xn--knig-5qa.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + (b'xn--knig-5qa.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + + ('königsgäßchen.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + ('xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + (b'xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + + # ('königsgäßchen.idna2008.pythontest.net', + # 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + ('xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + (b'xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + + ] + for server_hostname, expected_hostname in idn_hostnames: + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname=server_hostname) as s: + self.assertEqual(s.server_hostname, expected_hostname) + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertEqual(s.server_hostname, expected_hostname) + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="python.example.org") as s: + with self.assertRaises(ssl.CertificateError): + s.connect((HOST, server.port)) + + def test_wrong_cert_tls12(self): + """Connecting when the server rejects the client's certificate + + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with a wrong client certificate fails. + """ + client_context, server_context, hostname = testing_context() + # load client cert + client_context.load_cert_chain(WRONG_CERT) + # require TLS client authentication + server_context.verify_mode = ssl.CERT_REQUIRED + # TLS 1.3 has different handshake + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer( + context=server_context, chatty=True, connectionchatty=True, + ) + + with server, \ + client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + try: + # Expect either an SSL error about the server rejecting + # the connection, or a low-level connection reset (which + # sometimes happens on Windows) + s.connect((HOST, server.port)) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3") + def test_wrong_cert_tls13(self): + client_context, server_context, hostname = testing_context() + client_context.load_cert_chain(WRONG_CERT) + server_context.verify_mode = ssl.CERT_REQUIRED + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + + server = ThreadedEchoServer( + context=server_context, chatty=True, connectionchatty=True, + ) + with server, \ + client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # TLS 1.3 perform client cert exchange after handshake + s.connect((HOST, server.port)) + try: + s.write(b'data') + s.read(4) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = support.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen() + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with socket.socket() as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = test_wrap_socket(c) + except OSError: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + def test_ssl_cert_verify_error(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + try: + s.connect((HOST, server.port)) + except ssl.SSLError as e: + msg = 'unable to get local issuer certificate' + self.assertIsInstance(e, ssl.SSLCertVerificationError) + self.assertEqual(e.verify_code, 20) + self.assertEqual(e.verify_message, msg) + self.assertIn(msg, repr(e)) + self.assertIn('certificate verify failed', repr(e)) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") + def test_protocol_sslv2(self): + """Connecting to an SSLv2 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLS, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) + # SSLv23 client with specific SSL options + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_SSLv2) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + def test_PROTOCOL_TLS(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try: + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv2, True) + except OSError as x: + # this fails on some older versions of OpenSSL (0.9.7l, for instance) + if support.verbose: + sys.stdout.write( + " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" + % str(x)) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1') + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + + # Server with specific SSL options + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), + "OpenSSL is compiled without SSLv3 support") + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, + False, client_options=ssl.OP_NO_SSLv2) + + @skip_if_broken_ubuntu_ssl + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), + "TLS version 1.1 not supported.") + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), + "TLS version 1.2 not supported.") + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(1) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = test_wrap_socket(s) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using socketserver to create and manage SSL connections.""" + server = make_https_server(self, certfile=SIGNED_CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + with open(CERTFILE, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = 'https://localhost:%d/%s' % ( + server.port, os.path.split(CERTFILE)[1]) + context = ssl.create_default_context(cafile=SIGNING_CA) + f = urllib.request.urlopen(url, context=context) + try: + dlen = f.info().get("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = test_wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLS_CLIENT) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, expect success?, *args, return value func) + send_methods = [ + ('send', s.send, True, [], len), + ('sendto', s.sendto, False, ["some.address"], len), + ('sendall', s.sendall, True, [], lambda x: None), + ] + # (name, method, whether to expect success, *args) + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = "PREFIX_" + + for (meth_name, send_meth, expect_success, args, + ret_val_meth) in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + ret = send_meth(indata, *args) + msg = "sending with {}".format(meth_name) + self.assertEqual(ret, ret_val_meth(indata), msg=msg) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # read(-1, buffer) is supported, even though read(-1) is not + data = b"data" + s.send(data) + buffer = bytearray(len(data)) + self.assertEqual(s.read(-1, buffer), len(data)) + self.assertEqual(buffer, data) + + # sendall accepts bytes-like objects + if ctypes is not None: + ubyte = ctypes.c_ubyte * len(data) + byteslike = ubyte.from_buffer_copy(data) + s.sendall(byteslike) + self.assertEqual(s.read(), data) + + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, bytearray(100)) + s.write(b"over\n") + + self.assertRaises(ValueError, s.recv, -1) + self.assertRaises(ValueError, s.read, -1) + + s.close() + + def test_recv_zero(self): + server = ThreadedEchoServer(CERTFILE) + server.__enter__() + self.addCleanup(server.__exit__, None, None) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + # recv/read(0) should return no data + s.send(b"data") + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.read(0), b"") + self.assertEqual(s.read(), b"data") + + # Should not block if the other end sends no data + s.setblocking(False) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.recv_into(bytearray()), 0) + + def test_nonblocking_send(self): + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLS_CLIENT) + s.connect((HOST, server.port)) + s.setblocking(False) + + # If we keep sending data, at some point the buffers + # will be full and the call will block + buf = bytearray(8192) + def fill_buffer(): + while True: + s.send(buf) + self.assertRaises((ssl.SSLWantWriteError, + ssl.SSLWantReadError), fill_buffer) + + # Now read all the output and discard it + s.setblocking(True) + s.close() + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen() + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + test_wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = test_wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegex(socket.timeout, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(SIGNING_CA) + context.load_cert_chain(SIGNED_CERTFILE) + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = support.bind_port(server) + server = context.wrap_socket(server, server_side=True) + self.assertTrue(server.server_side) + + evt = threading.Event() + remote = None + peer = None + def serve(): + nonlocal remote, peer + server.listen() + # Block on the accept and wait on the connection to close. + evt.set() + remote, peer = server.accept() + remote.send(remote.recv(4)) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = context.wrap_socket(socket.socket()) + client.connect((host, port)) + client.send(b'data') + client.recv() + client_addr = client.getsockname() + client.close() + t.join() + remote.close() + server.close() + # Sanity checks. + self.assertIsInstance(remote, ssl.SSLSocket) + self.assertEqual(peer, client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_no_shared_ciphers(self): + client_context, server_context, hostname = testing_context() + # OpenSSL enables all TLS 1.3 ciphers, enforce TLS 1.2 for test + client_context.options |= ssl.OP_NO_TLSv1_3 + # Force different suites on client and master + client_context.set_ciphers("AES128") + server_context.set_ciphers("AES256") + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) + self.assertIn("no shared cipher", server.conn_errors[0]) + + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + self.assertIs(s.version(), None) + self.assertIs(s._sslobj, None) + s.connect((HOST, server.port)) + if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3: + self.assertEqual(s.version(), 'TLSv1.3') + elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): + self.assertEqual(s.version(), 'TLSv1.2') + else: # 0.9.8 to 1.0.1 + self.assertIn(s.version(), ('TLSv1', 'TLSv1.2')) + self.assertIs(s._sslobj, None) + self.assertIs(s.version(), None) + + @unittest.skipUnless(ssl.HAS_TLSv1_3, + "test requires TLSv1.3 enabled OpenSSL") + def test_tls1_3(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + context.load_cert_chain(CERTFILE) + context.options |= ( + ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 | ssl.OP_NO_TLSv1_2 + ) + with ThreadedEchoServer(context=context) as server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + self.assertIn(s.cipher()[0], { + 'TLS_AES_256_GCM_SHA384', + 'TLS_CHACHA20_POLY1305_SHA256', + 'TLS_AES_128_GCM_SHA256', + }) + self.assertEqual(s.version(), 'TLSv1.3') + + @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'), + "required OpenSSL 1.1.0g") + def test_min_max_version(self): + client_context, server_context, hostname = testing_context() + # client TLSv1.0 to 1.2 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # server only TLSv1.2 + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1.2') + + # client 1.0 to 1.2, server 1.0 to 1.1 + server_context.minimum_version = ssl.TLSVersion.TLSv1 + server_context.maximum_version = ssl.TLSVersion.TLSv1_1 + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1.1') + + # client 1.0, server 1.2 (mismatch) + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + client_context.maximum_version = ssl.TLSVersion.TLSv1 + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError) as e: + s.connect((HOST, server.port)) + self.assertIn("alert", str(e.exception)) + + + @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'), + "required OpenSSL 1.1.0g") + @unittest.skipUnless(ssl.HAS_SSLv3, "requires SSLv3 support") + def test_min_max_version_sslv3(self): + client_context, server_context, hostname = testing_context() + server_context.minimum_version = ssl.TLSVersion.SSLv3 + client_context.minimum_version = ssl.TLSVersion.SSLv3 + client_context.maximum_version = ssl.TLSVersion.SSLv3 + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'SSLv3') + + @unittest.skipUnless(ssl.HAS_ECDH, "test requires ECDH-enabled OpenSSL") + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + context.load_cert_chain(CERTFILE) + # TLSv1.3 defaults to PFS key agreement and no longer has KEA in + # cipher name. + context.options |= ssl.OP_NO_TLSv1_3 + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + if ssl.OPENSSL_VERSION_INFO < (1, 0, 0): + context.set_ciphers("ECCdraft:ECDH") + with ThreadedEchoServer(context=context) as server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + server = ThreadedEchoServer(context=server_context, + chatty=True, + connectionchatty=False) + + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write( + " got channel binding data: {0!r}\n".format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + + # now, again + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write( + "got another channel binding data: {0!r}\n".format( + new_cb_data) + ) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + + def test_compression(self): + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_NO_COMPRESSION + server_context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['compression'], None) + + def test_dh_params(self): + # Check we can get a connection with ephemeral Diffie-Hellman + client_context, server_context, hostname = testing_context() + # test scenario needs TLS <= 1.2 + client_context.options |= ssl.OP_NO_TLSv1_3 + server_context.load_dh_params(DHFILE) + server_context.set_ciphers("kEDH") + server_context.options |= ssl.OP_NO_TLSv1_3 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: + self.fail("Non-DH cipher: " + cipher[0]) + + @unittest.skipUnless(HAVE_SECP_CURVES, "needs secp384r1 curve support") + @unittest.skipIf(IS_OPENSSL_1_1_1, "TODO: Test doesn't work on 1.1.1") + def test_ecdh_curve(self): + # server secp384r1, client auto + client_context, server_context, hostname = testing_context() + + server_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + # server auto, client secp384r1 + client_context, server_context, hostname = testing_context() + client_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + # server / client curve mismatch + client_context, server_context, hostname = testing_context() + client_context.set_ecdh_curve("prime256v1") + server_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + try: + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + except ssl.SSLError: + pass + else: + # OpenSSL 1.0.2 does not fail although it should. + if IS_OPENSSL_1_1_0: + self.fail("mismatch curve did not fail") + + def test_selected_alpn_protocol(self): + # selected_alpn_protocol() is None unless ALPN is used. + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support required") + def test_selected_alpn_protocol_if_server_uses_alpn(self): + # selected_alpn_protocol() is None unless ALPN is used by the client. + client_context, server_context, hostname = testing_context() + server_context.set_alpn_protocols(['foo', 'bar']) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['client_alpn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_ALPN, "ALPN support needed for this test") + def test_alpn_protocols(self): + server_protocols = ['foo', 'bar', 'milkshake'] + protocol_tests = [ + (['foo', 'bar'], 'foo'), + (['bar', 'foo'], 'foo'), + (['milkshake'], 'milkshake'), + (['http/3.0', 'http/4.0'], None) + ] + for client_protocols, expected in protocol_tests: + client_context, server_context, hostname = testing_context() + server_context.set_alpn_protocols(server_protocols) + client_context.set_alpn_protocols(client_protocols) + + try: + stats = server_params_test(client_context, + server_context, + chatty=True, + connectionchatty=True, + sni_name=hostname) + except ssl.SSLError as e: + stats = e + + if (expected is None and IS_OPENSSL_1_1_0 + and ssl.OPENSSL_VERSION_INFO < (1, 1, 0, 6)): + # OpenSSL 1.1.0 to 1.1.0e raises handshake error + self.assertIsInstance(stats, ssl.SSLError) + else: + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_alpn_protocol'] + self.assertEqual(client_result, expected, + msg % (client_result, "client")) + server_result = stats['server_alpn_protocols'][-1] \ + if len(stats['server_alpn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, + msg % (server_result, "server")) + + def test_selected_npn_protocol(self): + # selected_npn_protocol() is None unless NPN is used + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['client_npn_protocol'], None) + + @unittest.skipUnless(ssl.HAS_NPN, "NPN support needed for this test") + def test_npn_protocols(self): + server_protocols = ['http/1.1', 'spdy/2'] + protocol_tests = [ + (['http/1.1', 'spdy/2'], 'http/1.1'), + (['spdy/2', 'http/1.1'], 'http/1.1'), + (['spdy/2', 'test'], 'spdy/2'), + (['abc', 'def'], 'abc') + ] + for client_protocols, expected in protocol_tests: + client_context, server_context, hostname = testing_context() + server_context.set_npn_protocols(server_protocols) + client_context.set_npn_protocols(client_protocols) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_npn_protocol'] + self.assertEqual(client_result, expected, msg % (client_result, "client")) + server_result = stats['server_npn_protocols'][-1] \ + if len(stats['server_npn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, msg % (server_result, "server")) + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + @needs_sni + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + client_context.check_hostname = False + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, SIGNED_CERTFILE_HOSTNAME) + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, SIGNED_CERTFILE_HOSTNAME) + self.assertEqual(calls, []) + + @needs_sni + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @needs_sni + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1/0 + server_context.set_servername_callback(cb_raising) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'SSLV3_ALERT_HANDSHAKE_FAILURE') + self.assertIn("ZeroDivisionError", stderr.getvalue()) + + @needs_sni + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with self.assertRaises(ssl.SSLError) as cm, \ + support.captured_stderr() as stderr: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertIn("TypeError", stderr.getvalue()) + + def test_shared_ciphers(self): + client_context, server_context, hostname = testing_context() + client_context.set_ciphers("AES128:AES256") + server_context.set_ciphers("AES256") + expected_algs = [ + "AES256", "AES-256", + # TLS 1.3 ciphers are always enabled + "TLS_CHACHA20", "TLS_AES", + ] + + stats = server_params_test(client_context, server_context, + sni_name=hostname) + ciphers = stats['server_shared_ciphers'][0] + self.assertGreater(len(ciphers), 0) + for name, tls_version, bits in ciphers: + if not any(alg in name for alg in expected_algs): + self.fail(name) + + def test_read_write_after_close_raises_valuerror(self): + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context, chatty=False) + + with server: + s = client_context.wrap_socket(socket.socket(), + server_hostname=hostname) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + def test_sendfile(self): + TEST_DATA = b"x" * 512 + with open(support.TESTFN, 'wb') as f: + f.write(TEST_DATA) + self.addCleanup(support.unlink, support.TESTFN) + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(SIGNING_CA) + context.load_cert_chain(SIGNED_CERTFILE) + server = ThreadedEchoServer(context=context, chatty=False) + with server: + with context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + with open(support.TESTFN, 'rb') as file: + s.sendfile(file) + self.assertEqual(s.recv(1024), TEST_DATA) + + def test_session(self): + client_context, server_context, hostname = testing_context() + # TODO: sessions aren't compatible with TLSv1.3 yet + client_context.options |= ssl.OP_NO_TLSv1_3 + + # first connection without session + stats = server_params_test(client_context, server_context, + sni_name=hostname) + session = stats['session'] + self.assertTrue(session.id) + self.assertGreater(session.time, 0) + self.assertGreater(session.timeout, 0) + self.assertTrue(session.has_ticket) + if ssl.OPENSSL_VERSION_INFO > (1, 0, 1): + self.assertGreater(session.ticket_lifetime_hint, 0) + self.assertFalse(stats['session_reused']) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 1) + self.assertEqual(sess_stat['hits'], 0) + + # reuse session + stats = server_params_test(client_context, server_context, + session=session, sni_name=hostname) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 2) + self.assertEqual(sess_stat['hits'], 1) + self.assertTrue(stats['session_reused']) + session2 = stats['session'] + self.assertEqual(session2.id, session.id) + self.assertEqual(session2, session) + self.assertIsNot(session2, session) + self.assertGreaterEqual(session2.time, session.time) + self.assertGreaterEqual(session2.timeout, session.timeout) + + # another one without session + stats = server_params_test(client_context, server_context, + sni_name=hostname) + self.assertFalse(stats['session_reused']) + session3 = stats['session'] + self.assertNotEqual(session3.id, session.id) + self.assertNotEqual(session3, session) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 3) + self.assertEqual(sess_stat['hits'], 1) + + # reuse session again + stats = server_params_test(client_context, server_context, + session=session, sni_name=hostname) + self.assertTrue(stats['session_reused']) + session4 = stats['session'] + self.assertEqual(session4.id, session.id) + self.assertEqual(session4, session) + self.assertGreaterEqual(session4.time, session.time) + self.assertGreaterEqual(session4.timeout, session.timeout) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 4) + self.assertEqual(sess_stat['hits'], 2) + + def test_session_handling(self): + client_context, server_context, hostname = testing_context() + client_context2, _, _ = testing_context() + + # TODO: session reuse does not work with TLSv1.3 + client_context.options |= ssl.OP_NO_TLSv1_3 + client_context2.options |= ssl.OP_NO_TLSv1_3 + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # session is None before handshake + self.assertEqual(s.session, None) + self.assertEqual(s.session_reused, None) + s.connect((HOST, server.port)) + session = s.session + self.assertTrue(session) + with self.assertRaises(TypeError) as e: + s.session = object + self.assertEqual(str(e.exception), 'Value is not a SSLSession.') + + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # cannot set session after handshake + with self.assertRaises(ValueError) as e: + s.session = session + self.assertEqual(str(e.exception), + 'Cannot set session after handshake.') + + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # can set session before handshake and before the + # connection was established + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(s.session.id, session.id) + self.assertEqual(s.session, session) + self.assertEqual(s.session_reused, True) + + with client_context2.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # cannot re-use session with a different SSLContext + with self.assertRaises(ValueError) as e: + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(str(e.exception), + 'Session refers to a different SSLContext.') + + +def test_main(verbose=False): + if support.verbose: + import warnings + plats = { + 'Linux': platform.linux_distribution, + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', + r'dist\(\) and linux_distribution\(\) ' + 'functions are deprecated .*', + PendingDeprecationWarning, + ) + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + tests = [ + ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests, + SSLObjectTests, SimpleBackgroundTests, ThreadedTests, + ] + + if support.is_resource_enabled('network'): + tests.append(NetworkedTests) + + thread_info = support.threading_setup() + try: + support.run_unittest(*tests) + finally: + support.threading_cleanup(*thread_info) + +if __name__ == "__main__": + test_main() diff --git a/src/greentest/3.7/test_subprocess.py b/src/greentest/3.7/test_subprocess.py new file mode 100644 index 0000000..55d045e --- /dev/null +++ b/src/greentest/3.7/test_subprocess.py @@ -0,0 +1,3229 @@ +import unittest +from unittest import mock +from test import support +import subprocess +import sys +import platform +import signal +import io +import itertools +import os +import errno +import tempfile +import time +import selectors +import sysconfig +import select +import shutil +import threading +import gc +import textwrap +from test.support import FakePath + +try: + import ctypes +except ImportError: + ctypes = None +else: + import ctypes.util + +try: + import _testcapi +except ImportError: + _testcapi = None + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +mswindows = (sys.platform == "win32") + +# +# Depends on the following external programs: Python +# + +if mswindows: + SETBINARY = ('import msvcrt; msvcrt.setmode(sys.stdout.fileno(), ' + 'os.O_BINARY);') +else: + SETBINARY = '' + +NONEXISTING_CMD = ('nonexisting_i_hope',) +# Ignore errors that indicate the command was not found +NONEXISTING_ERRORS = (FileNotFoundError, NotADirectoryError, PermissionError) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + # Try to minimize the number of children we have so this test + # doesn't crash on some buildbots (Alphas in particular). + support.reap_children() + + def tearDown(self): + for inst in subprocess._active: + inst.wait() + subprocess._cleanup() + self.assertFalse(subprocess._active, "subprocess._active not empty") + self.doCleanups() + support.reap_children() + + def assertStderrEqual(self, stderr, expected, msg=None): + # In a debug build, stuff like "[6580 refs]" is printed to stderr at + # shutdown time. That frustrates tests trying to check stderr produced + # from a spawned Python process. + actual = support.strip_python_stderr(stderr) + # strip_python_stderr also strips whitespace, so we do too. + expected = expected.strip() + self.assertEqual(actual, expected, msg) + + +class PopenTestException(Exception): + pass + + +class PopenExecuteChildRaises(subprocess.Popen): + """Popen subclass for testing cleanup of subprocess.PIPE filehandles when + _execute_child fails. + """ + def _execute_child(self, *args, **kwargs): + raise PopenTestException("Forced Exception for Test") + + +class ProcessTestCase(BaseTestCase): + + def test_io_buffered_by_default(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + self.assertIsInstance(p.stdin, io.BufferedIOBase) + self.assertIsInstance(p.stdout, io.BufferedIOBase) + self.assertIsInstance(p.stderr, io.BufferedIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_io_unbuffered_works(self): + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, bufsize=0) + try: + self.assertIsInstance(p.stdin, io.RawIOBase) + self.assertIsInstance(p.stdout, io.RawIOBase) + self.assertIsInstance(p.stderr, io.RawIOBase) + finally: + p.stdin.close() + p.stdout.close() + p.stderr.close() + p.wait() + + def test_call_seq(self): + # call() function with sequence argument + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(rc, 47) + + def test_call_timeout(self): + # call() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.call waits for the + # child. + self.assertRaises(subprocess.TimeoutExpired, subprocess.call, + [sys.executable, "-c", "while True: pass"], + timeout=0.1) + + def test_check_call_zero(self): + # check_call() function with zero return code + rc = subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(0)"]) + self.assertEqual(rc, 0) + + def test_check_call_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_call([sys.executable, "-c", + "import sys; sys.exit(47)"]) + self.assertEqual(c.exception.returncode, 47) + + def test_check_output(self): + # check_output() function with zero return code + output = subprocess.check_output( + [sys.executable, "-c", "print('BDFL')"]) + self.assertIn(b'BDFL', output) + + def test_check_output_nonzero(self): + # check_call() function with non-zero return code + with self.assertRaises(subprocess.CalledProcessError) as c: + subprocess.check_output( + [sys.executable, "-c", "import sys; sys.exit(5)"]) + self.assertEqual(c.exception.returncode, 5) + + def test_check_output_stderr(self): + # check_output() function stderr redirected to stdout + output = subprocess.check_output( + [sys.executable, "-c", "import sys; sys.stderr.write('BDFL')"], + stderr=subprocess.STDOUT) + self.assertIn(b'BDFL', output) + + def test_check_output_stdin_arg(self): + # check_output() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + stdin=tf) + self.assertIn(b'PEAR', output) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + output = subprocess.check_output( + [sys.executable, "-c", + "import sys; sys.stdout.write(sys.stdin.read().upper())"], + input=b'pear') + self.assertIn(b'PEAR', output) + + def test_check_output_stdout_arg(self): + # check_output() refuses to accept 'stdout' argument + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdout=sys.stdout) + self.fail("Expected ValueError when stdout arg supplied.") + self.assertIn('stdout', c.exception.args[0]) + + def test_check_output_stdin_with_input_arg(self): + # check_output() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError) as c: + output = subprocess.check_output( + [sys.executable, "-c", "print('will not be run')"], + stdin=tf, input=b'hare') + self.fail("Expected ValueError when stdin and input args supplied.") + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + # check_output() function with timeout arg + with self.assertRaises(subprocess.TimeoutExpired) as c: + output = subprocess.check_output( + [sys.executable, "-c", + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"], + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3) + self.fail("Expected TimeoutExpired.") + self.assertEqual(c.exception.output, b'BDFL') + + def test_call_kwargs(self): + # call() function with keyword args + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + rc = subprocess.call([sys.executable, "-c", + 'import sys, os;' + 'sys.exit(os.getenv("FRUIT")=="banana")'], + env=newenv) + self.assertEqual(rc, 1) + + def test_invalid_args(self): + # Popen() called with invalid arguments should raise TypeError + # but Popen.__del__ should not complain (issue #12085) + with support.captured_stderr() as s: + self.assertRaises(TypeError, subprocess.Popen, invalid_arg_name=1) + argcount = subprocess.Popen.__init__.__code__.co_argcount + too_many_args = [0] * (argcount + 1) + self.assertRaises(TypeError, subprocess.Popen, *too_many_args) + self.assertEqual(s.getvalue(), '') + + def test_stdin_none(self): + # .stdin is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + p.wait() + self.assertEqual(p.stdin, None) + + def test_stdout_none(self): + # .stdout is None when not redirected, and the child's stdout will + # be inherited from the parent. In order to test this we run a + # subprocess in a subprocess: + # this_test + # \-- subprocess created by this test (parent) + # \-- subprocess created by the parent subprocess (child) + # The parent doesn't specify stdout, so the child will use the + # parent's stdout. This test checks that the message printed by the + # child goes to the parent stdout. The parent also checks that the + # child's stdout is None. See #11963. + code = ('import sys; from subprocess import Popen, PIPE;' + 'p = Popen([sys.executable, "-c", "print(\'test_stdout_none\')"],' + ' stdin=PIPE, stderr=PIPE);' + 'p.wait(); assert p.stdout is None;') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test_stdout_none') + + def test_stderr_none(self): + # .stderr is None when not redirected + p = subprocess.Popen([sys.executable, "-c", 'print("banana")'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + p.wait() + self.assertEqual(p.stderr, None) + + def _assert_python(self, pre_args, **kwargs): + # We include sys.exit() to prevent the test runner from hanging + # whenever python is found. + args = pre_args + ["import sys; sys.exit(47)"] + p = subprocess.Popen(args, **kwargs) + p.wait() + self.assertEqual(47, p.returncode) + + def test_executable(self): + # Check that the executable argument works. + # + # On Unix (non-Mac and non-Windows), Python looks at args[0] to + # determine where its standard library is, so we need the directory + # of args[0] to be valid for the Popen() call to Python to succeed. + # See also issue #16170 and issue #7774. + doesnotexist = os.path.join(os.path.dirname(sys.executable), + "doesnotexist") + self._assert_python([doesnotexist, "-c"], executable=sys.executable) + + def test_executable_takes_precedence(self): + # Check that the executable argument takes precedence over args[0]. + # + # Verify first that the call succeeds without the executable arg. + pre_args = [sys.executable, "-c"] + self._assert_python(pre_args) + self.assertRaises(NONEXISTING_ERRORS, + self._assert_python, pre_args, + executable=NONEXISTING_CMD[0]) + + @unittest.skipIf(mswindows, "executable argument replaces shell") + def test_executable_replaces_shell(self): + # Check that the executable argument replaces the default shell + # when shell=True. + self._assert_python([], executable=sys.executable, shell=True) + + # For use in the test_cwd* tests below. + def _normalize_cwd(self, cwd): + # Normalize an expected cwd (for Tru64 support). + # We can't use os.path.realpath since it doesn't expand Tru64 {memb} + # strings. See bug #1063571. + with support.change_cwd(cwd): + return os.getcwd() + + # For use in the test_cwd* tests below. + def _split_python_path(self): + # Return normalized (python_dir, python_base). + python_path = os.path.realpath(sys.executable) + return os.path.split(python_path) + + # For use in the test_cwd* tests below. + def _assert_cwd(self, expected_cwd, python_arg, **kwargs): + # Invoke Python via Popen, and assert that (1) the call succeeds, + # and that (2) the current working directory of the child process + # matches *expected_cwd*. + p = subprocess.Popen([python_arg, "-c", + "import os, sys; " + "sys.stdout.write(os.getcwd()); " + "sys.exit(47)"], + stdout=subprocess.PIPE, + **kwargs) + self.addCleanup(p.stdout.close) + p.wait() + self.assertEqual(47, p.returncode) + normcase = os.path.normcase + self.assertEqual(normcase(expected_cwd), + normcase(p.stdout.read().decode("utf-8"))) + + def test_cwd(self): + # Check that cwd changes the cwd for the child process. + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + self._assert_cwd(temp_dir, sys.executable, cwd=temp_dir) + + def test_cwd_with_pathlike(self): + temp_dir = tempfile.gettempdir() + temp_dir = self._normalize_cwd(temp_dir) + self._assert_cwd(temp_dir, sys.executable, cwd=FakePath(temp_dir)) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_arg(self): + # Check that Popen looks for args[0] relative to cwd if args[0] + # is relative. + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + with support.temp_cwd('test_cwd_with_relative_arg' + str(os.getpid()), quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure) + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python]) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, rel_python, cwd=python_dir) + + @unittest.skipIf(mswindows, "pending resolution of issue #15533") + def test_cwd_with_relative_executable(self): + # Check that Popen looks for executable relative to cwd if executable + # is relative (and that executable takes precedence over args[0]). + python_dir, python_base = self._split_python_path() + rel_python = os.path.join(os.curdir, python_base) + doesntexist = "somethingyoudonthave" + with support.temp_cwd('test_cwd_with_relative_executable' + str(os.getpid()), quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure) + # Before calling with the correct cwd, confirm that the call fails + # without cwd and with the wrong cwd. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python) + self.assertRaises(FileNotFoundError, subprocess.Popen, + [doesntexist], executable=rel_python, + cwd=wrong_dir) + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, doesntexist, executable=rel_python, + cwd=python_dir) + + def test_cwd_with_absolute_arg(self): + # Check that Popen can find the executable when the cwd is wrong + # if args[0] is an absolute path. + python_dir, python_base = self._split_python_path() + abs_python = os.path.join(python_dir, python_base) + rel_python = os.path.join(os.curdir, python_base) + with support.temp_cwd('test_cwd_with_absolute_arg' + str(os.getpid()), quiet=True) as wrong_dir: # gevent: use distinct name, avoid Travis CI failure) + # Before calling with an absolute path, confirm that using a + # relative path fails. + self.assertRaises(FileNotFoundError, subprocess.Popen, + [rel_python], cwd=wrong_dir) + wrong_dir = self._normalize_cwd(wrong_dir) + self._assert_cwd(wrong_dir, abs_python, cwd=wrong_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + def test_executable_with_cwd(self): + python_dir, python_base = self._split_python_path() + python_dir = self._normalize_cwd(python_dir) + self._assert_cwd(python_dir, "somethingyoudonthave", + executable=sys.executable, cwd=python_dir) + + @unittest.skipIf(sys.base_prefix != sys.prefix, + 'Test is not venv-compatible') + @unittest.skipIf(sysconfig.is_python_build(), + "need an installed Python. See #7774") + def test_executable_without_cwd(self): + # For a normal installation, it should work without 'cwd' + # argument. For test runs in the build directory, see #7774. + self._assert_cwd(os.getcwd(), "somethingyoudonthave", + executable=sys.executable) + + def test_stdin_pipe(self): + # stdin redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.stdin.write(b"pear") + p.stdin.close() + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_filedes(self): + # stdin is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + os.write(d, b"pear") + os.lseek(d, 0, 0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=d) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdin_fileobj(self): + # stdin is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b"pear") + tf.seek(0) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(sys.stdin.read() == "pear")'], + stdin=tf) + p.wait() + self.assertEqual(p.returncode, 1) + + def test_stdout_pipe(self): + # stdout redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=subprocess.PIPE) + with p: + self.assertEqual(p.stdout.read(), b"orange") + + def test_stdout_filedes(self): + # stdout is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=d) + p.wait() + os.lseek(d, 0, 0) + self.assertEqual(os.read(d, 1024), b"orange") + + def test_stdout_fileobj(self): + # stdout is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("orange")'], + stdout=tf) + p.wait() + tf.seek(0) + self.assertEqual(tf.read(), b"orange") + + def test_stderr_pipe(self): + # stderr redirection + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=subprocess.PIPE) + with p: + self.assertStderrEqual(p.stderr.read(), b"strawberry") + + def test_stderr_filedes(self): + # stderr is set to open file descriptor + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + d = tf.fileno() + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=d) + p.wait() + os.lseek(d, 0, 0) + self.assertStderrEqual(os.read(d, 1024), b"strawberry") + + def test_stderr_fileobj(self): + # stderr is set to open file object + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("strawberry")'], + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"strawberry") + + def test_stderr_redirect_with_no_stdout_redirect(self): + # test stderr=STDOUT while stdout=None (not set) + + # - grandchild prints to stderr + # - child redirects grandchild's stderr to its stdout + # - the parent should get grandchild's stderr in child's stdout + p = subprocess.Popen([sys.executable, "-c", + 'import sys, subprocess;' + 'rc = subprocess.call([sys.executable, "-c",' + ' "import sys;"' + ' "sys.stderr.write(\'42\')"],' + ' stderr=subprocess.STDOUT);' + 'sys.exit(rc)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + #NOTE: stdout should get stderr from grandchild + self.assertStderrEqual(stdout, b'42') + self.assertStderrEqual(stderr, b'') # should be empty + self.assertEqual(p.returncode, 0) + + def test_stdout_stderr_pipe(self): + # capture stdout and stderr to the same pipe + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + with p: + self.assertStderrEqual(p.stdout.read(), b"appleorange") + + def test_stdout_stderr_file(self): + # capture stdout and stderr to the same open file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdout=tf, + stderr=tf) + p.wait() + tf.seek(0) + self.assertStderrEqual(tf.read(), b"appleorange") + + def test_stdout_filedes_of_stdout(self): + # stdout is set to 1 (#1531862). + # To avoid printing the text on stdout, we do something similar to + # test_stdout_none (see above). The parent subprocess calls the child + # subprocess passing stdout=1, and this test uses stdout=PIPE in + # order to capture and check the output of the parent. See #11963. + code = ('import sys, subprocess; ' + 'rc = subprocess.call([sys.executable, "-c", ' + ' "import os, sys; sys.exit(os.write(sys.stdout.fileno(), ' + 'b\'test with stdout=1\'))"], stdout=1); ' + 'assert rc == 18') + p = subprocess.Popen([sys.executable, "-c", code], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + out, err = p.communicate() + self.assertEqual(p.returncode, 0, err) + self.assertEqual(out.rstrip(), b'test with stdout=1') + + def test_stdout_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'for i in range(10240):' + 'print("x" * 1024)'], + stdout=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdout, None) + + def test_stderr_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys\n' + 'for i in range(10240):' + 'sys.stderr.write("x" * 1024)'], + stderr=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stderr, None) + + def test_stdin_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdin.read(1)'], + stdin=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdin, None) + + def test_env(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "orange" + with subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange") + + # Windows requires at least the SYSTEMROOT environment variable to start + # Python + @unittest.skipIf(sys.platform == 'win32', + 'cannot test an empty env on Windows') + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') == 1, + 'The Python shared library cannot be loaded ' + 'with an empty environment.') + def test_empty_env(self): + """Verify that env={} is as empty as possible.""" + + def is_env_var_to_ignore(n): + """Determine if an environment variable is under our control.""" + # This excludes some __CF_* and VERSIONER_* keys MacOS insists + # on adding even when the environment in exec is empty. + # Gentoo sandboxes also force LD_PRELOAD and SANDBOX_* to exist. + return ('VERSIONER' in n or '__CF' in n or # MacOS + '__PYVENV_LAUNCHER__' in n or # MacOS framework build + n == 'LD_PRELOAD' or n.startswith('SANDBOX') or # Gentoo + n == 'LC_CTYPE') # Locale coercion triggered + + with subprocess.Popen([sys.executable, "-c", + 'import os; print(list(os.environ.keys()))'], + stdout=subprocess.PIPE, env={}) as p: + stdout, stderr = p.communicate() + child_env_names = eval(stdout.strip()) + self.assertIsInstance(child_env_names, list) + child_env_names = [k for k in child_env_names + if not is_env_var_to_ignore(k)] + self.assertEqual(child_env_names, []) + + def test_invalid_cmd(self): + # null character in the command name + cmd = sys.executable + '\0' + with self.assertRaises(ValueError): + subprocess.Popen([cmd, "-c", "pass"]) + + # null character in the command argument + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass#\0"]) + + def test_invalid_env(self): + # null character in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # null character in the environment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + subprocess.Popen([sys.executable, "-c", "pass"], env=newenv) + + # equal character in the environment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + with subprocess.Popen([sys.executable, "-c", + 'import sys, os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + env=newenv) as p: + stdout, stderr = p.communicate() + self.assertEqual(stdout, b"orange=lemon") + + def test_communicate_stdin(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.exit(sys.stdin.read() == "pear")'], + stdin=subprocess.PIPE) + p.communicate(b"pear") + self.assertEqual(p.returncode, 1) + + def test_communicate_stdout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write("pineapple")'], + stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, b"pineapple") + self.assertEqual(stderr, None) + + def test_communicate_stderr(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stderr.write("pineapple")'], + stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stderr.write("pineapple");' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(b"banana") + self.assertEqual(stdout, b"banana") + self.assertStderrEqual(stderr, b"pineapple") + + def test_communicate_timeout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stderr.write("pineapple\\n");' + 'time.sleep(1);' + 'sys.stderr.write("pear\\n");' + 'sys.stdout.write(sys.stdin.read())'], + universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, "banana", + timeout=0.3) + # Make sure we can keep waiting for it, and that we get the whole output + # after it completes. + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr.encode(), b"pineapple\npear\n") + + def test_communicate_timeout_large_output(self): + # Test an expiring timeout while the child is outputting lots of data. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));'], + stdout=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, timeout=0.4) + (stdout, _) = p.communicate() + self.assertEqual(len(stdout), 4 * 64 * 1024) + + # Test for the fd leak reported in http://bugs.python.org/issue2791. + def test_communicate_pipe_fd_leak(self): + for stdin_pipe in (False, True): + for stdout_pipe in (False, True): + for stderr_pipe in (False, True): + options = {} + if stdin_pipe: + options['stdin'] = subprocess.PIPE + if stdout_pipe: + options['stdout'] = subprocess.PIPE + if stderr_pipe: + options['stderr'] = subprocess.PIPE + if not options: + continue + p = subprocess.Popen((sys.executable, "-c", "pass"), **options) + p.communicate() + if p.stdin is not None: + self.assertTrue(p.stdin.closed) + if p.stdout is not None: + self.assertTrue(p.stdout.closed) + if p.stderr is not None: + self.assertTrue(p.stderr.closed) + + def test_communicate_returns(self): + # communicate() should return None if no redirection is active + p = subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(47)"]) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, None) + self.assertEqual(stderr, None) + + def test_communicate_pipe_buf(self): + # communicate() with writes larger than pipe_buf + # This test will probably deadlock rather than fail, if + # communicate() does not work properly. + x, y = os.pipe() + os.close(x) + os.close(y) + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read(47));' + 'sys.stderr.write("x" * %d);' + 'sys.stdout.write(sys.stdin.read())' % + support.PIPE_MAX_SIZE], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + string_to_write = b"a" * support.PIPE_MAX_SIZE + (stdout, stderr) = p.communicate(string_to_write) + self.assertEqual(stdout, string_to_write) + + def test_writes_before_communicate(self): + # stdin.write before communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.stdin.write(b"banana") + (stdout, stderr) = p.communicate(b"split") + self.assertEqual(stdout, b"bananasplit") + self.assertStderrEqual(stderr, b"") + + def test_universal_newlines_and_text(self): + args = [ + sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(sys.stdin.readline().encode());' + 'buf.flush();' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(sys.stdin.read().encode());' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'] + + for extra_kwarg in ('universal_newlines', 'text'): + p = subprocess.Popen(args, **{'stdin': subprocess.PIPE, + 'stdout': subprocess.PIPE, + extra_kwarg: True}) + with p: + p.stdin.write("line1\n") + p.stdin.flush() + self.assertEqual(p.stdout.readline(), "line1\n") + p.stdin.write("line3\n") + p.stdin.close() + self.addCleanup(p.stdout.close) + self.assertEqual(p.stdout.readline(), + "line2\n") + self.assertEqual(p.stdout.read(6), + "line3\n") + self.assertEqual(p.stdout.read(), + "line4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate(self): + # universal newlines through communicate() + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + + 'buf = sys.stdout.buffer;' + 'buf.write(b"line2\\n");' + 'buf.flush();' + 'buf.write(b"line4\\n");' + 'buf.flush();' + 'buf.write(b"line5\\r\\n");' + 'buf.flush();' + 'buf.write(b"line6\\r");' + 'buf.flush();' + 'buf.write(b"\\nline7");' + 'buf.flush();' + 'buf.write(b"\\nline8");'], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=1) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, + "line2\nline4\nline5\nline6\nline7\nline8") + + def test_universal_newlines_communicate_stdin(self): + # universal newlines through communicate(), with only stdin + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.readline() + assert s == "line1\\n", repr(s) + s = sys.stdin.read() + assert s == "line3\\n", repr(s) + ''')], + stdin=subprocess.PIPE, + universal_newlines=1) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_input_none(self): + # Test communicate(input=None) with universal newlines. + # + # We set stdout to PIPE because, as of this writing, a different + # code path is tested when the number of pipes is zero or one. + p = subprocess.Popen([sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + p.communicate() + self.assertEqual(p.returncode, 0) + + def test_universal_newlines_communicate_stdin_stdout_stderr(self): + # universal newlines through communicate(), with stdin, stdout, stderr + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.buffer.readline() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line2\\r") + sys.stderr.buffer.write(b"eline2\\n") + s = sys.stdin.buffer.read() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line4\\n") + sys.stdout.buffer.write(b"line5\\r\\n") + sys.stderr.buffer.write(b"eline6\\r") + sys.stderr.buffer.write(b"eline7\\r\\nz") + ''')], + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) + # Python debug build push something like "[42442 refs]\n" + # to stderr at exit of subprocess. + # Don't use assertStderrEqual because it strips CR and LF from output. + self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + + def test_universal_newlines_communicate_encodings(self): + # Check that universal newlines mode works for various encodings, + # in particular for encodings in the UTF-16 and UTF-32 families. + # See issue #15595. + # + # UTF-16 and UTF-32-BE are sufficient to check both with BOM and + # without, and UTF-16 and UTF-32. + for encoding in ['utf-16', 'utf-32-be']: + code = ("import sys; " + r"sys.stdout.buffer.write('1\r\n2\r3\n4'.encode('%s'))" % + encoding) + args = [sys.executable, '-c', code] + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding=encoding) + stdout, stderr = popen.communicate(input='') + self.assertEqual(stdout, '1\n2\n3\n4') + + def test_communicate_errors(self): + for errors, expected in [ + ('ignore', ''), + ('replace', '\ufffd\ufffd'), + ('surrogateescape', '\udc80\udc80'), + ('backslashreplace', '\\x80\\x80'), + ]: + code = ("import sys; " + r"sys.stdout.buffer.write(b'[\x80\x80]')") + args = [sys.executable, '-c', code] + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding='utf-8', + errors=errors) + stdout, stderr = popen.communicate(input='') + self.assertEqual(stdout, '[{}]'.format(expected)) + + def test_no_leaking(self): + # Make sure we leak no resources + if not mswindows: + max_handles = 1026 # too much for most UNIX systems + else: + max_handles = 2050 # too much for (at least some) Windows setups + handles = [] + tmpdir = tempfile.mkdtemp() + try: + for i in range(max_handles): + try: + tmpfile = os.path.join(tmpdir, support.TESTFN) + handles.append(os.open(tmpfile, os.O_WRONLY|os.O_CREAT)) + except OSError as e: + if e.errno != errno.EMFILE: + raise + break + else: + self.skipTest("failed to reach the file descriptor limit " + "(tried %d)" % max_handles) + # Close a couple of them (should be enough for a subprocess) + for i in range(10): + os.close(handles.pop()) + # Loop creating some subprocesses. If one of them leaks some fds, + # the next loop iteration will fail by reaching the max fd limit. + for i in range(15): + p = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write(sys.stdin.read())"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + data = p.communicate(b"lime")[0] + self.assertEqual(data, b"lime") + finally: + for h in handles: + os.close(h) + shutil.rmtree(tmpdir) + + def test_list2cmdline(self): + self.assertEqual(subprocess.list2cmdline(['a b c', 'd', 'e']), + '"a b c" d e') + self.assertEqual(subprocess.list2cmdline(['ab"c', '\\', 'd']), + 'ab\\"c \\ d') + self.assertEqual(subprocess.list2cmdline(['ab"c', ' \\', 'd']), + 'ab\\"c " \\\\" d') + self.assertEqual(subprocess.list2cmdline(['a\\\\\\b', 'de fg', 'h']), + 'a\\\\\\b "de fg" h') + self.assertEqual(subprocess.list2cmdline(['a\\"b', 'c', 'd']), + 'a\\\\\\"b c d') + self.assertEqual(subprocess.list2cmdline(['a\\\\b c', 'd', 'e']), + '"a\\\\b c" d e') + self.assertEqual(subprocess.list2cmdline(['a\\\\b\\ c', 'd', 'e']), + '"a\\\\b\\ c" d e') + self.assertEqual(subprocess.list2cmdline(['ab', '']), + 'ab ""') + + def test_poll(self): + p = subprocess.Popen([sys.executable, "-c", + "import os; os.read(0, 1)"], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + self.assertIsNone(p.poll()) + os.write(p.stdin.fileno(), b'A') + p.wait() + # Subsequent invocations should just return the returncode + self.assertEqual(p.poll(), 0) + + def test_wait(self): + p = subprocess.Popen([sys.executable, "-c", "pass"]) + self.assertEqual(p.wait(), 0) + # Subsequent invocations should just return the returncode + self.assertEqual(p.wait(), 0) + + def test_wait_timeout(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(0.3)"]) + with self.assertRaises(subprocess.TimeoutExpired) as c: + p.wait(timeout=0.0001) + self.assertIn("0.0001", str(c.exception)) # For coverage of __str__. + # Some heavily loaded buildbots (sparc Debian 3.x) require this much + # time to start. + self.assertEqual(p.wait(timeout=3), 0) + + def test_invalid_bufsize(self): + # an invalid type of the bufsize argument should raise + # TypeError. + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], "orange") + + def test_bufsize_is_none(self): + # bufsize=None should be the same as bufsize=0. + p = subprocess.Popen([sys.executable, "-c", "pass"], None) + self.assertEqual(p.wait(), 0) + # Again with keyword arg + p = subprocess.Popen([sys.executable, "-c", "pass"], bufsize=None) + self.assertEqual(p.wait(), 0) + + def _test_bufsize_equal_one(self, line, expected, universal_newlines): + # subprocess may deadlock with bufsize=1, see issue #21332 + with subprocess.Popen([sys.executable, "-c", "import sys;" + "sys.stdout.write(sys.stdin.readline());" + "sys.stdout.flush()"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + bufsize=1, + universal_newlines=universal_newlines) as p: + p.stdin.write(line) # expect that it flushes the line in text mode + os.close(p.stdin.fileno()) # close it without flushing the buffer + read_line = p.stdout.readline() + with support.SuppressCrashReport(): + try: + p.stdin.close() + except OSError: + pass + p.stdin = None + self.assertEqual(p.returncode, 0) + self.assertEqual(read_line, expected) + + def test_bufsize_equal_one_text_mode(self): + # line is flushed in text mode with bufsize=1. + # we should get the full line in return + line = "line\n" + self._test_bufsize_equal_one(line, line, universal_newlines=True) + + def test_bufsize_equal_one_binary_mode(self): + # line is not flushed in binary mode with bufsize=1. + # we should get empty response + line = b'line' + os.linesep.encode() # assume ascii-based locale + self._test_bufsize_equal_one(line, b'', universal_newlines=False) + + def test_leaking_fds_on_error(self): + # see bug #5179: Popen leaks file descriptors to PIPEs if + # the child fails to execute; this will eventually exhaust + # the maximum number of open fds. 1024 seems a very common + # value for that limit, but Windows has 2048, so we loop + # 1024 times (each call leaked two fds). + for i in range(1024): + with self.assertRaises(NONEXISTING_ERRORS): + subprocess.Popen(NONEXISTING_CMD, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + def test_nonexisting_with_pipes(self): + # bpo-30121: Popen with pipes must close properly pipes on error. + # Previously, os.close() was called with a Windows handle which is not + # a valid file descriptor. + # + # Run the test in a subprocess to control how the CRT reports errors + # and to get stderr content. + try: + import msvcrt + msvcrt.CrtSetReportMode + except (AttributeError, ImportError): + self.skipTest("need msvcrt.CrtSetReportMode") + + code = textwrap.dedent(f""" + import msvcrt + import subprocess + + cmd = {NONEXISTING_CMD!r} + + for report_type in [msvcrt.CRT_WARN, + msvcrt.CRT_ERROR, + msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(report_type, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(report_type, msvcrt.CRTDBG_FILE_STDERR) + + try: + subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError: + pass + """) + cmd = [sys.executable, "-c", code] + proc = subprocess.Popen(cmd, + stderr=subprocess.PIPE, + universal_newlines=True) + with proc: + stderr = proc.communicate()[1] + self.assertEqual(stderr, "") + self.assertEqual(proc.returncode, 0) + + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(NONEXISTING_CMD, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + + def test_threadsafe_wait(self): + """Issue21291: Popen.wait() needs to be threadsafe for returncode.""" + proc = subprocess.Popen([sys.executable, '-c', + 'import time; time.sleep(12)']) + self.assertEqual(proc.returncode, None) + results = [] + + def kill_proc_timer_thread(): + results.append(('thread-start-poll-result', proc.poll())) + # terminate it from the thread and wait for the result. + proc.kill() + proc.wait() + results.append(('thread-after-kill-and-wait', proc.returncode)) + # this wait should be a no-op given the above. + proc.wait() + results.append(('thread-after-second-wait', proc.returncode)) + + # This is a timing sensitive test, the failure mode is + # triggered when both the main thread and this thread are in + # the wait() call at once. The delay here is to allow the + # main thread to most likely be blocked in its wait() call. + t = threading.Timer(0.2, kill_proc_timer_thread) + t.start() + + if mswindows: + expected_errorcode = 1 + else: + # Should be -9 because of the proc.kill() from the thread. + expected_errorcode = -9 + + # Wait for the process to finish; the thread should kill it + # long before it finishes on its own. Supplying a timeout + # triggers a different code path for better coverage. + proc.wait(timeout=20) + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in wait from main thread") + + # This should be a no-op with no change in returncode. + proc.wait() + self.assertEqual(proc.returncode, expected_errorcode, + msg="unexpected result in second main wait.") + + t.join() + # Ensure that all of the thread results are as expected. + # When a race condition occurs in wait(), the returncode could + # be set by the wrong thread that doesn't actually have it + # leading to an incorrect value. + self.assertEqual([('thread-start-poll-result', None), + ('thread-after-kill-and-wait', expected_errorcode), + ('thread-after-second-wait', expected_errorcode)], + results) + + def test_issue8780(self): + # Ensure that stdout is inherited from the parent + # if stdout=PIPE is not used + code = ';'.join(( + 'import subprocess, sys', + 'retcode = subprocess.call(' + "[sys.executable, '-c', 'print(\"Hello World!\")'])", + 'assert retcode == 0')) + output = subprocess.check_output([sys.executable, '-c', code]) + self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + + def test_handles_closed_on_exception(self): + # If CreateProcess exits with an error, ensure the + # duplicate output handles are released + ifhandle, ifname = tempfile.mkstemp() + ofhandle, ofname = tempfile.mkstemp() + efhandle, efname = tempfile.mkstemp() + try: + subprocess.Popen (["*"], stdin=ifhandle, stdout=ofhandle, + stderr=efhandle) + except OSError: + os.close(ifhandle) + os.remove(ifname) + os.close(ofhandle) + os.remove(ofname) + os.close(efhandle) + os.remove(efname) + self.assertFalse(os.path.exists(ifname)) + self.assertFalse(os.path.exists(ofname)) + self.assertFalse(os.path.exists(efname)) + + def test_communicate_epipe(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + self.addCleanup(p.stdin.close) + p.communicate(b"x" * 2**20) + + def test_communicate_epipe_only_stdin(self): + # Issue 10963: communicate() should hide EPIPE + p = subprocess.Popen([sys.executable, "-c", 'pass'], + stdin=subprocess.PIPE) + self.addCleanup(p.stdin.close) + p.wait() + p.communicate(b"x" * 2**20) + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), + "Requires signal.SIGUSR1") + @unittest.skipUnless(hasattr(os, 'kill'), + "Requires os.kill") + @unittest.skipUnless(hasattr(os, 'getppid'), + "Requires os.getppid") + def test_communicate_eintr(self): + # Issue #12493: communicate() should handle EINTR + def handler(signum, frame): + pass + old_handler = signal.signal(signal.SIGUSR1, handler) + self.addCleanup(signal.signal, signal.SIGUSR1, old_handler) + + args = [sys.executable, "-c", + 'import os, signal;' + 'os.kill(os.getppid(), signal.SIGUSR1)'] + for stream in ('stdout', 'stderr'): + kw = {stream: subprocess.PIPE} + with subprocess.Popen(args, **kw) as process: + # communicate() will be interrupted by SIGUSR1 + process.communicate() + + + # This test is Linux-ish specific for simplicity to at least have + # some coverage. It is not a platform specific bug. + @unittest.skipUnless(os.path.isdir('/proc/%d/fd' % os.getpid()), + "Linux specific") + def test_failed_child_execute_fd_leak(self): + """Test for the fork() failure fd leak reported in issue16327.""" + fd_directory = '/proc/%d/fd' % os.getpid() + fds_before_popen = os.listdir(fd_directory) + with self.assertRaises(PopenTestException): + PopenExecuteChildRaises( + [sys.executable, '-c', 'pass'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # NOTE: This test doesn't verify that the real _execute_child + # does not close the file descriptors itself on the way out + # during an exception. Code inspection has confirmed that. + + fds_after_exception = os.listdir(fd_directory) + self.assertEqual(fds_before_popen, fds_after_exception) + + @unittest.skipIf(mswindows, "behavior currently not supported on Windows") + def test_file_not_found_includes_filename(self): + with self.assertRaises(FileNotFoundError) as c: + subprocess.call(['/opt/nonexistent_binary', 'with', 'some', 'args']) + self.assertEqual(c.exception.filename, '/opt/nonexistent_binary') + + @unittest.skipIf(mswindows, "behavior currently not supported on Windows") + def test_file_not_found_with_bad_cwd(self): + with self.assertRaises(FileNotFoundError) as c: + subprocess.Popen(['exit', '0'], cwd='/some/nonexistent/directory') + self.assertEqual(c.exception.filename, '/some/nonexistent/directory') + + +class RunFuncTestCase(BaseTestCase): + def run_python(self, code, **kwargs): + """Run Python code in a subprocess using subprocess.run""" + argv = [sys.executable, "-c", code] + return subprocess.run(argv, **kwargs) + + def test_returncode(self): + # call() function with sequence argument + cp = self.run_python("import sys; sys.exit(47)") + self.assertEqual(cp.returncode, 47) + with self.assertRaises(subprocess.CalledProcessError): + cp.check_returncode() + + def test_check(self): + with self.assertRaises(subprocess.CalledProcessError) as c: + self.run_python("import sys; sys.exit(47)", check=True) + self.assertEqual(c.exception.returncode, 47) + + def test_check_zero(self): + # check_returncode shouldn't raise when returncode is zero + cp = self.run_python("import sys; sys.exit(0)", check=True) + self.assertEqual(cp.returncode, 0) + + def test_timeout(self): + # run() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.run waits for the + # child. + with self.assertRaises(subprocess.TimeoutExpired): + self.run_python("while True: pass", timeout=0.0001) + + def test_capture_stdout(self): + # capture stdout with zero return code + cp = self.run_python("print('BDFL')", stdout=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stdout) + + def test_capture_stderr(self): + cp = self.run_python("import sys; sys.stderr.write('BDFL')", + stderr=subprocess.PIPE) + self.assertIn(b'BDFL', cp.stderr) + + def test_check_output_stdin_arg(self): + # run() can be called with stdin set to a file + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + stdin=tf, stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_input_arg(self): + # check_output() can be called with input set to a string + cp = self.run_python( + "import sys; sys.stdout.write(sys.stdin.read().upper())", + input=b'pear', stdout=subprocess.PIPE) + self.assertIn(b'PEAR', cp.stdout) + + def test_check_output_stdin_with_input_arg(self): + # run() refuses to accept 'stdin' with 'input' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + tf.write(b'pear') + tf.seek(0) + with self.assertRaises(ValueError, + msg="Expected ValueError when stdin and input args supplied.") as c: + output = self.run_python("print('will not be run')", + stdin=tf, input=b'hare') + self.assertIn('stdin', c.exception.args[0]) + self.assertIn('input', c.exception.args[0]) + + def test_check_output_timeout(self): + with self.assertRaises(subprocess.TimeoutExpired) as c: + cp = self.run_python(( + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"), + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3, stdout=subprocess.PIPE) + self.assertEqual(c.exception.output, b'BDFL') + # output is aliased to stdout + self.assertEqual(c.exception.stdout, b'BDFL') + + def test_run_kwargs(self): + newenv = os.environ.copy() + newenv["FRUIT"] = "banana" + cp = self.run_python(('import sys, os;' + 'sys.exit(33 if os.getenv("FRUIT")=="banana" else 31)'), + env=newenv) + self.assertEqual(cp.returncode, 33) + + def test_capture_output(self): + cp = self.run_python(("import sys;" + "sys.stdout.write('BDFL'); " + "sys.stderr.write('FLUFL')"), + capture_output=True) + self.assertIn(b'BDFL', cp.stdout) + self.assertIn(b'FLUFL', cp.stderr) + + def test_stdout_with_capture_output_arg(self): + # run() refuses to accept 'stdout' with 'capture_output' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + with self.assertRaises(ValueError, + msg=("Expected ValueError when stdout and capture_output " + "args supplied.")) as c: + output = self.run_python("print('will not be run')", + capture_output=True, stdout=tf) + self.assertIn('stdout', c.exception.args[0]) + self.assertIn('capture_output', c.exception.args[0]) + + def test_stderr_with_capture_output_arg(self): + # run() refuses to accept 'stderr' with 'capture_output' + tf = tempfile.TemporaryFile() + self.addCleanup(tf.close) + with self.assertRaises(ValueError, + msg=("Expected ValueError when stderr and capture_output " + "args supplied.")) as c: + output = self.run_python("print('will not be run')", + capture_output=True, stderr=tf) + self.assertIn('stderr', c.exception.args[0]) + self.assertIn('capture_output', c.exception.args[0]) + + +@unittest.skipIf(mswindows, "POSIX specific tests") +class POSIXProcessTestCase(BaseTestCase): + + def setUp(self): + super().setUp() + self._nonexistent_dir = "/_this/pa.th/does/not/exist" + + def _get_chdir_exception(self): + try: + os.chdir(self._nonexistent_dir) + except OSError as e: + # This avoids hard coding the errno value or the OS perror() + # string and instead capture the exception that we want to see + # below for comparison. + desired_exception = e + desired_exception.strerror += ': ' + repr(self._nonexistent_dir) + else: + self.fail("chdir to nonexistent directory %s succeeded." % + self._nonexistent_dir) + return desired_exception + + def test_exception_cwd(self): + """Test error in the child raised in the parent for a bad cwd.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + cwd=self._nonexistent_dir) + except OSError as e: + # Test that the child process chdir failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_executable(self): + """Test error in the child raised in the parent for a bad executable.""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([sys.executable, "-c", ""], + executable=self._nonexistent_dir) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + def test_exception_bad_args_0(self): + """Test error in the child raised in the parent for a bad args[0].""" + desired_exception = self._get_chdir_exception() + try: + p = subprocess.Popen([self._nonexistent_dir, "-c", ""]) + except OSError as e: + # Test that the child process exec failure actually makes + # it up to the parent process as the correct exception. + self.assertEqual(desired_exception.errno, e.errno) + self.assertEqual(desired_exception.strerror, e.strerror) + else: + self.fail("Expected OSError: %s" % desired_exception) + + # We mock the __del__ method for Popen in the next two tests + # because it does cleanup based on the pid returned by fork_exec + # along with issuing a resource warning if it still exists. Since + # we don't actually spawn a process in these tests we can forego + # the destructor. An alternative would be to set _child_created to + # False before the destructor is called but there is no easy way + # to do that + class PopenNoDestructor(subprocess.Popen): + def __del__(self): + pass + + @mock.patch("subprocess._posixsubprocess.fork_exec") + def test_exception_errpipe_normal(self, fork_exec): + """Test error passing done through errpipe_write in the good case""" + def proper_error(*args): + errpipe_write = args[13] + # Write the hex for the error code EISDIR: 'is a directory' + err_code = '{:x}'.format(errno.EISDIR).encode() + os.write(errpipe_write, b"OSError:" + err_code + b":") + return 0 + + fork_exec.side_effect = proper_error + + with mock.patch("subprocess.os.waitpid", + side_effect=ChildProcessError): + with self.assertRaises(IsADirectoryError): + self.PopenNoDestructor(["non_existent_command"]) + + @mock.patch("subprocess._posixsubprocess.fork_exec") + def test_exception_errpipe_bad_data(self, fork_exec): + """Test error passing done through errpipe_write where its not + in the expected format""" + error_data = b"\xFF\x00\xDE\xAD" + def bad_error(*args): + errpipe_write = args[13] + # Anything can be in the pipe, no assumptions should + # be made about its encoding, so we'll write some + # arbitrary hex bytes to test it out + os.write(errpipe_write, error_data) + return 0 + + fork_exec.side_effect = bad_error + + with mock.patch("subprocess.os.waitpid", + side_effect=ChildProcessError): + with self.assertRaises(subprocess.SubprocessError) as e: + self.PopenNoDestructor(["non_existent_command"]) + + self.assertIn(repr(error_data), str(e.exception)) + + @unittest.skipIf(not os.path.exists('/proc/self/status'), + "need /proc/self/status") + def test_restore_signals(self): + # Blindly assume that cat exists on systems with /proc/self/status... + default_proc_status = subprocess.check_output( + ['cat', '/proc/self/status'], + restore_signals=False) + for line in default_proc_status.splitlines(): + if line.startswith(b'SigIgn'): + default_sig_ign_mask = line + break + else: + self.skipTest("SigIgn not found in /proc/self/status.") + restored_proc_status = subprocess.check_output( + ['cat', '/proc/self/status'], + restore_signals=True) + for line in restored_proc_status.splitlines(): + if line.startswith(b'SigIgn'): + restored_sig_ign_mask = line + break + self.assertNotEqual(default_sig_ign_mask, restored_sig_ign_mask, + msg="restore_signals=True should've unblocked " + "SIGPIPE and friends.") + + def test_start_new_session(self): + # For code coverage of calling setsid(). We don't care if we get an + # EPERM error from it depending on the test execution environment, that + # still indicates that it was called. + try: + output = subprocess.check_output( + [sys.executable, "-c", + "import os; print(os.getpgid(os.getpid()))"], + start_new_session=True) + except OSError as e: + if e.errno != errno.EPERM: + raise + else: + parent_pgid = os.getpgid(os.getpid()) + child_pgid = int(output) + self.assertNotEqual(parent_pgid, child_pgid) + + def test_run_abort(self): + # returncode handles signal termination + with support.SuppressCrashReport(): + p = subprocess.Popen([sys.executable, "-c", + 'import os; os.abort()']) + p.wait() + self.assertEqual(-p.returncode, signal.SIGABRT) + + def test_CalledProcessError_str_signal(self): + err = subprocess.CalledProcessError(-int(signal.SIGABRT), "fake cmd") + error_string = str(err) + # We're relying on the repr() of the signal.Signals intenum to provide + # the word signal, the signal name and the numeric value. + self.assertIn("signal", error_string.lower()) + # We're not being specific about the signal name as some signals have + # multiple names and which name is revealed can vary. + self.assertIn("SIG", error_string) + self.assertIn(str(signal.SIGABRT), error_string) + + def test_CalledProcessError_str_unknown_signal(self): + err = subprocess.CalledProcessError(-9876543, "fake cmd") + error_string = str(err) + self.assertIn("unknown signal 9876543.", error_string) + + def test_CalledProcessError_str_non_zero(self): + err = subprocess.CalledProcessError(2, "fake cmd") + error_string = str(err) + self.assertIn("non-zero exit status 2.", error_string) + + def test_preexec(self): + # DISCLAIMER: Setting environment variables is *not* a good use + # of a preexec_fn. This is merely a test. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + 'sys.stdout.write(os.getenv("FRUIT"))'], + stdout=subprocess.PIPE, + preexec_fn=lambda: os.putenv("FRUIT", "apple")) + with p: + self.assertEqual(p.stdout.read(), b"apple") + + def test_preexec_exception(self): + def raise_it(): + raise ValueError("What if two swallows carried a coconut?") + try: + p = subprocess.Popen([sys.executable, "-c", ""], + preexec_fn=raise_it) + except subprocess.SubprocessError as e: + self.assertTrue( + subprocess._posixsubprocess, + "Expected a ValueError from the preexec_fn") + except ValueError as e: + self.assertIn("coconut", e.args[0]) + else: + self.fail("Exception raised by preexec_fn did not make it " + "to the parent process.") + + class _TestExecuteChildPopen(subprocess.Popen): + """Used to test behavior at the end of _execute_child.""" + def __init__(self, testcase, *args, **kwargs): + self._testcase = testcase + subprocess.Popen.__init__(self, *args, **kwargs) + + def _execute_child(self, *args, **kwargs): + try: + subprocess.Popen._execute_child(self, *args, **kwargs) + finally: + # Open a bunch of file descriptors and verify that + # none of them are the same as the ones the Popen + # instance is using for stdin/stdout/stderr. + devzero_fds = [os.open("/dev/zero", os.O_RDONLY) + for _ in range(8)] + try: + for fd in devzero_fds: + self._testcase.assertNotIn( + fd, (self.stdin.fileno(), self.stdout.fileno(), + self.stderr.fileno()), + msg="At least one fd was closed early.") + finally: + for fd in devzero_fds: + os.close(fd) + + @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") + def test_preexec_errpipe_does_not_double_close_pipes(self): + """Issue16140: Don't double close pipes on preexec error.""" + + def raise_it(): + raise subprocess.SubprocessError( + "force the _execute_child() errpipe_data path.") + + with self.assertRaises(subprocess.SubprocessError): + self._TestExecuteChildPopen( + self, [sys.executable, "-c", "pass"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, preexec_fn=raise_it) + + def test_preexec_gc_module_failure(self): + # This tests the code that disables garbage collection if the child + # process will execute any Python. + def raise_runtime_error(): + raise RuntimeError("this shouldn't escape") + enabled = gc.isenabled() + orig_gc_disable = gc.disable + orig_gc_isenabled = gc.isenabled + try: + gc.disable() + self.assertFalse(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertFalse(gc.isenabled(), + "Popen enabled gc when it shouldn't.") + + gc.enable() + self.assertTrue(gc.isenabled()) + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + self.assertTrue(gc.isenabled(), "Popen left gc disabled.") + + gc.disable = raise_runtime_error + self.assertRaises(RuntimeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + + del gc.isenabled # force an AttributeError + self.assertRaises(AttributeError, subprocess.Popen, + [sys.executable, '-c', ''], + preexec_fn=lambda: None) + finally: + gc.disable = orig_gc_disable + gc.isenabled = orig_gc_isenabled + if not enabled: + gc.disable() + + @unittest.skipIf( + sys.platform == 'darwin', 'setrlimit() seems to fail on OS X') + def test_preexec_fork_failure(self): + # The internal code did not preserve the previous exception when + # re-enabling garbage collection + try: + from resource import getrlimit, setrlimit, RLIMIT_NPROC + except ImportError as err: + self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD + limits = getrlimit(RLIMIT_NPROC) + [_, hard] = limits + setrlimit(RLIMIT_NPROC, (0, hard)) + self.addCleanup(setrlimit, RLIMIT_NPROC, limits) + try: + subprocess.call([sys.executable, '-c', ''], + preexec_fn=lambda: None) + except BlockingIOError: + # Forking should raise EAGAIN, translated to BlockingIOError + pass + else: + self.skipTest('RLIMIT_NPROC had no effect; probably superuser') + + def test_args_string(self): + # args is a string + fd, fname = tempfile.mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!%s\n" % support.unix_shell) + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + p = subprocess.Popen(fname) + p.wait() + os.remove(fname) + self.assertEqual(p.returncode, 47) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + startupinfo=47) + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + creationflags=47) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen(["echo $FRUIT"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + with p: + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "apple" + p = subprocess.Popen("echo $FRUIT", shell=1, + stdout=subprocess.PIPE, + env=newenv) + with p: + self.assertEqual(p.stdout.read().strip(b" \t\r\n\f"), b"apple") + + def test_call_string(self): + # call() function with string argument on UNIX + fd, fname = tempfile.mkstemp() + # reopen in text mode + with open(fd, "w", errors="surrogateescape") as fobj: + fobj.write("#!%s\n" % support.unix_shell) + fobj.write("exec '%s' -c 'import sys; sys.exit(47)'\n" % + sys.executable) + os.chmod(fname, 0o700) + rc = subprocess.call(fname) + os.remove(fname) + self.assertEqual(rc, 47) + + def test_specific_shell(self): + # Issue #9265: Incorrect name passed as arg[0]. + shells = [] + for prefix in ['/bin', '/usr/bin/', '/usr/local/bin']: + for name in ['bash', 'ksh']: + sh = os.path.join(prefix, name) + if os.path.isfile(sh): + shells.append(sh) + if not shells: # Will probably work for any shell but csh. + self.skipTest("bash or ksh required for this test") + sh = '/bin/sh' + if os.path.isfile(sh) and not os.path.islink(sh): + # Test will fail if /bin/sh is a symlink to csh. + shells.append(sh) + for sh in shells: + p = subprocess.Popen("echo $0", executable=sh, shell=True, + stdout=subprocess.PIPE) + with p: + self.assertEqual(p.stdout.read().strip(), bytes(sh, 'ascii')) + + def _kill_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + # Also set the SIGINT handler to the default to make sure it's not + # being ignored (some tests rely on that.) + old_handler = signal.signal(signal.SIGINT, signal.default_int_handler) + try: + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + signal.signal(signal.SIGINT, old_handler) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + return p + + @unittest.skipIf(sys.platform.startswith(('netbsd', 'openbsd')), + "Due to known OS bug (issue #16762)") + def _kill_dead_process(self, method, *args): + # Do not inherit file handles from the parent. + # It should fix failures on some platforms. + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + """], + close_fds=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + p.communicate() + + def test_send_signal(self): + p = self._kill_process('send_signal', signal.SIGINT) + _, stderr = p.communicate() + self.assertIn(b'KeyboardInterrupt', stderr) + self.assertNotEqual(p.wait(), 0) + + def test_kill(self): + p = self._kill_process('kill') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGKILL) + + def test_terminate(self): + p = self._kill_process('terminate') + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + self.assertEqual(p.wait(), -signal.SIGTERM) + + def test_send_signal_dead(self): + # Sending a signal to a dead process + self._kill_dead_process('send_signal', signal.SIGINT) + + def test_kill_dead(self): + # Killing a dead process + self._kill_dead_process('kill') + + def test_terminate_dead(self): + # Terminating a dead process + self._kill_dead_process('terminate') + + def _save_fds(self, save_fds): + fds = [] + for fd in save_fds: + inheritable = os.get_inheritable(fd) + saved = os.dup(fd) + fds.append((fd, saved, inheritable)) + return fds + + def _restore_fds(self, fds): + for fd, saved, inheritable in fds: + os.dup2(saved, fd, inheritable=inheritable) + os.close(saved) + + def check_close_std_fds(self, fds): + # Issue #9905: test that subprocess pipes still work properly with + # some standard fds closed + stdin = 0 + saved_fds = self._save_fds(fds) + for fd, saved, inheritable in saved_fds: + if fd == 0: + stdin = saved + break + try: + for fd in fds: + os.close(fd) + out, err = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple");' + 'sys.stdout.flush();' + 'sys.stderr.write("orange")'], + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + err = support.strip_python_stderr(err) + self.assertEqual((out, err), (b'apple', b'orange')) + finally: + self._restore_fds(saved_fds) + + def test_close_fd_0(self): + self.check_close_std_fds([0]) + + def test_close_fd_1(self): + self.check_close_std_fds([1]) + + def test_close_fd_2(self): + self.check_close_std_fds([2]) + + def test_close_fds_0_1(self): + self.check_close_std_fds([0, 1]) + + def test_close_fds_0_2(self): + self.check_close_std_fds([0, 2]) + + def test_close_fds_1_2(self): + self.check_close_std_fds([1, 2]) + + def test_close_fds_0_1_2(self): + # Issue #10806: test that subprocess pipes still work properly with + # all standard fds closed. + self.check_close_std_fds([0, 1, 2]) + + def test_small_errpipe_write_fd(self): + """Issue #15798: Popen should work when stdio fds are available.""" + new_stdin = os.dup(0) + new_stdout = os.dup(1) + try: + os.close(0) + os.close(1) + + # Side test: if errpipe_write fails to have its CLOEXEC + # flag set this should cause the parent to think the exec + # failed. Extremely unlikely: everyone supports CLOEXEC. + subprocess.Popen([ + sys.executable, "-c", + "print('AssertionError:0:CLOEXEC failure.')"]).wait() + finally: + # Restore original stdin and stdout + os.dup2(new_stdin, 0) + os.dup2(new_stdout, 1) + os.close(new_stdin) + os.close(new_stdout) + + def test_remapping_std_fds(self): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + try: + temp_fds = [fd for fd, fname in temps] + + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # write some data to what will become stdin, and rewind + os.write(temp_fds[1], b"STDIN") + os.lseek(temp_fds[1], 0, 0) + + # move the standard file descriptors out of the way + saved_fds = self._save_fds(range(3)) + try: + # duplicate the file objects over the standard fd's + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # now use those files in the "wrong" order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=temp_fds[1], + stdout=temp_fds[2], + stderr=temp_fds[0]) + p.wait() + finally: + self._restore_fds(saved_fds) + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(temp_fds[2], 1024) + err = support.strip_python_stderr(os.read(temp_fds[0], 1024)) + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + def check_swap_fds(self, stdin_no, stdout_no, stderr_no): + # open up some temporary files + temps = [tempfile.mkstemp() for i in range(3)] + temp_fds = [fd for fd, fname in temps] + try: + # unlink the files -- we won't need to reopen them + for fd, fname in temps: + os.unlink(fname) + + # save a copy of the standard file descriptors + saved_fds = self._save_fds(range(3)) + try: + # duplicate the temp files over the standard fd's 0, 1, 2 + for fd, temp_fd in enumerate(temp_fds): + os.dup2(temp_fd, fd) + + # write some data to what will become stdin, and rewind + os.write(stdin_no, b"STDIN") + os.lseek(stdin_no, 0, 0) + + # now use those files in the given order, so that subprocess + # has to rearrange them in the child + p = subprocess.Popen([sys.executable, "-c", + 'import sys; got = sys.stdin.read();' + 'sys.stdout.write("got %s"%got); sys.stderr.write("err")'], + stdin=stdin_no, + stdout=stdout_no, + stderr=stderr_no) + p.wait() + + for fd in temp_fds: + os.lseek(fd, 0, 0) + + out = os.read(stdout_no, 1024) + err = support.strip_python_stderr(os.read(stderr_no, 1024)) + finally: + self._restore_fds(saved_fds) + + self.assertEqual(out, b"got STDIN") + self.assertEqual(err, b"err") + + finally: + for fd in temp_fds: + os.close(fd) + + # When duping fds, if there arises a situation where one of the fds is + # either 0, 1 or 2, it is possible that it is overwritten (#12607). + # This tests all combinations of this. + def test_swap_fds(self): + self.check_swap_fds(0, 1, 2) + self.check_swap_fds(0, 2, 1) + self.check_swap_fds(1, 0, 2) + self.check_swap_fds(1, 2, 0) + self.check_swap_fds(2, 0, 1) + self.check_swap_fds(2, 1, 0) + + def _check_swap_std_fds_with_one_closed(self, from_fds, to_fds): + saved_fds = self._save_fds(range(3)) + try: + for from_fd in from_fds: + with tempfile.TemporaryFile() as f: + os.dup2(f.fileno(), from_fd) + + fd_to_close = (set(range(3)) - set(from_fds)).pop() + os.close(fd_to_close) + + arg_names = ['stdin', 'stdout', 'stderr'] + kwargs = {} + for from_fd, to_fd in zip(from_fds, to_fds): + kwargs[arg_names[to_fd]] = from_fd + + code = textwrap.dedent(r''' + import os, sys + skipped_fd = int(sys.argv[1]) + for fd in range(3): + if fd != skipped_fd: + os.write(fd, str(fd).encode('ascii')) + ''') + + skipped_fd = (set(range(3)) - set(to_fds)).pop() + + rc = subprocess.call([sys.executable, '-c', code, str(skipped_fd)], + **kwargs) + self.assertEqual(rc, 0) + + for from_fd, to_fd in zip(from_fds, to_fds): + os.lseek(from_fd, 0, os.SEEK_SET) + read_bytes = os.read(from_fd, 1024) + read_fds = list(map(int, read_bytes.decode('ascii'))) + msg = textwrap.dedent(f""" + When testing {from_fds} to {to_fds} redirection, + parent descriptor {from_fd} got redirected + to descriptor(s) {read_fds} instead of descriptor {to_fd}. + """) + self.assertEqual([to_fd], read_fds, msg) + finally: + self._restore_fds(saved_fds) + + # Check that subprocess can remap std fds correctly even + # if one of them is closed (#32844). + def test_swap_std_fds_with_one_closed(self): + for from_fds in itertools.combinations(range(3), 2): + for to_fds in itertools.permutations(range(3), 2): + self._check_swap_std_fds_with_one_closed(from_fds, to_fds) + + def test_surrogates_error_message(self): + def prepare(): + raise ValueError("surrogate:\uDCff") + + try: + subprocess.call( + [sys.executable, "-c", "pass"], + preexec_fn=prepare) + except ValueError as err: + # Pure Python implementations keeps the message + self.assertIsNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "surrogate:\uDCff") + except subprocess.SubprocessError as err: + # _posixsubprocess uses a default message + self.assertIsNotNone(subprocess._posixsubprocess) + self.assertEqual(str(err), "Exception occurred in preexec_fn.") + else: + self.fail("Expected ValueError or subprocess.SubprocessError") + + def test_undecodable_env(self): + for key, value in (('test', 'abc\uDCFF'), ('test\uDCFF', '42')): + encoded_value = value.encode("ascii", "surrogateescape") + + # test str with surrogates + script = "import os; print(ascii(os.getenv(%s)))" % repr(key) + env = os.environ.copy() + env[key] = value + # Use C locale to get ASCII for the locale encoding to force + # surrogate-escaping of \xFF in the child process; otherwise it can + # be decoded as-is if the default locale is latin-1. + env['LC_ALL'] = 'C' + if sys.platform.startswith("aix"): + # On AIX, the C locale uses the Latin1 encoding + decoded_value = encoded_value.decode("latin1", "surrogateescape") + else: + # On other UNIXes, the C locale uses the ASCII encoding + decoded_value = value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(decoded_value)) + + # test bytes + key = key.encode("ascii", "surrogateescape") + script = "import os; print(ascii(os.getenvb(%s)))" % repr(key) + env = os.environ.copy() + env[key] = encoded_value + stdout = subprocess.check_output( + [sys.executable, "-c", script], + env=env) + stdout = stdout.rstrip(b'\n\r') + self.assertEqual(stdout.decode('ascii'), ascii(encoded_value)) + + def test_bytes_program(self): + abs_program = os.fsencode(sys.executable) + path, program = os.path.split(sys.executable) + program = os.fsencode(program) + + # absolute bytes path + exitcode = subprocess.call([abs_program, "-c", "pass"]) + self.assertEqual(exitcode, 0) + + # absolute bytes path as a string + cmd = b"'" + abs_program + b"' -c pass" + exitcode = subprocess.call(cmd, shell=True) + self.assertEqual(exitcode, 0) + + # bytes program, unicode PATH + env = os.environ.copy() + env["PATH"] = path + exitcode = subprocess.call([program, "-c", "pass"], env=env) + self.assertEqual(exitcode, 0) + + # bytes program, bytes PATH + envb = os.environb.copy() + envb[b"PATH"] = os.fsencode(path) + exitcode = subprocess.call([program, "-c", "pass"], env=envb) + self.assertEqual(exitcode, 0) + + def test_pipe_cloexec(self): + sleeper = support.findfile("input_reader.py", subdir="subprocessdata") + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + p1 = subprocess.Popen([sys.executable, sleeper], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=False) + + self.addCleanup(p1.communicate, b'') + + p2 = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + + output, error = p2.communicate() + result_fds = set(map(int, output.split(b','))) + unwanted_fds = set([p1.stdin.fileno(), p1.stdout.fileno(), + p1.stderr.fileno()]) + + self.assertFalse(result_fds & unwanted_fds, + "Expected no fds from %r to be open in child, " + "found %r" % + (unwanted_fds, result_fds & unwanted_fds)) + + def test_pipe_cloexec_real_tools(self): + qcat = support.findfile("qcat.py", subdir="subprocessdata") + qgrep = support.findfile("qgrep.py", subdir="subprocessdata") + + subdata = b'zxcvbn' + data = subdata * 4 + b'\n' + + p1 = subprocess.Popen([sys.executable, qcat], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + close_fds=False) + + p2 = subprocess.Popen([sys.executable, qgrep, subdata], + stdin=p1.stdout, stdout=subprocess.PIPE, + close_fds=False) + + self.addCleanup(p1.wait) + self.addCleanup(p2.wait) + def kill_p1(): + try: + p1.terminate() + except ProcessLookupError: + pass + def kill_p2(): + try: + p2.terminate() + except ProcessLookupError: + pass + self.addCleanup(kill_p1) + self.addCleanup(kill_p2) + + p1.stdin.write(data) + p1.stdin.close() + + readfiles, ignored1, ignored2 = select.select([p2.stdout], [], [], 10) + + self.assertTrue(readfiles, "The child hung") + self.assertEqual(p2.stdout.read(), data) + + p1.stdout.close() + p2.stdout.close() + + def test_close_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + + open_fds = set(fds) + # add a bunch more fds + for _ in range(9): + fd = os.open(os.devnull, os.O_RDONLY) + self.addCleanup(os.close, fd) + open_fds.add(fd) + + for fd in open_fds: + os.set_inheritable(fd, True) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=False) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertEqual(remaining_fds & open_fds, open_fds, + "Some fds were closed") + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse(remaining_fds & open_fds, + "Some fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + # Keep some of the fd's we opened open in the subprocess. + # This tests _posixsubprocess.c's proper handling of fds_to_keep. + fds_to_keep = set(open_fds.pop() for _ in range(8)) + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=fds_to_keep) + output, ignored = p.communicate() + remaining_fds = set(map(int, output.split(b','))) + + self.assertFalse((remaining_fds - fds_to_keep) & open_fds, + "Some fds not in pass_fds were left open") + self.assertIn(1, remaining_fds, "Subprocess failed") + + + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD.") + def test_close_fds_when_max_fd_is_lowered(self): + """Confirm that issue21618 is fixed (may fail under valgrind).""" + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # This launches the meat of the test in a child process to + # avoid messing with the larger unittest processes maximum + # number of file descriptors. + # This process launches: + # +--> Process that lowers its RLIMIT_NOFILE aftr setting up + # a bunch of high open fds above the new lower rlimit. + # Those are reported via stdout before launching a new + # process with close_fds=False to run the actual test: + # +--> The TEST: This one launches a fd_status.py + # subprocess with close_fds=True so we can find out if + # any of the fds above the lowered rlimit are still open. + p = subprocess.Popen([sys.executable, '-c', textwrap.dedent( + ''' + import os, resource, subprocess, sys, textwrap + open_fds = set() + # Add a bunch more fds to pass down. + for _ in range(40): + fd = os.open(os.devnull, os.O_RDONLY) + open_fds.add(fd) + + # Leave a two pairs of low ones available for use by the + # internal child error pipe and the stdout pipe. + # We also leave 10 more open as some Python buildbots run into + # "too many open files" errors during the test if we do not. + for fd in sorted(open_fds)[:14]: + os.close(fd) + open_fds.remove(fd) + + for fd in open_fds: + #self.addCleanup(os.close, fd) + os.set_inheritable(fd, True) + + max_fd_open = max(open_fds) + + # Communicate the open_fds to the parent unittest.TestCase process. + print(','.join(map(str, sorted(open_fds)))) + sys.stdout.flush() + + rlim_cur, rlim_max = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + # 29 is lower than the highest fds we are leaving open. + resource.setrlimit(resource.RLIMIT_NOFILE, (29, rlim_max)) + # Launch a new Python interpreter with our low fd rlim_cur that + # inherits open fds above that limit. It then uses subprocess + # with close_fds=True to get a report of open fds in the child. + # An explicit list of fds to check is passed to fd_status.py as + # letting fd_status rely on its default logic would miss the + # fds above rlim_cur as it normally only checks up to that limit. + subprocess.Popen( + [sys.executable, '-c', + textwrap.dedent(""" + import subprocess, sys + subprocess.Popen([sys.executable, %r] + + [str(x) for x in range({max_fd})], + close_fds=True).wait() + """.format(max_fd=max_fd_open+1))], + close_fds=False).wait() + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, (rlim_cur, rlim_max)) + ''' % fd_status)], stdout=subprocess.PIPE) + + output, unused_stderr = p.communicate() + output_lines = output.splitlines() + self.assertEqual(len(output_lines), 2, + msg="expected exactly two lines of output:\n%r" % output) + opened_fds = set(map(int, output_lines[0].strip().split(b','))) + remaining_fds = set(map(int, output_lines[1].strip().split(b','))) + + self.assertFalse(remaining_fds & opened_fds, + msg="Some fds were left open.") + + + # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file + # descriptor of a pipe closed in the parent process is valid in the + # child process according to fstat(), but the mode of the file + # descriptor is invalid, and read or write raise an error. + @support.requires_mac_ver(10, 5) + def test_pass_fds(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + open_fds = set() + + for x in range(5): + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + os.set_inheritable(fds[0], True) + os.set_inheritable(fds[1], True) + open_fds.update(fds) + + for fd in open_fds: + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + pass_fds=(fd, )) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + to_be_closed = open_fds - {fd} + + self.assertIn(fd, remaining_fds, "fd to be passed not passed") + self.assertFalse(remaining_fds & to_be_closed, + "fd to be closed passed") + + # pass_fds overrides close_fds with a warning. + with self.assertWarns(RuntimeWarning) as context: + self.assertFalse(subprocess.call( + [sys.executable, "-c", "import sys; sys.exit(0)"], + close_fds=False, pass_fds=(fd, ))) + self.assertIn('overriding close_fds', str(context.warning)) + + def test_pass_fds_inheritable(self): + script = support.findfile("fd_status.py", subdir="subprocessdata") + + inheritable, non_inheritable = os.pipe() + self.addCleanup(os.close, inheritable) + self.addCleanup(os.close, non_inheritable) + os.set_inheritable(inheritable, True) + os.set_inheritable(non_inheritable, False) + pass_fds = (inheritable, non_inheritable) + args = [sys.executable, script] + args += list(map(str, pass_fds)) + + p = subprocess.Popen(args, + stdout=subprocess.PIPE, close_fds=True, + pass_fds=pass_fds) + output, ignored = p.communicate() + fds = set(map(int, output.split(b','))) + + # the inheritable file descriptor must be inherited, so its inheritable + # flag must be set in the child process after fork() and before exec() + self.assertEqual(fds, set(pass_fds), "output=%a" % output) + + # inheritable flag must not be changed in the parent process + self.assertEqual(os.get_inheritable(inheritable), True) + self.assertEqual(os.get_inheritable(non_inheritable), False) + + def test_stdout_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stdin=inout) + p.wait() + + def test_stdout_stderr_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stdout=inout, stderr=inout) + p.wait() + + def test_stderr_stdin_are_single_inout_fd(self): + with io.open(os.devnull, "r+") as inout: + p = subprocess.Popen([sys.executable, "-c", "import sys; sys.exit(0)"], + stderr=inout, stdin=inout) + p.wait() + + def test_wait_when_sigchild_ignored(self): + # NOTE: sigchild_ignore.py may not be an effective test on all OSes. + sigchild_ignore = support.findfile("sigchild_ignore.py", + subdir="subprocessdata") + p = subprocess.Popen([sys.executable, sigchild_ignore], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" + " non-zero with this error:\n%s" % + stderr.decode('utf-8')) + + def test_select_unbuffered(self): + # Issue #11459: bufsize=0 should really set the pipes as + # unbuffered (and therefore let select() work properly). + select = support.import_module("select") + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdout.write("apple")'], + stdout=subprocess.PIPE, + bufsize=0) + f = p.stdout + self.addCleanup(f.close) + try: + self.assertEqual(f.read(4), b"appl") + self.assertIn(f, select.select([f], [], [], 0.0)[0]) + finally: + p.wait() + + def test_zombie_fast_process_del(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, it wouldn't be added to subprocess._active, and would + # remain a zombie. + # spawn a Popen, and delete its reference before it exits + p = subprocess.Popen([sys.executable, "-c", + 'import sys, time;' + 'time.sleep(0.2)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + with support.check_warnings(('', ResourceWarning)): + p = None + + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + def test_leak_fast_process_del_killed(self): + # Issue #12650: on Unix, if Popen.__del__() was called before the + # process exited, and the process got killed by a signal, it would never + # be removed from subprocess._active, which triggered a FD and memory + # leak. + # spawn a Popen, delete its reference and kill it + p = subprocess.Popen([sys.executable, "-c", + 'import time;' + 'time.sleep(3)'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + ident = id(p) + pid = p.pid + with support.check_warnings(('', ResourceWarning)): + p = None + + os.kill(pid, signal.SIGKILL) + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) + + # let some time for the process to exit, and create a new Popen: this + # should trigger the wait() of p + time.sleep(0.2) + with self.assertRaises(OSError): + with subprocess.Popen(NONEXISTING_CMD, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + # p should have been wait()ed on, and removed from the _active list + self.assertRaises(OSError, os.waitpid, pid, 0) + self.assertNotIn(ident, [id(o) for o in subprocess._active]) + + def test_close_fds_after_preexec(self): + fd_status = support.findfile("fd_status.py", subdir="subprocessdata") + + # this FD is used as dup2() target by preexec_fn, and should be closed + # in the child process + fd = os.dup(1) + self.addCleanup(os.close, fd) + + p = subprocess.Popen([sys.executable, fd_status], + stdout=subprocess.PIPE, close_fds=True, + preexec_fn=lambda: os.dup2(1, fd)) + output, ignored = p.communicate() + + remaining_fds = set(map(int, output.split(b','))) + + self.assertNotIn(fd, remaining_fds) + + @support.cpython_only + def test_fork_exec(self): + # Issue #22290: fork_exec() must not crash on memory allocation failure + # or other errors + import _posixsubprocess + gc_enabled = gc.isenabled() + try: + # Use a preexec function and enable the garbage collector + # to force fork_exec() to re-enable the garbage collector + # on error. + func = lambda: None + gc.enable() + + for args, exe_list, cwd, env_list in ( + (123, [b"exe"], None, [b"env"]), + ([b"arg"], 123, None, [b"env"]), + ([b"arg"], [b"exe"], 123, [b"env"]), + ([b"arg"], [b"exe"], None, 123), + ): + with self.assertRaises(TypeError): + _posixsubprocess.fork_exec( + args, exe_list, + True, (), cwd, env_list, + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, func) + finally: + if not gc_enabled: + gc.disable() + + @support.cpython_only + def test_fork_exec_sorted_fd_sanity_check(self): + # Issue #23564: sanity check the fork_exec() fds_to_keep sanity check. + import _posixsubprocess + class BadInt: + first = True + def __init__(self, value): + self.value = value + def __int__(self): + if self.first: + self.first = False + return self.value + raise ValueError + + gc_enabled = gc.isenabled() + try: + gc.enable() + + for fds_to_keep in ( + (-1, 2, 3, 4, 5), # Negative number. + ('str', 4), # Not an int. + (18, 23, 42, 2**63), # Out of range. + (5, 4), # Not sorted. + (6, 7, 7, 8), # Duplicate. + (BadInt(1), BadInt(2)), + ): + with self.assertRaises( + ValueError, + msg='fds_to_keep={}'.format(fds_to_keep)) as c: + _posixsubprocess.fork_exec( + [b"false"], [b"false"], + True, fds_to_keep, None, [b"env"], + -1, -1, -1, -1, + 1, 2, 3, 4, + True, True, None) + self.assertIn('fds_to_keep', str(c.exception)) + finally: + if not gc_enabled: + gc.disable() + + def test_communicate_BrokenPipeError_stdin_close(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + proc.communicate() # Should swallow BrokenPipeError from close. + mock_proc_stdin.close.assert_called_with() + + def test_communicate_BrokenPipeError_stdin_write(self): + # By not setting stdout or stderr or a timeout we force the fast path + # that just calls _stdin_write() internally due to our mock. + proc = subprocess.Popen([sys.executable, '-c', 'pass']) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.write.side_effect = BrokenPipeError + proc.communicate(b'stuff') # Should swallow the BrokenPipeError. + mock_proc_stdin.write.assert_called_once_with(b'stuff') + mock_proc_stdin.close.assert_called_once_with() + + def test_communicate_BrokenPipeError_stdin_flush(self): + # Setting stdin and stdout forces the ._communicate() code path. + # python -h exits faster than python -c pass (but spams stdout). + proc = subprocess.Popen([sys.executable, '-h'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin, \ + open(os.devnull, 'wb') as dev_null: + mock_proc_stdin.flush.side_effect = BrokenPipeError + # because _communicate registers a selector using proc.stdin... + mock_proc_stdin.fileno.return_value = dev_null.fileno() + # _communicate() should swallow BrokenPipeError from flush. + proc.communicate(b'stuff') + mock_proc_stdin.flush.assert_called_once_with() + + def test_communicate_BrokenPipeError_stdin_close_with_timeout(self): + # Setting stdin and stdout forces the ._communicate() code path. + # python -h exits faster than python -c pass (but spams stdout). + proc = subprocess.Popen([sys.executable, '-h'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + with proc, mock.patch.object(proc, 'stdin') as mock_proc_stdin: + mock_proc_stdin.close.side_effect = BrokenPipeError + # _communicate() should swallow BrokenPipeError from close. + proc.communicate(timeout=999) + mock_proc_stdin.close.assert_called_once_with() + + @unittest.skipUnless(_testcapi is not None + and hasattr(_testcapi, 'W_STOPCODE'), + 'need _testcapi.W_STOPCODE') + def test_stopped(self): + """Test wait() behavior when waitpid returns WIFSTOPPED; issue29335.""" + args = [sys.executable, '-c', 'pass'] + proc = subprocess.Popen(args) + + # Wait until the real process completes to avoid zombie process + pid = proc.pid + pid, status = os.waitpid(pid, 0) + self.assertEqual(status, 0) + + status = _testcapi.W_STOPCODE(3) + with mock.patch('subprocess.os.waitpid', return_value=(pid, status)): + returncode = proc.wait() + + self.assertEqual(returncode, -3) + + +@unittest.skipUnless(mswindows, "Windows specific tests") +class Win32ProcessTestCase(BaseTestCase): + + def test_startupinfo(self): + # startupinfo argument + # We uses hardcoded constants, because we do not want to + # depend on win32all. + STARTF_USESHOWWINDOW = 1 + SW_MAXIMIZE = 3 + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags = STARTF_USESHOWWINDOW + startupinfo.wShowWindow = SW_MAXIMIZE + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_startupinfo_keywords(self): + # startupinfo argument + # We use hardcoded constants, because we do not want to + # depend on win32all. + STARTF_USERSHOWWINDOW = 1 + SW_MAXIMIZE = 3 + startupinfo = subprocess.STARTUPINFO( + dwFlags=STARTF_USERSHOWWINDOW, + wShowWindow=SW_MAXIMIZE + ) + # Since Python is a console process, it won't be affected + # by wShowWindow, but the argument should be silently + # ignored + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_creationflags(self): + # creationflags argument + CREATE_NEW_CONSOLE = 16 + sys.stderr.write(" a DOS box should flash briefly ...\n") + subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + + def test_invalid_args(self): + # invalid arguments should raise ValueError + self.assertRaises(ValueError, subprocess.call, + [sys.executable, "-c", + "import sys; sys.exit(47)"], + preexec_fn=lambda: 1) + + @support.cpython_only + def test_issue31471(self): + # There shouldn't be an assertion failure in Popen() in case the env + # argument has a bad keys() method. + class BadEnv(dict): + keys = None + with self.assertRaises(TypeError): + subprocess.Popen([sys.executable, "-c", "pass"], env=BadEnv()) + + def test_close_fds(self): + # close file descriptors + rc = subprocess.call([sys.executable, "-c", + "import sys; sys.exit(47)"], + close_fds=True) + self.assertEqual(rc, 47) + + def test_close_fds_with_stdio(self): + import msvcrt + + fds = os.pipe() + self.addCleanup(os.close, fds[0]) + self.addCleanup(os.close, fds[1]) + + handles = [] + for fd in fds: + os.set_inheritable(fd, True) + handles.append(msvcrt.get_osfhandle(fd)) + + p = subprocess.Popen([sys.executable, "-c", + "import msvcrt; print(msvcrt.open_osfhandle({}, 0))".format(handles[0])], + stdout=subprocess.PIPE, close_fds=False) + stdout, stderr = p.communicate() + self.assertEqual(p.returncode, 0) + int(stdout.strip()) # Check that stdout is an integer + + p = subprocess.Popen([sys.executable, "-c", + "import msvcrt; print(msvcrt.open_osfhandle({}, 0))".format(handles[0])], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True) + stdout, stderr = p.communicate() + self.assertEqual(p.returncode, 1) + self.assertIn(b"OSError", stderr) + + # The same as the previous call, but with an empty handle_list + handle_list = [] + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": handle_list} + p = subprocess.Popen([sys.executable, "-c", + "import msvcrt; print(msvcrt.open_osfhandle({}, 0))".format(handles[0])], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + startupinfo=startupinfo, close_fds=True) + stdout, stderr = p.communicate() + self.assertEqual(p.returncode, 1) + self.assertIn(b"OSError", stderr) + + # Check for a warning due to using handle_list and close_fds=False + with support.check_warnings((".*overriding close_fds", RuntimeWarning)): + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": handles[:]} + p = subprocess.Popen([sys.executable, "-c", + "import msvcrt; print(msvcrt.open_osfhandle({}, 0))".format(handles[0])], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + startupinfo=startupinfo, close_fds=False) + stdout, stderr = p.communicate() + self.assertEqual(p.returncode, 0) + + def test_empty_attribute_list(self): + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {} + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_empty_handle_list(self): + startupinfo = subprocess.STARTUPINFO() + startupinfo.lpAttributeList = {"handle_list": []} + subprocess.call([sys.executable, "-c", "import sys; sys.exit(0)"], + startupinfo=startupinfo) + + def test_shell_sequence(self): + # Run command through the shell (sequence) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen(["set"], shell=1, + stdout=subprocess.PIPE, + env=newenv) + with p: + self.assertIn(b"physalis", p.stdout.read()) + + def test_shell_string(self): + # Run command through the shell (string) + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv) + with p: + self.assertIn(b"physalis", p.stdout.read()) + + def test_shell_encodings(self): + # Run command through the shell (string) + for enc in ['ansi', 'oem']: + newenv = os.environ.copy() + newenv["FRUIT"] = "physalis" + p = subprocess.Popen("set", shell=1, + stdout=subprocess.PIPE, + env=newenv, + encoding=enc) + with p: + self.assertIn("physalis", p.stdout.read(), enc) + + def test_call_string(self): + # call() function with string argument on Windows + rc = subprocess.call(sys.executable + + ' -c "import sys; sys.exit(47)"') + self.assertEqual(rc, 47) + + def _kill_process(self, method, *args): + # Some win32 buildbot raises EOFError if stdin is inherited + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + time.sleep(30) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + with p: + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + returncode = p.wait() + self.assertNotEqual(returncode, 0) + + def _kill_dead_process(self, method, *args): + p = subprocess.Popen([sys.executable, "-c", """if 1: + import sys, time + sys.stdout.write('x\\n') + sys.stdout.flush() + sys.exit(42) + """], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + with p: + # Wait for the interpreter to be completely initialized before + # sending any signal. + p.stdout.read(1) + # The process should end after this + time.sleep(1) + # This shouldn't raise even though the child is now dead + getattr(p, method)(*args) + _, stderr = p.communicate() + self.assertStderrEqual(stderr, b'') + rc = p.wait() + self.assertEqual(rc, 42) + + def test_send_signal(self): + self._kill_process('send_signal', signal.SIGTERM) + + def test_kill(self): + self._kill_process('kill') + + def test_terminate(self): + self._kill_process('terminate') + + def test_send_signal_dead(self): + self._kill_dead_process('send_signal', signal.SIGTERM) + + def test_kill_dead(self): + self._kill_dead_process('kill') + + def test_terminate_dead(self): + self._kill_dead_process('terminate') + +class MiscTests(unittest.TestCase): + + class RecordingPopen(subprocess.Popen): + """A Popen that saves a reference to each instance for testing.""" + instances_created = [] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.instances_created.append(self) + + @mock.patch.object(subprocess.Popen, "_communicate") + def _test_keyboardinterrupt_no_kill(self, popener, mock__communicate, + **kwargs): + """Fake a SIGINT happening during Popen._communicate() and ._wait(). + + This avoids the need to actually try and get test environments to send + and receive signals reliably across platforms. The net effect of a ^C + happening during a blocking subprocess execution which we want to clean + up from is a KeyboardInterrupt coming out of communicate() or wait(). + """ + + mock__communicate.side_effect = KeyboardInterrupt + try: + with mock.patch.object(subprocess.Popen, "_wait") as mock__wait: + # We patch out _wait() as no signal was involved so the + # child process isn't actually going to exit rapidly. + mock__wait.side_effect = KeyboardInterrupt + with mock.patch.object(subprocess, "Popen", + self.RecordingPopen): + with self.assertRaises(KeyboardInterrupt): + popener([sys.executable, "-c", + "import time\ntime.sleep(9)\nimport sys\n" + "sys.stderr.write('\\n!runaway child!\\n')"], + stdout=subprocess.DEVNULL, **kwargs) + for call in mock__wait.call_args_list[1:]: + self.assertNotEqual( + call, mock.call(timeout=None), + "no open-ended wait() after the first allowed: " + f"{mock__wait.call_args_list}") + sigint_calls = [] + for call in mock__wait.call_args_list: + if call == mock.call(timeout=0.25): # from Popen.__init__ + sigint_calls.append(call) + self.assertLessEqual(mock__wait.call_count, 2, + msg=mock__wait.call_args_list) + self.assertEqual(len(sigint_calls), 1, + msg=mock__wait.call_args_list) + finally: + # cleanup the forgotten (due to our mocks) child process + process = self.RecordingPopen.instances_created.pop() + process.kill() + process.wait() + self.assertEqual([], self.RecordingPopen.instances_created) + + def test_call_keyboardinterrupt_no_kill(self): + self._test_keyboardinterrupt_no_kill(subprocess.call, timeout=6.282) + + def test_run_keyboardinterrupt_no_kill(self): + self._test_keyboardinterrupt_no_kill(subprocess.run, timeout=6.282) + + def test_context_manager_keyboardinterrupt_no_kill(self): + def popen_via_context_manager(*args, **kwargs): + with subprocess.Popen(*args, **kwargs) as unused_process: + raise KeyboardInterrupt # Test how __exit__ handles ^C. + self._test_keyboardinterrupt_no_kill(popen_via_context_manager) + + def test_getoutput(self): + self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') + self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), + (0, 'xyzzy')) + + # we use mkdtemp in the next line to create an empty directory + # under our exclusive control; from that, we can invent a pathname + # that we _know_ won't exist. This is guaranteed to fail. + dir = None + try: + dir = tempfile.mkdtemp() + name = os.path.join(dir, "foo") + status, output = subprocess.getstatusoutput( + ("type " if mswindows else "cat ") + name) + self.assertNotEqual(status, 0) + finally: + if dir is not None: + os.rmdir(dir) + + def test__all__(self): + """Ensure that __all__ is populated properly.""" + intentionally_excluded = {"list2cmdline", "Handle"} + exported = set(subprocess.__all__) + possible_exports = set() + import types + for name, value in subprocess.__dict__.items(): + if name.startswith('_'): + continue + if isinstance(value, (types.ModuleType,)): + continue + possible_exports.add(name) + self.assertEqual(exported, possible_exports - intentionally_excluded) + + +@unittest.skipUnless(hasattr(selectors, 'PollSelector'), + "Test needs selectors.PollSelector") +class ProcessTestCaseNoPoll(ProcessTestCase): + def setUp(self): + self.orig_selector = subprocess._PopenSelector + subprocess._PopenSelector = selectors.SelectSelector + ProcessTestCase.setUp(self) + + def tearDown(self): + subprocess._PopenSelector = self.orig_selector + ProcessTestCase.tearDown(self) + + +@unittest.skipUnless(mswindows, "Windows-specific tests") +class CommandsWithSpaces (BaseTestCase): + + def setUp(self): + super().setUp() + f, fname = tempfile.mkstemp(".py", "te st") + self.fname = fname.lower () + os.write(f, b"import sys;" + b"sys.stdout.write('%d %s' % (len(sys.argv), [a.lower () for a in sys.argv]))" + ) + os.close(f) + + def tearDown(self): + os.remove(self.fname) + super().tearDown() + + def with_spaces(self, *args, **kwargs): + kwargs['stdout'] = subprocess.PIPE + p = subprocess.Popen(*args, **kwargs) + with p: + self.assertEqual( + p.stdout.read ().decode("mbcs"), + "2 [%r, 'ab cd']" % self.fname + ) + + def test_shell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd"), shell=1) + + def test_shell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) + + def test_noshell_string_with_spaces(self): + # call() function with string argument with spaces on Windows + self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, + "ab cd")) + + def test_noshell_sequence_with_spaces(self): + # call() function with sequence argument with spaces on Windows + self.with_spaces([sys.executable, self.fname, "ab cd"]) + + +class ContextManagerTests(BaseTestCase): + + def test_pipe(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write('stdout');" + "sys.stderr.write('stderr');"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + self.assertEqual(proc.stdout.read(), b"stdout") + self.assertStderrEqual(proc.stderr.read(), b"stderr") + + self.assertTrue(proc.stdout.closed) + self.assertTrue(proc.stderr.closed) + + def test_returncode(self): + with subprocess.Popen([sys.executable, "-c", + "import sys; sys.exit(100)"]) as proc: + pass + # __exit__ calls wait(), so the returncode should be set + self.assertEqual(proc.returncode, 100) + + def test_communicate_stdin(self): + with subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.exit(sys.stdin.read() == 'context')"], + stdin=subprocess.PIPE) as proc: + proc.communicate(b"context") + self.assertEqual(proc.returncode, 1) + + def test_invalid_args(self): + with self.assertRaises(NONEXISTING_ERRORS): + with subprocess.Popen(NONEXISTING_CMD, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + pass + + def test_broken_pipe_cleanup(self): + """Broken pipe error should not prevent wait() (Issue 21619)""" + proc = subprocess.Popen([sys.executable, '-c', 'pass'], + stdin=subprocess.PIPE, + bufsize=support.PIPE_MAX_SIZE*2) + proc = proc.__enter__() + # Prepare to send enough data to overflow any OS pipe buffering and + # guarantee a broken pipe error. Data is held in BufferedWriter + # buffer until closed. + proc.stdin.write(b'x' * support.PIPE_MAX_SIZE) + self.assertIsNone(proc.returncode) + # EPIPE expected under POSIX; EINVAL under Windows + self.assertRaises(OSError, proc.__exit__, None, None, None) + self.assertEqual(proc.returncode, 0) + self.assertTrue(proc.stdin.closed) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.7/test_threading.py b/src/greentest/3.7/test_threading.py new file mode 100644 index 0000000..dd36b08 --- /dev/null +++ b/src/greentest/3.7/test_threading.py @@ -0,0 +1,1174 @@ +""" +Tests for the threading module. +""" + +import test.support +from test.support import (verbose, import_module, cpython_only, + requires_type_collecting) +from test.support.script_helper import assert_python_ok, assert_python_failure + +import random +import sys +import _thread +import threading +import time +import unittest +import weakref +import os +import subprocess + +from gevent.tests import lock_tests # gevent: use our local copy +from test import support + + +# Between fork() and exec(), only async-safe functions are allowed (issues +# #12316 and #11870), and fork() from a worker thread is known to trigger +# problems with some operating systems (issue #3863): skip problematic tests +# on platforms known to behave badly. +platforms_to_skip = ('netbsd5', 'hp-ux11') + + +# A trivial mutable counter. +class Counter(object): + def __init__(self): + self.value = 0 + def inc(self): + self.value += 1 + def dec(self): + self.value -= 1 + def get(self): + return self.value + +class TestThread(threading.Thread): + def __init__(self, name, testcase, sema, mutex, nrunning): + threading.Thread.__init__(self, name=name) + self.testcase = testcase + self.sema = sema + self.mutex = mutex + self.nrunning = nrunning + + def run(self): + delay = random.random() / 10000.0 + if verbose: + print('task %s will run for %.1f usec' % + (self.name, delay * 1e6)) + + with self.sema: + with self.mutex: + self.nrunning.inc() + if verbose: + print(self.nrunning.get(), 'tasks are running') + self.testcase.assertLessEqual(self.nrunning.get(), 3) + + time.sleep(delay) + if verbose: + print('task', self.name, 'done') + + with self.mutex: + self.nrunning.dec() + self.testcase.assertGreaterEqual(self.nrunning.get(), 0) + if verbose: + print('%s is finished. %d tasks are running' % + (self.name, self.nrunning.get())) + + +class BaseTestCase(unittest.TestCase): + def setUp(self): + self._threads = test.support.threading_setup() + + def tearDown(self): + test.support.threading_cleanup(*self._threads) + test.support.reap_children() + + +class ThreadTests(BaseTestCase): + + # Create a bunch of threads, let each do some work, wait until all are + # done. + def test_various_ops(self): + # This takes about n/3 seconds to run (about n/3 clumps of tasks, + # times about 1 second per clump). + NUMTASKS = 10 + + # no more than 3 of the 10 can run at once + sema = threading.BoundedSemaphore(value=3) + mutex = threading.RLock() + numrunning = Counter() + + threads = [] + + for i in range(NUMTASKS): + t = TestThread(""%i, self, sema, mutex, numrunning) + threads.append(t) + self.assertIsNone(t.ident) + self.assertRegex(repr(t), r'^$') + t.start() + + if verbose: + print('waiting for all tasks to complete') + for t in threads: + t.join() + self.assertFalse(t.is_alive()) + self.assertNotEqual(t.ident, 0) + self.assertIsNotNone(t.ident) + self.assertRegex(repr(t), r'^$') + if verbose: + print('all tasks done') + self.assertEqual(numrunning.get(), 0) + + def test_ident_of_no_threading_threads(self): + # The ident still must work for the main thread and dummy threads. + self.assertIsNotNone(threading.currentThread().ident) + def f(): + ident.append(threading.currentThread().ident) + done.set() + done = threading.Event() + ident = [] + with support.wait_threads_exit(): + tid = _thread.start_new_thread(f, ()) + done.wait() + self.assertEqual(ident[0], tid) + # Kill the "immortal" _DummyThread + del threading._active[ident[0]] + + # run with a small(ish) thread stack size (256 KiB) + def test_various_ops_small_stack(self): + if verbose: + print('with 256 KiB thread stack size...') + try: + threading.stack_size(262144) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1 MiB) + def test_various_ops_large_stack(self): + if verbose: + print('with 1 MiB thread stack size...') + try: + threading.stack_size(0x100000) + except _thread.error: + raise unittest.SkipTest( + 'platform does not support changing thread stack size') + self.test_various_ops() + threading.stack_size(0) + + def test_foreign_thread(self): + # Check that a "foreign" thread can use the threading module. + def f(mutex): + # Calling current_thread() forces an entry for the foreign + # thread to get made in the threading._active map. + threading.current_thread() + mutex.release() + + mutex = threading.Lock() + mutex.acquire() + with support.wait_threads_exit(): + tid = _thread.start_new_thread(f, (mutex,)) + # Wait for the thread to finish. + mutex.acquire() + self.assertIn(tid, threading._active) + self.assertIsInstance(threading._active[tid], threading._DummyThread) + #Issue 29376 + self.assertTrue(threading._active[tid].is_alive()) + self.assertRegex(repr(threading._active[tid]), '_DummyThread') + del threading._active[tid] + + # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) + # exposed at the Python level. This test relies on ctypes to get at it. + def test_PyThreadState_SetAsyncExc(self): + ctypes = import_module("ctypes") + + set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc + set_async_exc.argtypes = (ctypes.c_ulong, ctypes.py_object) + + class AsyncExc(Exception): + pass + + exception = ctypes.py_object(AsyncExc) + + # First check it works when setting the exception from the same thread. + tid = threading.get_ident() + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + try: + result = set_async_exc(tid, exception) + # The exception is async, so we might have to keep the VM busy until + # it notices. + while True: + pass + except AsyncExc: + pass + else: + # This code is unreachable but it reflects the intent. If we wanted + # to be smarter the above loop wouldn't be infinite. + self.fail("AsyncExc not raised") + try: + self.assertEqual(result, 1) # one thread state modified + except UnboundLocalError: + # The exception was raised too quickly for us to get the result. + pass + + # `worker_started` is set by the thread when it's inside a try/except + # block waiting to catch the asynchronously set AsyncExc exception. + # `worker_saw_exception` is set by the thread upon catching that + # exception. + worker_started = threading.Event() + worker_saw_exception = threading.Event() + + class Worker(threading.Thread): + def run(self): + self.id = threading.get_ident() + self.finished = False + + try: + while True: + worker_started.set() + time.sleep(0.1) + except AsyncExc: + self.finished = True + worker_saw_exception.set() + + t = Worker() + t.daemon = True # so if this fails, we don't hang Python at shutdown + t.start() + if verbose: + print(" started worker thread") + + # Try a thread id that doesn't make sense. + if verbose: + print(" trying nonsensical thread id") + result = set_async_exc(-1, exception) + self.assertEqual(result, 0) # no thread states modified + + # Now raise an exception in the worker thread. + if verbose: + print(" waiting for worker thread to get started") + ret = worker_started.wait() + self.assertTrue(ret) + if verbose: + print(" verifying worker hasn't exited") + self.assertFalse(t.finished) + if verbose: + print(" attempting to raise asynch exception in worker") + result = set_async_exc(t.id, exception) + self.assertEqual(result, 1) # one thread state modified + if verbose: + print(" waiting for worker to say it caught the exception") + worker_saw_exception.wait(timeout=10) + self.assertTrue(t.finished) + if verbose: + print(" all OK -- joining worker") + if t.finished: + t.join() + # else the thread is still running, and we have no way to kill it + + def test_limbo_cleanup(self): + # Issue 7481: Failure to start thread should cleanup the limbo map. + def fail_new_thread(*args): + raise threading.ThreadError() + _start_new_thread = threading._start_new_thread + threading._start_new_thread = fail_new_thread + try: + t = threading.Thread(target=lambda: None) + self.assertRaises(threading.ThreadError, t.start) + self.assertFalse( + t in threading._limbo, + "Failed to cleanup _limbo map on failure of Thread.start().") + finally: + threading._start_new_thread = _start_new_thread + + def test_finalize_runnning_thread(self): + # Issue 1402: the PyGILState_Ensure / _Release functions may be called + # very late on python exit: on deallocation of a running thread for + # example. + import_module("ctypes") + + rc, out, err = assert_python_failure("-c", """if 1: + import ctypes, sys, time, _thread + + # This lock is used as a simple event variable. + ready = _thread.allocate_lock() + ready.acquire() + + # Module globals are cleared before __del__ is run + # So we save the functions in class dict + class C: + ensure = ctypes.pythonapi.PyGILState_Ensure + release = ctypes.pythonapi.PyGILState_Release + def __del__(self): + state = self.ensure() + self.release(state) + + def waitingThread(): + x = C() + ready.release() + time.sleep(100) + + _thread.start_new_thread(waitingThread, ()) + ready.acquire() # Be sure the other thread is waiting. + sys.exit(42) + """) + self.assertEqual(rc, 42) + + def test_finalize_with_trace(self): + # Issue1733757 + # Avoid a deadlock when sys.settrace steps into threading._shutdown + assert_python_ok("-c", """if 1: + import sys, threading + + # A deadlock-killer, to prevent the + # testsuite to hang forever + def killer(): + import os, time + time.sleep(2) + print('program blocked; aborting') + os._exit(2) + t = threading.Thread(target=killer) + t.daemon = True + t.start() + + # This is the trace function + def func(frame, event, arg): + threading.current_thread() + return func + + sys.settrace(func) + """) + + def test_join_nondaemon_on_shutdown(self): + # Issue 1722344 + # Raising SystemExit skipped threading._shutdown + rc, out, err = assert_python_ok("-c", """if 1: + import threading + from time import sleep + + def child(): + sleep(1) + # As a non-daemon thread we SHOULD wake up and nothing + # should be torn down yet + print("Woke up, sleep function is:", sleep) + + threading.Thread(target=child).start() + raise SystemExit + """) + self.assertEqual(out.strip(), + b"Woke up, sleep function is: ") + self.assertEqual(err, b"") + + def test_enumerate_after_join(self): + # Try hard to trigger #1703448: a thread is still returned in + # threading.enumerate() after it has been join()ed. + enum = threading.enumerate + old_interval = sys.getswitchinterval() + try: + for i in range(1, 100): + sys.setswitchinterval(i * 0.0002) + t = threading.Thread(target=lambda: None) + t.start() + t.join() + l = enum() + self.assertNotIn(t, l, + "#1703448 triggered after %d trials: %s" % (i, l)) + finally: + sys.setswitchinterval(old_interval) + + def test_no_refcycle_through_target(self): + class RunSelfFunction(object): + def __init__(self, should_raise): + # The links in this refcycle from Thread back to self + # should be cleaned up when the thread completes. + self.should_raise = should_raise + self.thread = threading.Thread(target=self._run, + args=(self,), + kwargs={'yet_another':self}) + self.thread.start() + + def _run(self, other_ref, yet_another): + if self.should_raise: + raise SystemExit + + cyclic_object = RunSelfFunction(should_raise=False) + weak_cyclic_object = weakref.ref(cyclic_object) + cyclic_object.thread.join() + del cyclic_object + self.assertIsNone(weak_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_cyclic_object()))) + + raising_cyclic_object = RunSelfFunction(should_raise=True) + weak_raising_cyclic_object = weakref.ref(raising_cyclic_object) + raising_cyclic_object.thread.join() + del raising_cyclic_object + self.assertIsNone(weak_raising_cyclic_object(), + msg=('%d references still around' % + sys.getrefcount(weak_raising_cyclic_object()))) + + def test_old_threading_api(self): + # Just a quick sanity check to make sure the old method names are + # still present + t = threading.Thread() + t.isDaemon() + t.setDaemon(True) + t.getName() + t.setName("name") + t.isAlive() + e = threading.Event() + e.isSet() + threading.activeCount() + + def test_repr_daemon(self): + t = threading.Thread() + self.assertNotIn('daemon', repr(t)) + t.daemon = True + self.assertIn('daemon', repr(t)) + + def test_daemon_param(self): + t = threading.Thread() + self.assertFalse(t.daemon) + t = threading.Thread(daemon=False) + self.assertFalse(t.daemon) + t = threading.Thread(daemon=True) + self.assertTrue(t.daemon) + + @unittest.skipUnless(hasattr(os, 'fork'), 'test needs fork()') + def test_dummy_thread_after_fork(self): + # Issue #14308: a dummy thread in the active list doesn't mess up + # the after-fork mechanism. + code = """if 1: + import _thread, threading, os, time + + def background_thread(evt): + # Creates and registers the _DummyThread instance + threading.current_thread() + evt.set() + time.sleep(10) + + evt = threading.Event() + _thread.start_new_thread(background_thread, (evt,)) + evt.wait() + assert threading.active_count() == 2, threading.active_count() + if os.fork() == 0: + assert threading.active_count() == 1, threading.active_count() + os._exit(0) + else: + os.wait() + """ + _, out, err = assert_python_ok("-c", code) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + old_interval = sys.getswitchinterval() + self.addCleanup(sys.setswitchinterval, old_interval) + + # Make the bug more likely to manifest. + test.support.setswitchinterval(1e-6) + + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + pid = os.fork() + if pid == 0: + os._exit(11 if t.is_alive() else 10) + else: + t.join() + + pid, status = os.waitpid(pid, 0) + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(10, os.WEXITSTATUS(status)) + + def test_main_thread(self): + main = threading.main_thread() + self.assertEqual(main.name, 'MainThread') + self.assertEqual(main.ident, threading.current_thread().ident) + self.assertEqual(main.ident, threading.get_ident()) + + def f(): + self.assertNotEqual(threading.main_thread().ident, + threading.current_thread().ident) + th = threading.Thread(target=f) + th.start() + th.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork(self): + code = """if 1: + import os, threading + + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + else: + os.waitpid(pid, 0) + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "MainThread\nTrue\nTrue\n") + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_main_thread_after_fork_from_nonmain_thread(self): + code = """if 1: + import os, threading, sys + + def f(): + pid = os.fork() + if pid == 0: + main = threading.main_thread() + print(main.name) + print(main.ident == threading.current_thread().ident) + print(main.ident == threading.get_ident()) + # stdout is fully buffered because not a tty, + # we have to flush before exit. + sys.stdout.flush() + else: + os.waitpid(pid, 0) + + th = threading.Thread(target=f) + th.start() + th.join() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode().replace('\r', '') + self.assertEqual(err, b"") + self.assertEqual(data, "Thread-1\nTrue\nTrue\n") + + def test_main_thread_during_shutdown(self): + # bpo-31516: current_thread() should still point to the main thread + # at shutdown + code = """if 1: + import gc, threading + + main_thread = threading.current_thread() + assert main_thread is threading.main_thread() # sanity check + + class RefCycle: + def __init__(self): + self.cycle = self + + def __del__(self): + print("GC:", + threading.current_thread() is main_thread, + threading.main_thread() is main_thread, + threading.enumerate() == [main_thread]) + + RefCycle() + gc.collect() # sanity check + x = RefCycle() + """ + _, out, err = assert_python_ok("-c", code) + data = out.decode() + self.assertEqual(err, b"") + self.assertEqual(data.splitlines(), + ["GC: True True True"] * 2) + + def test_tstate_lock(self): + # Test an implementation detail of Thread objects. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + time.sleep(0.01) + # The tstate lock is None until the thread is started + t = threading.Thread(target=f) + self.assertIs(t._tstate_lock, None) + t.start() + started.acquire() + self.assertTrue(t.is_alive()) + # The tstate lock can't be acquired when the thread is running + # (or suspended). + tstate_lock = t._tstate_lock + self.assertFalse(tstate_lock.acquire(timeout=0), False) + finish.release() + # When the thread ends, the state_lock can be successfully + # acquired. + self.assertTrue(tstate_lock.acquire(timeout=5), False) + # But is_alive() is still True: we hold _tstate_lock now, which + # prevents is_alive() from knowing the thread's end-of-life C code + # is done. + self.assertTrue(t.is_alive()) + # Let is_alive() find out the C code is done. + tstate_lock.release() + self.assertFalse(t.is_alive()) + # And verify the thread disposed of _tstate_lock. + self.assertIsNone(t._tstate_lock) + t.join() + + def test_repr_stopped(self): + # Verify that "stopped" shows up in repr(Thread) appropriately. + started = _thread.allocate_lock() + finish = _thread.allocate_lock() + started.acquire() + finish.acquire() + def f(): + started.release() + finish.acquire() + t = threading.Thread(target=f) + t.start() + started.acquire() + self.assertIn("started", repr(t)) + finish.release() + # "stopped" should appear in the repr in a reasonable amount of time. + # Implementation detail: as of this writing, that's trivially true + # if .join() is called, and almost trivially true if .is_alive() is + # called. The detail we're testing here is that "stopped" shows up + # "all on its own". + LOOKING_FOR = "stopped" + for i in range(500): + if LOOKING_FOR in repr(t): + break + time.sleep(0.01) + self.assertIn(LOOKING_FOR, repr(t)) # we waited at least 5 seconds + t.join() + + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) + + @cpython_only + def test_frame_tstate_tracing(self): + # Issue #14432: Crash when a generator is created in a C thread that is + # destroyed while the generator is still used. The issue was that a + # generator contains a frame, and the frame kept a reference to the + # Python state of the destroyed C thread. The crash occurs when a trace + # function is setup. + + def noop_trace(frame, event, arg): + # no operation + return noop_trace + + def generator(): + while 1: + yield "generator" + + def callback(): + if callback.gen is None: + callback.gen = generator() + return next(callback.gen) + callback.gen = None + + old_trace = sys.gettrace() + sys.settrace(noop_trace) + try: + # Install a trace function + threading.settrace(noop_trace) + + # Create a generator in a C thread which exits after the call + import _testcapi + _testcapi.call_in_temporary_c_thread(callback) + + # Call the generator in a different Python thread, check that the + # generator didn't keep a reference to the destroyed thread state + for test in range(3): + # The trace function is still called here + callback() + finally: + sys.settrace(old_trace) + + +class ThreadJoinOnShutdown(BaseTestCase): + + def _run_and_join(self, script): + script = """if 1: + import sys, os, time, threading + + # a thread, which waits for the main program to terminate + def joiningfunc(mainthread): + mainthread.join() + print('end of thread') + # stdout is fully buffered because not a tty, we have to flush + # before exit. + sys.stdout.flush() + \n""" + script + + rc, out, err = assert_python_ok("-c", script) + data = out.decode().replace('\r', '') + self.assertEqual(data, "end of main\nend of thread\n") + + def test_1_join_on_shutdown(self): + # The usual case: on exit, wait for a non-daemon thread + script = """if 1: + import os + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + time.sleep(0.1) + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_2_join_in_forked_process(self): + # Like the test above, but from a forked interpreter + script = """if 1: + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(threading.current_thread(),)) + t.start() + print('end of main') + """ + self._run_and_join(script) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_3_join_in_forked_from_thread(self): + # Like the test above, but fork() was called from a worker thread + # In the forked process, the main Thread object must be marked as stopped. + + script = """if 1: + main_thread = threading.current_thread() + def worker(): + childpid = os.fork() + if childpid != 0: + os.waitpid(childpid, 0) + sys.exit(0) + + t = threading.Thread(target=joiningfunc, + args=(main_thread,)) + print('end of main') + t.start() + t.join() # Should not block: main_thread is already stopped + + w = threading.Thread(target=worker) + w.start() + """ + self._run_and_join(script) + + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_4_daemon_threads(self): + # Check that a daemon thread cannot crash the interpreter on shutdown + # by manipulating internal structures that are being disposed of in + # the main thread. + script = """if True: + import os + import random + import sys + import time + import threading + + thread_has_run = set() + + def random_io(): + '''Loop for a while sleeping random tiny amounts and doing some I/O.''' + while True: + in_f = open(os.__file__, 'rb') + stuff = in_f.read(200) + null_f = open(os.devnull, 'wb') + null_f.write(stuff) + time.sleep(random.random() / 1995) + null_f.close() + in_f.close() + thread_has_run.add(threading.current_thread()) + + def main(): + count = 0 + for _ in range(40): + new_thread = threading.Thread(target=random_io) + new_thread.daemon = True + new_thread.start() + count += 1 + while len(thread_has_run) < count: + time.sleep(0.001) + # Trigger process shutdown + sys.exit(0) + + main() + """ + rc, out, err = assert_python_ok('-c', script) + self.assertFalse(err) + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") + def test_reinit_tls_after_fork(self): + # Issue #13817: fork() would deadlock in a multithreaded program with + # the ad-hoc TLS implementation. + + def do_fork_and_wait(): + # just fork a child process and wait it + pid = os.fork() + if pid > 0: + os.waitpid(pid, 0) + else: + os._exit(0) + + # start a bunch of threads that will fork() child processes + threads = [] + for i in range(16): + t = threading.Thread(target=do_fork_and_wait) + threads.append(t) + t.start() + + for t in threads: + t.join() + + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_clear_threads_states_after_fork(self): + # Issue #17094: check that threads states are cleared after fork() + + # start a bunch of threads + threads = [] + for i in range(16): + t = threading.Thread(target=lambda : time.sleep(0.3)) + threads.append(t) + t.start() + + pid = os.fork() + if pid == 0: + # check that threads states have been cleared + if len(sys._current_frames()) == 1: + os._exit(0) + else: + os._exit(1) + else: + _, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + + for t in threads: + t.join() + + +class SubinterpThreadingTests(BaseTestCase): + + def test_threads_join(self): + # Non-daemon threads should be joined at subinterpreter shutdown + # (issue #18808) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + def test_threads_join_2(self): + # Same as above, but a delay gets introduced after the thread's + # Python code returned but before the thread state is deleted. + # To achieve this, we register a thread-local object which sleeps + # a bit when deallocated. + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + code = r"""if 1: + import os + import threading + import time + + class Sleeper: + def __del__(self): + time.sleep(0.05) + + tls = threading.local() + + def f(): + # Sleep a bit so that the thread is still running when + # Py_EndInterpreter is called. + time.sleep(0.05) + tls.x = Sleeper() + os.write(%d, b"x") + threading.Thread(target=f).start() + """ % (w,) + ret = test.support.run_in_subinterp(code) + self.assertEqual(ret, 0) + # The thread was joined properly. + self.assertEqual(os.read(r, 1), b"x") + + @cpython_only + def test_daemon_threads_fatal_error(self): + subinterp_code = r"""if 1: + import os + import threading + import time + + def f(): + # Make sure the daemon thread is still running when + # Py_EndInterpreter is called. + time.sleep(10) + threading.Thread(target=f, daemon=True).start() + """ + script = r"""if 1: + import _testcapi + + _testcapi.run_in_subinterp(%r) + """ % (subinterp_code,) + with test.support.SuppressCrashReport(): + rc, out, err = assert_python_failure("-c", script) + self.assertIn("Fatal Python error: Py_EndInterpreter: " + "not the last thread", err.decode()) + + +class ThreadingExceptionTests(BaseTestCase): + # A RuntimeError should be raised if Thread.start() is called + # multiple times. + def test_start_thread_again(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, thread.start) + thread.join() + + def test_joining_current_thread(self): + current_thread = threading.current_thread() + self.assertRaises(RuntimeError, current_thread.join); + + def test_joining_inactive_thread(self): + thread = threading.Thread() + self.assertRaises(RuntimeError, thread.join) + + def test_daemonize_active_thread(self): + thread = threading.Thread() + thread.start() + self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + thread.join() + + def test_releasing_unacquired_lock(self): + lock = threading.Lock() + self.assertRaises(RuntimeError, lock.release) + + @unittest.skipUnless(sys.platform == 'darwin' and test.support.python_is_optimized(), + 'test macosx problem') + def test_recursion_limit(self): + # Issue 9670 + # test that excessive recursion within a non-main thread causes + # an exception rather than crashing the interpreter on platforms + # like Mac OS X or FreeBSD which have small default stack sizes + # for threads + script = """if True: + import threading + + def recurse(): + return recurse() + + def outer(): + try: + recurse() + except RecursionError: + pass + + w = threading.Thread(target=outer) + w.start() + w.join() + print('end of main thread') + """ + expected_output = "end of main thread\n" + p = subprocess.Popen([sys.executable, "-c", script], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + data = stdout.decode().replace('\r', '') + self.assertEqual(p.returncode, 0, "Unexpected error: " + stderr.decode()) + self.assertEqual(data, expected_output) + + def test_print_exception(self): + script = r"""if True: + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + @requires_type_collecting + def test_print_exception_stderr_is_none_1(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + sys.stderr = None + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + err = err.decode() + self.assertIn("Exception in thread", err) + self.assertIn("Traceback (most recent call last):", err) + self.assertIn("ZeroDivisionError", err) + self.assertNotIn("Unhandled exception", err) + + def test_print_exception_stderr_is_none_2(self): + script = r"""if True: + import sys + import threading + import time + + running = False + def run(): + global running + running = True + while running: + time.sleep(0.01) + 1/0 + sys.stderr = None + t = threading.Thread(target=run) + t.start() + while not running: + time.sleep(0.01) + running = False + t.join() + """ + rc, out, err = assert_python_ok("-c", script) + self.assertEqual(out, b'') + self.assertNotIn("Unhandled exception", err.decode()) + + def test_bare_raise_in_brand_new_thread(self): + def bare_raise(): + raise + + class Issue27558(threading.Thread): + exc = None + + def run(self): + try: + bare_raise() + except Exception as exc: + self.exc = exc + + thread = Issue27558() + thread.start() + thread.join() + self.assertIsNotNone(thread.exc) + self.assertIsInstance(thread.exc, RuntimeError) + # explicitly break the reference cycle to not leak a dangling thread + thread.exc = None + +class TimerTests(BaseTestCase): + + def setUp(self): + BaseTestCase.setUp(self) + self.callback_args = [] + self.callback_event = threading.Event() + + def test_init_immutable_default_args(self): + # Issue 17435: constructor defaults were mutable objects, they could be + # mutated via the object attributes and affect other Timer objects. + timer1 = threading.Timer(0.01, self._callback_spy) + timer1.start() + self.callback_event.wait() + timer1.args.append("blah") + timer1.kwargs["foo"] = "bar" + self.callback_event.clear() + timer2 = threading.Timer(0.01, self._callback_spy) + timer2.start() + self.callback_event.wait() + self.assertEqual(len(self.callback_args), 2) + self.assertEqual(self.callback_args, [((), {}), ((), {})]) + timer1.join() + timer2.join() + + def _callback_spy(self, *args, **kwargs): + self.callback_args.append((args[:], kwargs.copy())) + self.callback_event.set() + +class LockTests(lock_tests.LockTests): + locktype = staticmethod(threading.Lock) + +class PyRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._PyRLock) + +@unittest.skipIf(threading._CRLock is None, 'RLock not implemented in C') +class CRLockTests(lock_tests.RLockTests): + locktype = staticmethod(threading._CRLock) + +class EventTests(lock_tests.EventTests): + eventtype = staticmethod(threading.Event) + +class ConditionAsRLockTests(lock_tests.RLockTests): + # Condition uses an RLock by default and exports its API. + locktype = staticmethod(threading.Condition) + +class ConditionTests(lock_tests.ConditionTests): + condtype = staticmethod(threading.Condition) + +class SemaphoreTests(lock_tests.SemaphoreTests): + semtype = staticmethod(threading.Semaphore) + +class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): + semtype = staticmethod(threading.BoundedSemaphore) + +class BarrierTests(lock_tests.BarrierTests): + barriertype = staticmethod(threading.Barrier) + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {"ThreadError"} + blacklist = {'currentThread', 'activeCount'} + support.check__all__(self, threading, ('threading', '_thread'), + extra=extra, blacklist=blacklist) + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.7/test_wsgiref.py b/src/greentest/3.7/test_wsgiref.py new file mode 100644 index 0000000..8422b30 --- /dev/null +++ b/src/greentest/3.7/test_wsgiref.py @@ -0,0 +1,785 @@ +from unittest import mock +from test import support +from test.test_httpservers import NoLogRequestHandler +from unittest import TestCase +from wsgiref.util import setup_testing_defaults +from wsgiref.headers import Headers +from wsgiref.handlers import BaseHandler, BaseCGIHandler, SimpleHandler +from wsgiref import util +from wsgiref.validate import validator +from wsgiref.simple_server import WSGIServer, WSGIRequestHandler +from wsgiref.simple_server import make_server +from http.client import HTTPConnection +from io import StringIO, BytesIO, BufferedReader +from socketserver import BaseServer +from platform import python_implementation + +import os +import re +import signal +import sys +import threading +import unittest + + +class MockServer(WSGIServer): + """Non-socket HTTP server""" + + def __init__(self, server_address, RequestHandlerClass): + BaseServer.__init__(self, server_address, RequestHandlerClass) + self.server_bind() + + def server_bind(self): + host, port = self.server_address + self.server_name = host + self.server_port = port + self.setup_environ() + + +class MockHandler(WSGIRequestHandler): + """Non-socket HTTP handler""" + def setup(self): + self.connection = self.request + self.rfile, self.wfile = self.connection + + def finish(self): + pass + + +def hello_app(environ,start_response): + start_response("200 OK", [ + ('Content-Type','text/plain'), + ('Date','Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return [b"Hello, world!"] + + +def header_app(environ, start_response): + start_response("200 OK", [ + ('Content-Type', 'text/plain'), + ('Date', 'Mon, 05 Jun 2006 18:49:54 GMT') + ]) + return [';'.join([ + environ['HTTP_X_TEST_HEADER'], environ['QUERY_STRING'], + environ['PATH_INFO'] + ]).encode('iso-8859-1')] + + +def run_amock(app=hello_app, data=b"GET / HTTP/1.0\n\n"): + server = make_server("", 80, app, MockServer, MockHandler) + inp = BufferedReader(BytesIO(data)) + out = BytesIO() + olderr = sys.stderr + err = sys.stderr = StringIO() + + try: + server.finish_request((inp, out), ("127.0.0.1",8888)) + finally: + sys.stderr = olderr + + return out.getvalue(), err.getvalue() + +def compare_generic_iter(make_it,match): + """Utility to compare a generic 2.1/2.2+ iterator with an iterable + + If running under Python 2.2+, this tests the iterator using iter()/next(), + as well as __getitem__. 'make_it' must be a function returning a fresh + iterator to be tested (since this may test the iterator twice).""" + + it = make_it() + n = 0 + for item in match: + if not it[n]==item: raise AssertionError + n+=1 + try: + it[n] + except IndexError: + pass + else: + raise AssertionError("Too many items from __getitem__",it) + + try: + iter, StopIteration + except NameError: + pass + else: + # Only test iter mode under 2.2+ + it = make_it() + if not iter(it) is it: raise AssertionError + for item in match: + if not next(it) == item: raise AssertionError + try: + next(it) + except StopIteration: + pass + else: + raise AssertionError("Too many items from .__next__()", it) + + +class IntegrationTests(TestCase): + + def check_hello(self, out, has_length=True): + pyver = (python_implementation() + "/" + + sys.version.split()[0]) + self.assertEqual(out, + ("HTTP/1.0 200 OK\r\n" + "Server: WSGIServer/0.2 " + pyver +"\r\n" + "Content-Type: text/plain\r\n" + "Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + + (has_length and "Content-Length: 13\r\n" or "") + + "\r\n" + "Hello, world!").encode("iso-8859-1") + ) + + def test_plain_hello(self): + out, err = run_amock() + self.check_hello(out) + + def test_environ(self): + request = ( + b"GET /p%61th/?query=test HTTP/1.0\n" + b"X-Test-Header: Python test \n" + b"X-Test-Header: Python test 2\n" + b"Content-Length: 0\n\n" + ) + out, err = run_amock(header_app, request) + self.assertEqual( + out.splitlines()[-1], + b"Python test,Python test 2;query=test;/path/" + ) + + def test_request_length(self): + out, err = run_amock(data=b"GET " + (b"x" * 65537) + b" HTTP/1.0\n\n") + self.assertEqual(out.splitlines()[0], + b"HTTP/1.0 414 Request-URI Too Long") + + def test_validated_hello(self): + out, err = run_amock(validator(hello_app)) + # the middleware doesn't support len(), so content-length isn't there + self.check_hello(out, has_length=False) + + def test_simple_validation_error(self): + def bad_app(environ,start_response): + start_response("200 OK", ('Content-Type','text/plain')) + return ["Hello, world!"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], + "AssertionError: Headers (('Content-Type', 'text/plain')) must" + " be of type list: " + ) + + def test_status_validation_errors(self): + def create_bad_app(status): + def bad_app(environ, start_response): + start_response(status, [("Content-Type", "text/plain; charset=utf-8")]) + return [b"Hello, world!"] + return bad_app + + tests = [ + ('200', 'AssertionError: Status must be at least 4 characters'), + ('20X OK', 'AssertionError: Status message must begin w/3-digit code'), + ('200OK', 'AssertionError: Status message must have a space after code'), + ] + + for status, exc_message in tests: + with self.subTest(status=status): + out, err = run_amock(create_bad_app(status)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual(err.splitlines()[-2], exc_message) + + def test_wsgi_input(self): + def bad_app(e,s): + e["wsgi.input"].read() + s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) + return [b"data"] + out, err = run_amock(validator(bad_app)) + self.assertTrue(out.endswith( + b"A server error occurred. Please contact the administrator." + )) + self.assertEqual( + err.splitlines()[-2], "AssertionError" + ) + + def test_bytes_validation(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain; charset=utf-8"), + ("Date", "Wed, 24 Dec 2008 13:29:32 GMT"), + ]) + return [b"data"] + out, err = run_amock(validator(app)) + self.assertTrue(err.endswith('"GET / HTTP/1.0" 200 4\n')) + ver = sys.version.split()[0].encode('ascii') + py = python_implementation().encode('ascii') + pyver = py + b"/" + ver + self.assertEqual( + b"HTTP/1.0 200 OK\r\n" + b"Server: WSGIServer/0.2 "+ pyver + b"\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"Date: Wed, 24 Dec 2008 13:29:32 GMT\r\n" + b"\r\n" + b"data", + out) + + def test_cp1252_url(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain"), + ("Date", "Wed, 24 Dec 2008 13:29:32 GMT"), + ]) + # PEP3333 says environ variables are decoded as latin1. + # Encode as latin1 to get original bytes + return [e["PATH_INFO"].encode("latin1")] + + out, err = run_amock( + validator(app), data=b"GET /\x80%80 HTTP/1.0") + self.assertEqual( + [ + b"HTTP/1.0 200 OK", + mock.ANY, + b"Content-Type: text/plain", + b"Date: Wed, 24 Dec 2008 13:29:32 GMT", + b"", + b"/\x80\x80", + ], + out.splitlines()) + + def test_interrupted_write(self): + # BaseHandler._write() and _flush() have to write all data, even if + # it takes multiple send() calls. Test this by interrupting a send() + # call with a Unix signal. + pthread_kill = support.get_attribute(signal, "pthread_kill") + + def app(environ, start_response): + start_response("200 OK", []) + return [b'\0' * support.SOCK_MAX_SIZE] + + class WsgiHandler(NoLogRequestHandler, WSGIRequestHandler): + pass + + server = make_server(support.HOST, 0, app, handler_class=WsgiHandler) + self.addCleanup(server.server_close) + interrupted = threading.Event() + + def signal_handler(signum, frame): + interrupted.set() + + original = signal.signal(signal.SIGUSR1, signal_handler) + self.addCleanup(signal.signal, signal.SIGUSR1, original) + received = None + main_thread = threading.get_ident() + + def run_client(): + http = HTTPConnection(*server.server_address) + http.request("GET", "/") + with http.getresponse() as response: + response.read(100) + # The main thread should now be blocking in a send() system + # call. But in theory, it could get interrupted by other + # signals, and then retried. So keep sending the signal in a + # loop, in case an earlier signal happens to be delivered at + # an inconvenient moment. + while True: + pthread_kill(main_thread, signal.SIGUSR1) + if interrupted.wait(timeout=float(1)): + break + nonlocal received + received = len(response.read()) + http.close() + + background = threading.Thread(target=run_client) + background.start() + server.handle_request() + background.join() + self.assertEqual(received, support.SOCK_MAX_SIZE - 100) + + +class UtilityTests(TestCase): + + def checkShift(self,sn_in,pi_in,part,sn_out,pi_out): + env = {'SCRIPT_NAME':sn_in,'PATH_INFO':pi_in} + util.setup_testing_defaults(env) + self.assertEqual(util.shift_path_info(env),part) + self.assertEqual(env['PATH_INFO'],pi_out) + self.assertEqual(env['SCRIPT_NAME'],sn_out) + return env + + def checkDefault(self, key, value, alt=None): + # Check defaulting when empty + env = {} + util.setup_testing_defaults(env) + if isinstance(value, StringIO): + self.assertIsInstance(env[key], StringIO) + elif isinstance(value,BytesIO): + self.assertIsInstance(env[key],BytesIO) + else: + self.assertEqual(env[key], value) + + # Check existing value + env = {key:alt} + util.setup_testing_defaults(env) + self.assertIs(env[key], alt) + + def checkCrossDefault(self,key,value,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(kw[key],value) + + def checkAppURI(self,uri,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.application_uri(kw),uri) + + def checkReqURI(self,uri,query=1,**kw): + util.setup_testing_defaults(kw) + self.assertEqual(util.request_uri(kw,query),uri) + + def checkFW(self,text,size,match): + + def make_it(text=text,size=size): + return util.FileWrapper(StringIO(text),size) + + compare_generic_iter(make_it,match) + + it = make_it() + self.assertFalse(it.filelike.closed) + + for item in it: + pass + + self.assertFalse(it.filelike.closed) + + it.close() + self.assertTrue(it.filelike.closed) + + def testSimpleShifts(self): + self.checkShift('','/', '', '/', '') + self.checkShift('','/x', 'x', '/x', '') + self.checkShift('/','', None, '/', '') + self.checkShift('/a','/x/y', 'x', '/a/x', '/y') + self.checkShift('/a','/x/', 'x', '/a/x', '/') + + def testNormalizedShifts(self): + self.checkShift('/a/b', '/../y', '..', '/a', '/y') + self.checkShift('', '/../y', '..', '', '/y') + self.checkShift('/a/b', '//y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '//y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '/./y', 'y', '/a/b/y', '') + self.checkShift('/a/b', '/./y/', 'y', '/a/b/y', '/') + self.checkShift('/a/b', '///./..//y/.//', '..', '/a', '/y/') + self.checkShift('/a/b', '///', '', '/a/b/', '') + self.checkShift('/a/b', '/.//', '', '/a/b/', '') + self.checkShift('/a/b', '/x//', 'x', '/a/b/x', '/') + self.checkShift('/a/b', '/.', None, '/a/b', '') + + def testDefaults(self): + for key, value in [ + ('SERVER_NAME','127.0.0.1'), + ('SERVER_PORT', '80'), + ('SERVER_PROTOCOL','HTTP/1.0'), + ('HTTP_HOST','127.0.0.1'), + ('REQUEST_METHOD','GET'), + ('SCRIPT_NAME',''), + ('PATH_INFO','/'), + ('wsgi.version', (1,0)), + ('wsgi.run_once', 0), + ('wsgi.multithread', 0), + ('wsgi.multiprocess', 0), + ('wsgi.input', BytesIO()), + ('wsgi.errors', StringIO()), + ('wsgi.url_scheme','http'), + ]: + self.checkDefault(key,value) + + def testCrossDefaults(self): + self.checkCrossDefault('HTTP_HOST',"foo.bar",SERVER_NAME="foo.bar") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="on") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="1") + self.checkCrossDefault('wsgi.url_scheme',"https",HTTPS="yes") + self.checkCrossDefault('wsgi.url_scheme',"http",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"80",HTTPS="foo") + self.checkCrossDefault('SERVER_PORT',"443",HTTPS="on") + + def testGuessScheme(self): + self.assertEqual(util.guess_scheme({}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"foo"}), "http") + self.assertEqual(util.guess_scheme({'HTTPS':"on"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"yes"}), "https") + self.assertEqual(util.guess_scheme({'HTTPS':"1"}), "https") + + def testAppURIs(self): + self.checkAppURI("http://127.0.0.1/") + self.checkAppURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkAppURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkAppURI("http://spam.example.com:2071/", + HTTP_HOST="spam.example.com:2071", SERVER_PORT="2071") + self.checkAppURI("http://spam.example.com/", + SERVER_NAME="spam.example.com") + self.checkAppURI("http://127.0.0.1/", + HTTP_HOST="127.0.0.1", SERVER_NAME="spam.example.com") + self.checkAppURI("https://127.0.0.1/", HTTPS="on") + self.checkAppURI("http://127.0.0.1:8000/", SERVER_PORT="8000", + HTTP_HOST=None) + + def testReqURIs(self): + self.checkReqURI("http://127.0.0.1/") + self.checkReqURI("http://127.0.0.1/spam", SCRIPT_NAME="/spam") + self.checkReqURI("http://127.0.0.1/sp%E4m", SCRIPT_NAME="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam", + SCRIPT_NAME="/spammity", PATH_INFO="/spam") + self.checkReqURI("http://127.0.0.1/spammity/sp%E4m", + SCRIPT_NAME="/spammity", PATH_INFO="/sp\xe4m") + self.checkReqURI("http://127.0.0.1/spammity/spam;ham", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;ham") + self.checkReqURI("http://127.0.0.1/spammity/spam;cookie=1234,5678", + SCRIPT_NAME="/spammity", PATH_INFO="/spam;cookie=1234,5678") + self.checkReqURI("http://127.0.0.1/spammity/spam?say=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam?s%E4y=ni", + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="s%E4y=ni") + self.checkReqURI("http://127.0.0.1/spammity/spam", 0, + SCRIPT_NAME="/spammity", PATH_INFO="/spam",QUERY_STRING="say=ni") + + def testFileWrapper(self): + self.checkFW("xyz"*50, 120, ["xyz"*40,"xyz"*10]) + + def testHopByHop(self): + for hop in ( + "Connection Keep-Alive Proxy-Authenticate Proxy-Authorization " + "TE Trailers Transfer-Encoding Upgrade" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertTrue(util.is_hop_by_hop(alt)) + + # Not comprehensive, just a few random header names + for hop in ( + "Accept Cache-Control Date Pragma Trailer Via Warning" + ).split(): + for alt in hop, hop.title(), hop.upper(), hop.lower(): + self.assertFalse(util.is_hop_by_hop(alt)) + +class HeaderTests(TestCase): + + def testMappingInterface(self): + test = [('x','y')] + self.assertEqual(len(Headers()), 0) + self.assertEqual(len(Headers([])),0) + self.assertEqual(len(Headers(test[:])),1) + self.assertEqual(Headers(test[:]).keys(), ['x']) + self.assertEqual(Headers(test[:]).values(), ['y']) + self.assertEqual(Headers(test[:]).items(), test) + self.assertIsNot(Headers(test).items(), test) # must be copy! + + h = Headers() + del h['foo'] # should not raise an error + + h['Foo'] = 'bar' + for m in h.__contains__, h.get, h.get_all, h.__getitem__: + self.assertTrue(m('foo')) + self.assertTrue(m('Foo')) + self.assertTrue(m('FOO')) + self.assertFalse(m('bar')) + + self.assertEqual(h['foo'],'bar') + h['foo'] = 'baz' + self.assertEqual(h['FOO'],'baz') + self.assertEqual(h.get_all('foo'),['baz']) + + self.assertEqual(h.get("foo","whee"), "baz") + self.assertEqual(h.get("zoo","whee"), "whee") + self.assertEqual(h.setdefault("foo","whee"), "baz") + self.assertEqual(h.setdefault("zoo","whee"), "whee") + self.assertEqual(h["foo"],"baz") + self.assertEqual(h["zoo"],"whee") + + def testRequireList(self): + self.assertRaises(TypeError, Headers, "foo") + + def testExtras(self): + h = Headers() + self.assertEqual(str(h),'\r\n') + + h.add_header('foo','bar',baz="spam") + self.assertEqual(h['foo'], 'bar; baz="spam"') + self.assertEqual(str(h),'foo: bar; baz="spam"\r\n\r\n') + + h.add_header('Foo','bar',cheese=None) + self.assertEqual(h.get_all('foo'), + ['bar; baz="spam"', 'bar; cheese']) + + self.assertEqual(str(h), + 'foo: bar; baz="spam"\r\n' + 'Foo: bar; cheese\r\n' + '\r\n' + ) + +class ErrorHandler(BaseCGIHandler): + """Simple handler subclass for testing BaseHandler""" + + # BaseHandler records the OS environment at import time, but envvars + # might have been changed later by other tests, which trips up + # HandlerTests.testEnviron(). + os_environ = dict(os.environ.items()) + + def __init__(self,**kw): + setup_testing_defaults(kw) + BaseCGIHandler.__init__( + self, BytesIO(), BytesIO(), StringIO(), kw, + multithread=True, multiprocess=True + ) + +class TestHandler(ErrorHandler): + """Simple handler subclass for testing BaseHandler, w/error passthru""" + + def handle_error(self): + raise # for testing, we want to see what's happening + + +class HandlerTests(TestCase): + + def checkEnvironAttrs(self, handler): + env = handler.environ + for attr in [ + 'version','multithread','multiprocess','run_once','file_wrapper' + ]: + if attr=='file_wrapper' and handler.wsgi_file_wrapper is None: + continue + self.assertEqual(getattr(handler,'wsgi_'+attr),env['wsgi.'+attr]) + + def checkOSEnviron(self,handler): + empty = {}; setup_testing_defaults(empty) + env = handler.environ + from os import environ + for k,v in environ.items(): + if k not in empty: + self.assertEqual(env[k],v) + for k,v in empty.items(): + self.assertIn(k, env) + + def testEnviron(self): + h = TestHandler(X="Y") + h.setup_environ() + self.checkEnvironAttrs(h) + self.checkOSEnviron(h) + self.assertEqual(h.environ["X"],"Y") + + def testCGIEnviron(self): + h = BaseCGIHandler(None,None,None,{}) + h.setup_environ() + for key in 'wsgi.url_scheme', 'wsgi.input', 'wsgi.errors': + self.assertIn(key, h.environ) + + def testScheme(self): + h=TestHandler(HTTPS="on"); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'https') + h=TestHandler(); h.setup_environ() + self.assertEqual(h.environ['wsgi.url_scheme'],'http') + + def testAbstractMethods(self): + h = BaseHandler() + for name in [ + '_flush','get_stdin','get_stderr','add_cgi_vars' + ]: + self.assertRaises(NotImplementedError, getattr(h,name)) + self.assertRaises(NotImplementedError, h._write, "test") + + def testContentLength(self): + # Demo one reason iteration is better than write()... ;) + + def trivial_app1(e,s): + s('200 OK',[]) + return [e['wsgi.url_scheme'].encode('iso-8859-1')] + + def trivial_app2(e,s): + s('200 OK',[])(e['wsgi.url_scheme'].encode('iso-8859-1')) + return [] + + def trivial_app3(e,s): + s('200 OK',[]) + return ['\u0442\u0435\u0441\u0442'.encode("utf-8")] + + def trivial_app4(e,s): + # Simulate a response to a HEAD request + s('200 OK',[('Content-Length', '12345')]) + return [] + + h = TestHandler() + h.run(trivial_app1) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "Content-Length: 4\r\n" + "\r\n" + "http").encode("iso-8859-1")) + + h = TestHandler() + h.run(trivial_app2) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "\r\n" + "http").encode("iso-8859-1")) + + h = TestHandler() + h.run(trivial_app3) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 8\r\n' + b'\r\n' + b'\xd1\x82\xd0\xb5\xd1\x81\xd1\x82') + + h = TestHandler() + h.run(trivial_app4) + self.assertEqual(h.stdout.getvalue(), + b'Status: 200 OK\r\n' + b'Content-Length: 12345\r\n' + b'\r\n') + + def testBasicErrorOutput(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + def error_app(e,s): + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(non_error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n").encode("iso-8859-1")) + self.assertEqual(h.stderr.getvalue(),"") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: %s\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: %d\r\n" + "\r\n" % (h.error_status,len(h.error_body))).encode('iso-8859-1') + + h.error_body) + + self.assertIn("AssertionError", h.stderr.getvalue()) + + def testErrorAfterOutput(self): + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + raise AssertionError("This should be caught by handler") + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(h.stdout.getvalue(), + ("Status: 200 OK\r\n" + "\r\n".encode("iso-8859-1")+MSG)) + self.assertIn("AssertionError", h.stderr.getvalue()) + + def testHeaderFormats(self): + + def non_error_app(e,s): + s('200 OK',[]) + return [] + + stdpat = ( + r"HTTP/%s 200 OK\r\n" + r"Date: \w{3}, [ 0123]\d \w{3} \d{4} \d\d:\d\d:\d\d GMT\r\n" + r"%s" r"Content-Length: 0\r\n" r"\r\n" + ) + shortpat = ( + "Status: 200 OK\r\n" "Content-Length: 0\r\n" "\r\n" + ).encode("iso-8859-1") + + for ssw in "FooBar/1.0", None: + sw = ssw and "Server: %s\r\n" % ssw or "" + + for version in "1.0", "1.1": + for proto in "HTTP/0.9", "HTTP/1.0", "HTTP/1.1": + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = False + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + self.assertEqual(shortpat,h.stdout.getvalue()) + + h = TestHandler(SERVER_PROTOCOL=proto) + h.origin_server = True + h.http_version = version + h.server_software = ssw + h.run(non_error_app) + if proto=="HTTP/0.9": + self.assertEqual(h.stdout.getvalue(),b"") + else: + self.assertTrue( + re.match((stdpat%(version,sw)).encode("iso-8859-1"), + h.stdout.getvalue()), + ((stdpat%(version,sw)).encode("iso-8859-1"), + h.stdout.getvalue()) + ) + + def testBytesData(self): + def app(e, s): + s("200 OK", [ + ("Content-Type", "text/plain; charset=utf-8"), + ]) + return [b"data"] + + h = TestHandler() + h.run(app) + self.assertEqual(b"Status: 200 OK\r\n" + b"Content-Type: text/plain; charset=utf-8\r\n" + b"Content-Length: 4\r\n" + b"\r\n" + b"data", + h.stdout.getvalue()) + + def testCloseOnError(self): + side_effects = {'close_called': False} + MSG = b"Some output has been sent" + def error_app(e,s): + s("200 OK",[])(MSG) + class CrashyIterable(object): + def __iter__(self): + while True: + yield b'blah' + raise AssertionError("This should be caught by handler") + def close(self): + side_effects['close_called'] = True + return CrashyIterable() + + h = ErrorHandler() + h.run(error_app) + self.assertEqual(side_effects['close_called'], True) + + def testPartialWrite(self): + written = bytearray() + + class PartialWriter: + def write(self, b): + partial = b[:7] + written.extend(partial) + return len(partial) + + def flush(self): + pass + + environ = {"SERVER_PROTOCOL": "HTTP/1.0"} + h = SimpleHandler(BytesIO(), PartialWriter(), sys.stderr, environ) + msg = "should not do partial writes" + with self.assertWarnsRegex(DeprecationWarning, msg): + h.run(hello_app) + self.assertEqual(b"HTTP/1.0 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + b"Content-Length: 13\r\n" + b"\r\n" + b"Hello, world!", + written) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/greentest/3.7/version b/src/greentest/3.7/version new file mode 100644 index 0000000..7c69a55 --- /dev/null +++ b/src/greentest/3.7/version @@ -0,0 +1 @@ +3.7.0 diff --git a/src/greentest/3.7/wrongcert.pem b/src/greentest/3.7/wrongcert.pem new file mode 100644 index 0000000..5f92f9b --- /dev/null +++ b/src/greentest/3.7/wrongcert.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnH +FlbsVUg2Xtk6+bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6T +f9lnNTwpSoeK24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQAB +AoGAQFko4uyCgzfxr4Ezb4Mp5pN3Npqny5+Jey3r8EjSAX9Ogn+CNYgoBcdtFgbq +1yif/0sK7ohGBJU9FUCAwrqNBI9ZHB6rcy7dx+gULOmRBGckln1o5S1+smVdmOsW +7zUVLBVByKuNWqTYFlzfVd6s4iiXtAE2iHn3GCyYdlICwrECQQDhMQVxHd3EFbzg +SFmJBTARlZ2GKA3c1g/h9/XbkEPQ9/RwI3vnjJ2RaSnjlfoLl8TOcf0uOGbOEyFe +19RvCLXjAkEA1s+UE5ziF+YVkW3WolDCQ2kQ5WG9+ccfNebfh6b67B7Ln5iG0Sbg +ky9cjsO3jbMJQtlzAQnH1850oRD5Gi51dQJAIbHCDLDZU9Ok1TI+I2BhVuA6F666 +lEZ7TeZaJSYq34OaUYUdrwG9OdqwZ9sy9LUav4ESzu2lhEQchCJrKMn23QJAReqs +ZLHUeTjfXkVk7dHhWPWSlUZ6AhmIlA/AQ7Payg2/8wM/JkZEJEPvGVykms9iPUrv +frADRr+hAGe43IewnQJBAJWKZllPgKuEBPwoEldHNS8nRu61D7HzxEzQ2xnfj+Nk +2fgf1MAzzTRsikfGENhVsVWeqOcijWb6g5gsyCmlRpc= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICsDCCAhmgAwIBAgIJAOqYOYFJfEEoMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIEwpTb21lLVN0YXRlMSEwHwYDVQQKExhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMDgwNjI2MTgxNTUyWhcNMDkwNjI2MTgxNTUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQC89ZNxjTgWgq7Z1g0tJ65w+k7lNAj5IgjLb155UkUrz0XsHDnHFlbsVUg2Xtk6 ++bo2UEYIzN7cIm5ImpmyW/2z0J1IDVDlvR2xJ659xrE0v5c2cB6Tf9lnNTwpSoeK +24Nd7Jwq4j9vk95fLrdqsBq0/KVlsCXeixS/CaqqduXfvwIDAQABo4GnMIGkMB0G +A1UdDgQWBBTctMtI3EO9OjLI0x9Zo2ifkwIiNjB1BgNVHSMEbjBsgBTctMtI3EO9 +OjLI0x9Zo2ifkwIiNqFJpEcwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUt +U3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZIIJAOqYOYFJ +fEEoMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAQwa7jya/DfhaDn7E +usPkpgIX8WCL2B1SqnRTXEZfBPPVq/cUmFGyEVRVATySRuMwi8PXbVcOhXXuocA+ +43W+iIsD9pXapCZhhOerCq18TC1dWK98vLUsoK8PMjB6e5H/O8bqojv0EeC+fyCw +eSHj5jpC8iZKjCHBn+mAi4cQ514= +-----END CERTIFICATE----- diff --git a/src/greentest/README.rst b/src/greentest/README.rst new file mode 100644 index 0000000..4befd05 --- /dev/null +++ b/src/greentest/README.rst @@ -0,0 +1,16 @@ +================= + Versioned Tests +================= + +The test directories that begin with a number (e.g., 2.7 and 3.5) are +copies of the standard library tests for that specific version of +Python. Each directory has a ``version`` file that identifies the +specific point release the tests come from. The tests are only +expected to pass if the version of python running the tests exactly +matches the version in that file. If this is not the case, the test +runner will print a warning. + +.. caution:: For ease of updating the standard library tests, gevent + tries very hard not to modify the tests if at all + possible. Prefer to use the ``patched_tests_setup.py`` or + ``known_failures.py`` file if necessary. diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..7723069 --- /dev/null +++ b/tox.ini @@ -0,0 +1,56 @@ +[tox] +envlist = + py27,py34,py35,py36,py37,py27-cffi,pypy,pypy3,py27-libuv,lint + +[testenv] +deps = + greenlet + cython >= 0.24 + coverage >= 4.0 + psutil + cffi +whitelist_externals = + * +commands = + make basictest + + +[testenv:py27-full] +basepython = python2.7 +commands = + make alltoxtest + +[testenv:pypy] +deps = + +[testenv:lint] +basepython = + python2.7 +deps = + {[testenv]deps} + prospector +commands = + make lint + +[testenv:py27-cffi] +basepython = + python2.7 +setenv = + GEVENT_LOOP=libev-cffi +commands = + make basictest + +[testenv:py27-libuv] +basepython = + python2.7 +setenv = + GEVENT_LOOP=libuv-cffi +commands = + make basictest + + +[testenv:leak] +basepython = + python2.7 +commands = + make leaktest diff --git a/util/cythonpp.py b/util/cythonpp.py new file mode 100755 index 0000000..5f53183 --- /dev/null +++ b/util/cythonpp.py @@ -0,0 +1,1122 @@ +#!/usr/bin/env python +# Copyright (C) 2011-2012 Denis Bilenko (http://denisbilenko.com) +# Copyright (C) 2015-2016 gevent contributors + +###################################### +###################################### +###################################### +### WARNING WARNING WARNING WARNING +## +## This script is unmaintained and no +## longer in use in this project due to +## bugs. +## See https://github.com/gevent/gevent/issues/1076 +## +### WARNING WARNING WARNING WARNING +###################################### +###################################### +###################################### + + +from __future__ import print_function +import sys +import os +import os.path +import re +import traceback +import datetime +import difflib +from hashlib import md5 +from itertools import combinations, product +import subprocess +import multiprocessing +import tempfile +import shutil +from collections import OrderedDict + +import threading + +class Thread(threading.Thread): + value = None + + def run(self): + target = getattr(self, '_target', None) # Py3 + if target is None: + target = getattr(self, '_Thread__target') + args = getattr(self, '_Thread__args') + else: + args = self._args + self.value = target(*args) + +do_exec = None +if sys.version_info >= (3, 0): + exec("def do_exec(co, loc): exec(co, loc)\n") +else: + exec("def do_exec(co, loc): exec co in loc\n") + + +CYTHON = os.environ.get('CYTHON') or 'cython' +DEBUG = os.environ.get('CYTHONPP_DEBUG', False) +TRACE = DEBUG == 'trace' +WRITE_OUTPUT = False + +if os.getenv('READTHEDOCS'): + # Sometimes RTD fails to put our virtualenv bin directory + # on the PATH, meaning we can't run cython. Fix that. + new_path = os.environ['PATH'] + os.pathsep + os.path.dirname(sys.executable) + os.environ['PATH'] = new_path + +# Parameter name in macros must match this regex: +param_name_re = re.compile(r'^[a-zA-Z_]\w*$') + +# First line of a definition of a new macro: +define_re = re.compile(r'^#define\s+([a-zA-Z_]\w*)(\((?:[^,)]+,)*[^,)]+\))?\s+(.*)$') + + +# cython header: +cython_header_re = re.compile(r'^/\* (generated by cython [^\s*]+)[^*]+\*/$', re.I) +#assert cython_header_re.match('/* Generated by Cython 0.21.1 */').group(1) == 'Generated by Cython 0.21.1' +#assert cython_header_re.match('/* Generated by Cython 0.19 on 55-555-555 */').group(1) == 'Generated by Cython 0.19' + +class EmptyConfigurationError(TypeError): + pass + +class Configuration(frozenset): + """ + A set of CPP conditions that apply to a given sequence + of lines. Sometimes referred to as a "tag". + + Configurations are iterated in sorted order for consistency + across runs. + """ + __slots__ = ('_sorted',) + _cache = {} + + def __new__(cls, iterable): + sorted_iterable = tuple(sorted(frozenset(iterable))) + if not sorted_iterable: + raise EmptyConfigurationError("Empty configurations not allowed") + + if sorted_iterable not in cls._cache: + if not all(isinstance(x, Condition) for x in sorted_iterable): + raise TypeError("Must be iterable of conditions") + self = frozenset.__new__(cls, sorted_iterable) + self._sorted = sorted_iterable + cls._cache[sorted_iterable] = self + + return cls._cache[sorted_iterable] + + def union(self, other): + return Configuration(frozenset.union(self, other)) + + def __add__(self, conditions): + return self.union(conditions) + + def difference(self, other): + try: + return Configuration(frozenset.difference(self, other)) + except EmptyConfigurationError: + raise EmptyConfigurationError( + "Couldn't subtract %r from %r" % (self, other)) + + def __sub__(self, other): + return self.difference(other) + + def __iter__(self): + return iter(self._sorted) + + def format_tag(self): + return ' && '.join([x.format_cond() for x in self]) + + def __repr__(self): + return "Configuration({" + ', '.join((repr(x) for x in self)) + '})' + + @property + def all_directives(self): + "All the directives in the conditions of this configuration" + return set(x.directive for x in self) + + def is_impossible(self): + """ + Return whether the configuration (a Configuration) contradicts itself. + """ + conds = {} + for cond_name, cond_setting in self: + if cond_name in conds: + if conds.get(cond_name) != cond_setting: + return True + conds[cond_name] = cond_setting + + def is_condition_true(self, directive): + if directive.startswith('#if '): + parameter = directive.split(' ', 1)[1] + elif directive.startswith('#ifdef '): + parameter = directive.split(' ', 1)[1] + parameter = 'defined(%s)' % parameter + else: + raise AssertionError('Invalid directive: %r' % directive) + cond = (parameter, True) + return cond in self + + def attach_tags(self, text): + result = [x for x in text.split('\n')] + if result and not result[-1]: + del result[-1] + return [Str(x + '\n', self) for x in result] + + @classmethod + def get_configurations(cls, filename): + """ + Returns a set of Configuration objects representing + the configurations seen in the file. + """ + conditions = set() + condition_stack = [] + linecount = 0 + match_condition = Condition.match_condition + with open(filename) as f: + for line in f: + linecount += 1 + try: + m = match_condition(line) + if m is None: + if condition_stack: # added + conditions.add(cls(condition_stack)) + continue + + split = m.group(1).strip().split(' ', 1) + directive = split[0].strip() + if len(split) == 1: + parameter = None + assert directive in ('else', 'endif'), directive + else: + parameter = split[1].strip() + assert directive in ('if', 'ifdef'), directive + + if directive == 'ifdef': + directive = 'if' + parameter = 'defined(%s)' % parameter + + if directive == 'if': + condition_stack.append(Condition(parameter, True)) + elif directive == 'else': + if not condition_stack: + raise SyntaxError('Unexpected "#else"') + last_cond, true = condition_stack.pop() + assert true is True, true + condition_stack.append(Condition(last_cond, not true)) + elif directive == 'endif': + if not condition_stack: + raise SyntaxError('Unexpected "#endif"') + condition_stack.pop() + else: + raise AssertionError('Internal error') + except BaseException as ex: + log('%s:%s: %s', filename, linecount, ex) + if isinstance(ex, SyntaxError): + sys.exit(1) + else: + raise + dbg("Found conditions %s", conditions) + return conditions + + + @classmethod + def get_permutations_of_configurations(cls, items): + """ + Returns a set of Configuration objects representing all the + possible permutations of the given list of configuration + objects. Impossible configurations are excluded. + """ + def flattened(tuple_of_configurations): + # product() produces a list of tuples. Each + # item in the tuple is a different configuration object. + set_of_configurations = set(tuple_of_configurations) + sorted_set_of_configurations = sorted(set_of_configurations) + conditions = [] + for configuration in sorted_set_of_configurations: + for condition in configuration: + conditions.append(condition) + return cls(conditions) + + flattened_configurations = (flattened(x) for x in product(items, repeat=len(items))) + possible_configurations = set((x for x in flattened_configurations if not x.is_impossible())) + + return possible_configurations + + @classmethod + def get_permutations_of_configurations_in_file(cls, filename): + """ + Returns a sorted list of unique configurations possible in the given + file. + """ + return sorted(cls.get_permutations_of_configurations(cls.get_configurations(filename))) + + @classmethod + def get_complete_configurations(cls, filename): + """ + Return a sorted list of the set of unique configurations possible + in the given file; each configuration will have the all the conditions + it specifies, plus the implicit conditions that it does not specify. + """ + configurations = cls.get_permutations_of_configurations_in_file(filename) + all_cond_names = set() + for config in configurations: + all_cond_names = all_cond_names.union(config.all_directives) + + result = set() + for configuration in configurations: + cond_names_in_configuration = configuration.all_directives + cond_names_not_in_configuration = all_cond_names - cond_names_in_configuration + for missing_cond_name in cond_names_not_in_configuration: + configuration = configuration + (Condition(missing_cond_name, False), ) + result.add(cls(sorted(configuration))) + + # XXX: Previously, this produced eight configurations for gevent/corecext.ppyx + # (containing all the possible permutations). + # But two of them produced identical results and were hashed as such + # by run_cython_on_files. We're now producing just the 6 results that + # are distinct in that case. I'm not exactly sure why + assert all(isinstance(x, Configuration) for x in result) + return sorted(result) + +class Condition(tuple): + """ + A single CPP directive. + + Two-tuple: (name, True|False) + """ + # Conditional directive: + condition_re = re.compile(r'^#(ifdef\s+.+|if\s+.+|else\s*|endif\s*)$') + + _cache = {} + + __slots__ = () + + def __new__(cls, *args): + if len(args) == 2: + # name, value; from literal constructor + sequence = args + elif len(args) == 1: + sequence = args[0] + else: + raise TypeError("wrong argument number", args) + + if sequence not in cls._cache: + if len(sequence) != 2: + raise TypeError("Must be len 2", sequence) + if not isinstance(sequence[0], str) or not isinstance(sequence[1], bool): + raise TypeError("Must be (str, bool)") + cls._cache[sequence] = tuple.__new__(cls, sequence) + return cls._cache[sequence] + + def __repr__(self): + return "Condition" + tuple.__repr__(self) + + @property + def directive(self): + return self[0] + + @property + def value(self): + return self[1] + + def format_cond(self): + if self.value: + return self.directive + + return '!' + self.directive + + def inverted(self): + return Condition(self.directive, not self.value) + + @classmethod + def match_condition(cls, line): + line = line.strip() + if line.endswith(':'): + return None + return cls.condition_re.match(line) + +class ConfigurationGroups(tuple): + """ + A sequence of Configurations that apply to the given line. + + These are maintained in sorted order. + """ + + _cache = {} + + def __new__(cls, tags): + sorted_tags = tuple(sorted(tags)) + if sorted_tags not in cls._cache: + if not all(isinstance(x, Configuration) for x in tags): + raise TypeError("Must be a Configuration", tags) + + self = tuple.__new__(cls, sorted(tags)) + self._simplified = False + cls._cache[sorted_tags] = self + return cls._cache[sorted_tags] + + def __repr__(self): + return "ConfigurationGroups" + tuple.__repr__(self) + + def __add__(self, other): + l = list(self) + l.extend(other) + return ConfigurationGroups(l) + + def exact_reverse(self, tags2): + if not self: + return + if not tags2: + return + if not isinstance(self, tuple): + raise TypeError(repr(self)) + if not isinstance(tags2, tuple): + raise TypeError(repr(tags2)) + if len(self) == 1 and len(tags2) == 1: + tag1 = self[0] + tag2 = tags2[0] + assert isinstance(tag1, Configuration), tag1 + assert isinstance(tag2, Configuration), tag2 + if len(tag1) == 1 and len(tag2) == 1: + tag1 = list(tag1)[0] + tag2 = list(tag2)[0] + if tag1[0] == tag2[0]: + return sorted([tag1[1], tag2[1]]) == [False, True] + + def format_tags(self): + return ' || '.join('(%s)' % x.format_tag() for x in sorted(self)) + + + def simplify_tags(self): + """ + >>> simplify_tags([set([('defined(world)', True), ('defined(hello)', True)]), + ... set([('defined(world)', False), ('defined(hello)', True)])]) + [set([('defined(hello)', True)])] + >>> simplify_tags([set([('defined(LIBEV_EMBED)', True), ('defined(_WIN32)', True)]), set([('defined(LIBEV_EMBED)', True), + ... ('defined(_WIN32)', False)]), set([('defined(_WIN32)', False), ('defined(LIBEV_EMBED)', False)]), + ... set([('defined(LIBEV_EMBED)', False), ('defined(_WIN32)', True)])]) + [] + """ + if self._simplified: + return self + + if (len(self) == 2 + and len(self[0]) == len(self[1]) == 1 + and list(self[0])[0] == list(self[1])[0].inverted()): + # This trivially simplifies to the empty group + # Its defined(foo, True) || defined(foo, False) + return ConfigurationGroups(()).simplify_tags() + + for tag1, tag2 in sorted(combinations(self, 2)): + if tag1 == tag2: + tags = list(self) + tags.remove(tag1) + return ConfigurationGroups(tags).simplify_tags() + + for condition in tag1: + inverted_condition = condition.inverted() + if inverted_condition == tag2: + continue + if inverted_condition in tag2: + tag1_copy = tag1 - {inverted_condition} + tag2_copy = tag2 - {inverted_condition} + + assert isinstance(tag1_copy, Configuration), tag1_copy + assert isinstance(tag2_copy, Configuration), tag2_copy + + if tag1_copy == tag2_copy: + tags = list(self) + tags.remove(tag1) + tags.remove(tag2) + tags.append(tag1_copy) + return ConfigurationGroups(tags).simplify_tags() + + self._simplified = True + return self + + +newline_token = ' ' + +def _run_cython_on_file(configuration, pyx_filename, + py_banner, banner, + output_filename, + counter, lines, + cache=None, + module_name=None): + value = ''.join(lines) + sourcehash = md5(value.encode("utf-8")).hexdigest() + comment = configuration.format_tag() + " hash:" + str(sourcehash) + if os.path.isabs(output_filename): + raise ValueError("output cannot be absolute") + # We can't change the actual name of the pyx file because + # cython generates function names based in that string. + # XXX: Note that this causes cython to generate + # a "corecext" name instead of "gevent.corecext" + tempdir = tempfile.mkdtemp() + #unique_pyx_filename = pyx_filename + #unique_output_filename = output_filename + unique_pyx_filename = os.path.join(tempdir, module_name or pyx_filename) + unique_output_filename = os.path.join(tempdir, output_filename) + + dirname = os.path.dirname(unique_pyx_filename) # output must be in same dir + dbg("Output filename %s", unique_output_filename) + if dirname and not os.path.exists(dirname): + dbg("Making dir %s", dirname) + os.makedirs(dirname) + try: + atomic_write(unique_pyx_filename, py_banner + value) + if WRITE_OUTPUT: + atomic_write(unique_pyx_filename + '.deb', '# %s (%s)\n%s' % (banner, comment, value)) + output = run_cython(unique_pyx_filename, sourcehash, unique_output_filename, banner, comment, + cache) + if WRITE_OUTPUT: + atomic_write(unique_output_filename + '.deb', output) + finally: + if not DEBUG: + shutil.rmtree(tempdir, True) + + return configuration.attach_tags(output), configuration, sourcehash + + +def _run_cython_on_files(pyx_filename, py_banner, banner, output_filename, preprocessed, + module_name=None): + counter = 0 + threads = [] + cache = {} + for configuration, lines in sorted(preprocessed.items()): + counter += 1 + threads.append(Thread(target=_run_cython_on_file, + args=(configuration, pyx_filename, + py_banner, banner, output_filename, + counter, lines, + cache, module_name))) + threads[-1].start() + + for t in threads: + t.join() + + same_results = {} # {sourcehash: tagged_str} + for t in threads: + if not t.value: + log("Thread %s failed.", t) + return + + sourcehash = t.value[2] + tagged_output = t.value[0] + if sourcehash not in same_results: + same_results[sourcehash] = tagged_output + else: + # Nice, something to combine with tags + other_tagged_output = same_results[sourcehash] + assert len(tagged_output) == len(other_tagged_output) + combined_lines = [] + for line_a, line_b in zip(tagged_output, other_tagged_output): + combined_tags = line_a.tags + line_b.tags + combined_lines.append(Str(line_a, combined_tags.simplify_tags())) + same_results[sourcehash] = combined_lines + + # Order them as they were processed for repeatability + ordered_results = [] + for t in threads: + if t.value[0] not in ordered_results: + ordered_results.append(same_results[t.value[2]]) + + return ordered_results + +def process_filename(filename, output_filename=None, module_name=None): + """Process the .ppyx file with preprocessor and compile it with cython. + + The algorithm is as following: + + 1) Identify all possible preprocessor conditions in *filename*. + 2) Run preprocess_filename(*filename*) for each of these conditions. + 3) Process the output of preprocessor with Cython (as many times as + there are different sources generated for different preprocessor + definitions. + 4) Merge the output of different Cython runs using preprocessor conditions + identified in (1). + """ + if output_filename is None: + output_filename = filename.rsplit('.', 1)[0] + '.c' + + pyx_filename = filename.rsplit('.', 1)[0] + '.pyx' + assert pyx_filename != filename + + timestamp = str(datetime.datetime.now().replace(microsecond=0)) + banner = 'Generated by cythonpp.py on %s' % timestamp + py_banner = '# %s\n' % banner + + preprocessed = {} + for configuration in Configuration.get_complete_configurations(filename): + dbg("Processing %s", configuration) + preprocessed[configuration] = preprocess_filename(filename, configuration) + preprocessed[None] = preprocess_filename(filename, None) + + preprocessed = expand_to_match(preprocessed.items()) + reference_pyx = preprocessed.pop(None) + + sources = _run_cython_on_files(pyx_filename, py_banner, banner, output_filename, + preprocessed, module_name) + if sources is None: + log("At least one thread failed to run") + sys.exit(1) + + log('Generating %s ', output_filename) + result = generate_merged(sources) + result_hash = md5(''.join(result.split('\n')[4:]).encode("utf-8")).hexdigest() + atomic_write(output_filename, result) + log('%s bytes of hash %s\n', len(result), result_hash) + + if filename != pyx_filename: + log('Saving %s', pyx_filename) + atomic_write(pyx_filename, py_banner + ''.join(reference_pyx)) + + +def generate_merged(sources): + result = [] + for line in produce_preprocessor(merge(sources)): + result.append(line.replace(newline_token, '\n')) + return ''.join(result) + + +def preprocess_filename(filename, config): + """Process given .ppyx file with preprocessor. + + This does the following + 1) Resolves "#if"s and "#ifdef"s using config + 2) Expands macro definitions (#define) + """ + linecount = 0 + current_name = None + definitions = OrderedDict() + result = [] + including_section = [] + with open(filename) as f: + for line in f: + linecount += 1 + rstripped = line.rstrip() + stripped = rstripped.lstrip() + try: + if current_name is not None: + name = current_name + value = rstripped + if value.endswith('\\'): + value = value[:-1].rstrip() + else: + current_name = None + definitions[name]['lines'].append(value) + else: + if not including_section or including_section[-1]: + m = define_re.match(stripped) + else: + m = None + if m is not None: + name, params, value = m.groups() + value = value.strip() + if value.endswith('\\'): + value = value[:-1].rstrip() + current_name = name + definitions[name] = {'lines': [value]} + if params is None: + trace('Adding definition for %r', name) + else: + definitions[name]['params'] = parse_parameter_names(params) + trace('Adding definition for %r: %s', name, definitions[name]['params']) + else: + m = Condition.match_condition(stripped) + if m is not None and config is not None: + if stripped == '#else': + if not including_section: + raise SyntaxError('unexpected "#else"') + if including_section[-1]: + including_section.pop() + including_section.append(False) + else: + including_section.pop() + including_section.append(True) + elif stripped == '#endif': + if not including_section: + raise SyntaxError('unexpected "#endif"') + including_section.pop() + else: + including_section.append(config.is_condition_true(stripped)) + else: + if including_section and not including_section[-1]: + pass # skip this line because last "#if" was false + else: + if stripped.startswith('#'): + # leave comments as is + result.append(Str_sourceline(line, linecount - 1)) + else: + lines = expand_definitions(line, definitions).split('\n') + if lines and not lines[-1]: + del lines[-1] + lines = [x + '\n' for x in lines] + lines = [Str_sourceline(x, linecount - 1) for x in lines] + result.extend(lines) + except BaseException as ex: + log('%s:%s: %s', filename, linecount, ex) + if isinstance(ex, SyntaxError): + sys.exit(1) + else: + raise + return result + + +def merge(sources): + r"""Merge different sources into a single one. Each line of the result + is a subclass of string that maintains the information for each configuration + it should appear in the result. + + >>> src1 = attach_tags('hello\nworld\n', set([('defined(hello)', True), ('defined(world)', True)])) + >>> src2 = attach_tags('goodbye\nworld\n', set([('defined(hello)', False), ('defined(world)', True)])) + >>> src3 = attach_tags('hello\neveryone\n', set([('defined(hello)', True), ('defined(world)', False)])) + >>> src4 = attach_tags('goodbye\neveryone\n', set([('defined(hello)', False), ('defined(world)', False)])) + >>> from pprint import pprint + >>> pprint(merge([src1, src2, src3, src4])) + [Str('hello\n', [set([('defined(hello)', True)])]), + Str('goodbye\n', [set([('defined(hello)', False)])]), + Str('world\n', [set([('defined(world)', True)])]), + Str('everyone\n', [set([('defined(world)', False)])])] + """ + sources = list(sources) # own copy + dbg("Merging %s", len(sources)) + if len(sources) <= 1: + return [Str(str(x), x.tags.simplify_tags()) for x in sources[0]] + + if not DEBUG: + pool = multiprocessing.Pool() + else: + class SerialPool(object): + def imap(self, func, arg_list): + return [func(*args) for args in arg_list] + + def apply(self, func, args): + return func(*args) + pool = SerialPool() + + groups = [] + + while len(sources) >= 2: + one, two = sources.pop(), sources.pop() + groups.append((one, two)) + + dbg("Merge groups %s", len(groups)) + # len sources == 0 or 1 + for merged in pool.imap(_merge, groups): + dbg("Completed a merge in %s", os.getpid()) + sources.append(merged) + # len sources == 1 or 2 + + if len(sources) == 2: + one, two = sources.pop(), sources.pop() + sources.append(pool.apply(_merge, (one, two))) + # len sources == 1 + + # len sources should now be 1 + dbg("Now merging %s", len(sources)) + return merge(sources) + + +def _merge(*args): + if isinstance(args[0], tuple): + a, b = args[0] + else: + a, b = args + return list(_imerge(a, b)) + +def _imerge(a, b): + # caching the tags speeds up serialization and future merges + tag_cache = {} + for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, a, b).get_opcodes(): + if tag == 'equal': + for line_a, line_b in zip(a[i1:i2], b[j1:j2]): + # tags is a tuple of frozensets + line_a_tags = line_a.tags #getattr(line_a, 'tags', ()) + line_b_tags = line_b.tags #getattr(line_b, 'tags', ()) + key = (line_a_tags, line_b_tags) + tags = tag_cache.setdefault(key, line_a_tags + line_b_tags) + assert isinstance(tags, ConfigurationGroups) + yield Str(line_a, tags) + else: + for line in a[i1:i2]: + yield line + for line in b[j1:j2]: + yield line + + +def expand_to_match(items): + """Insert empty lines so that all sources has matching line numbers for the same code""" + cfg2newlines = {} # maps configuration -> list + for configuration, lines in items: + cfg2newlines[configuration] = [] + + maxguard = 2 ** 30 + while True: + minimalsourceline = maxguard + for configuration, lines in items: + if lines: + minimalsourceline = min(minimalsourceline, lines[0].sourceline) + if minimalsourceline == maxguard: + break + + for configuration, lines in items: + if lines and lines[0].sourceline <= minimalsourceline: + cfg2newlines[configuration].append(lines[0]) + del lines[0] + + number_of_lines = max(len(x) for x in cfg2newlines.values()) + + for newlines in cfg2newlines.values(): + add = (number_of_lines - len(newlines)) + newlines.extend(['\n'] * add) + + return cfg2newlines + + +def produce_preprocessor(iterable): + + if TRACE: + current_line = [0] + + def wrap(line): + current_line[0] += 1 + dbg('%5d: %s', current_line[0], repr(str(line))[1:-1]) + return line + else: + def wrap(line): + return line + + state = None + for line in iterable: + key = line.tags# or None + + if key == state: + yield wrap(line) + else: + if key.exact_reverse(state): + yield wrap('#else /* %s */\n' % state.format_tags()) + else: + if state: + yield wrap('#endif /* %s */\n' % state.format_tags()) + if key: + yield wrap('#if %s\n' % key.format_tags()) + yield wrap(line) + state = key + if state: + yield wrap('#endif /* %s */\n' % state.format_tags()) + +class Str(str): + """This is a string subclass that has a set of tags attached to it. + + Used for merging the outputs. + """ + + def __new__(cls, string, tags): + if not isinstance(string, str): + raise TypeError('string must be str: %s' % (type(string), )) + if not isinstance(tags, Configuration) and not isinstance(tags, ConfigurationGroups): + raise TypeError("Must be tags or tag groups: %r" % (tags,)) + if isinstance(tags, Configuration): + tags = ConfigurationGroups((tags,)) + + self = str.__new__(cls, string) + self.tags = tags + return self + + def __getnewargs__(self): + return str(self), self.tags + + def __repr__(self): + return '%s(%s, %r)' % (self.__class__.__name__, str.__repr__(self), self.tags) + + def __add__(self, other): + if not isinstance(other, str): + raise TypeError + return self.__class__(str.__add__(self, other), self.tags) + + def __radd__(self, other): + if not isinstance(other, str): + raise TypeError + return self.__class__(str.__add__(other, self), self.tags) + + methods = ['__getslice__', '__getitem__', '__mul__', '__rmod__', '__rmul__', + 'join', 'replace', 'upper', 'lower'] + + for method in methods: + do_exec('''def %s(self, *args): + return self.__class__(str.%s(self, *args), self.tags)''' % (method, method), locals()) + + + + +def parse_parameter_names(x): + assert x.startswith('(') and x.endswith(')'), repr(x) + x = x[1:-1] + result = [] + for param in x.split(','): + param = param.strip() + if not param_name_re.match(param): + raise SyntaxError('Invalid parameter name: %r' % param) + result.append(param) + return result + + +def parse_parameter_values(x): + assert x.startswith('(') and x.endswith(')'), repr(x) + x = x[1:-1] + result = [] + for param in x.split(','): + result.append(param.strip()) + return result + + +def expand_definitions(code, definitions): + if not definitions: + return code + keys = list(definitions.keys()) + keys.sort(key=lambda x: (-len(x), x)) + keys = '|'.join(keys) + + # This regex defines a macro invocation + re_macro = re.compile(r'(^|##|[^\w])(%s)(\([^)]+\)|$|##|[^w])' % keys) + + def repl(m): + token = m.group(2) + definition = definitions[token] + + params = definition.get('params', []) + + if params: + arguments = m.group(3) + if arguments.startswith('(') and arguments.endswith(')'): + arguments = parse_parameter_values(arguments) + else: + arguments = None + if arguments and len(params) == len(arguments): + local_definitions = {} + trace('Macro %r params=%r arguments=%r source=%r', token, params, arguments, m.groups()) + for key, value in zip(params, arguments): + trace('Adding argument %r=%r', key, value) + local_definitions[key] = {'lines': [value]} + result = expand_definitions('\n'.join(definition['lines']), local_definitions) + else: + msg = 'Invalid number of arguments for macro %s: expected %s, got %s' + msg = msg % (token, len(params), len(arguments or [])) + raise SyntaxError(msg) + else: + result = '\n'.join(definition['lines']) + if m.group(3) != '##': + result += m.group(3) + if m.group(1) != '##': + result = m.group(1) + result + trace('Replace %r with %r', m.group(0), result) + return result + + for _ in range(20000): + newcode, count = re_macro.subn(repl, code, count=1) + if code == newcode: + if count > 0: + raise SyntaxError('Infinite recursion') + return newcode + code = newcode + raise SyntaxError('Too many substitutions or internal error.') + + +class Str_sourceline(str): + + def __new__(cls, source, sourceline): + self = str.__new__(cls, source) + self.sourceline = sourceline + return self + + def __getnewargs__(self): + return str(self), self.sourceline + +def atomic_write(filename, data): + dirname = os.path.dirname(os.path.abspath(filename)) + tmpfd, tmpname = tempfile.mkstemp(dir=dirname, text=True) + with os.fdopen(tmpfd, 'w') as f: + f.write(data) + f.flush() + os.fsync(f.fileno()) + + if os.path.exists(filename): + os.unlink(filename) + + dbg("Renaming %s to %s", tmpname, filename) + try: + os.rename(tmpname, filename) + except: + log("Failed to rename '%s' to '%s", tmpname, filename) + raise + dbg('Wrote %s bytes to %s', len(data), filename) + + +def run_cython(filename, sourcehash, output_filename, banner, comment, cache=None): + dbg("Cython output to %s hash %s", output_filename, sourcehash) + result = cache.get(sourcehash) if cache is not None else None + # Use an array for the argument so that filename arguments are properly + # quoted according to local convention + command = [CYTHON, '-o', output_filename, + '-I', os.path.join('src', 'gevent', 'libev'), + '-I', os.path.join('src', 'gevent'), # python.pxd, shared with c-ares + filename] + if result is not None: + log('Reusing %s # %s', command, comment) + return result + system(command, comment) + result = postprocess_cython_output(output_filename, banner) + if cache is not None: + cache[sourcehash] = result + return result + + +def system(command, comment): + command_str = ' '.join(command) + log('Running %s # %s', command_str, comment) + try: + subprocess.check_call(command) + dbg('\tDone running %s # %s', command_str, comment) + except (subprocess.CalledProcessError, OSError): + # Python 2 can raise OSError: No such file or directory + # debugging code + log("Path: %s", os.getenv("PATH")) + bin_dir = os.path.dirname(sys.executable) + bin_files = os.listdir(bin_dir) + bin_files.sort() + log("Bin: %s files: %s", bin_dir, ' '.join(bin_files)) + raise + + +def postprocess_cython_output(filename, banner): + # this does a few things: + # 1) converts multiline C-style (/**/) comments with a single line comment by + # replacing \n with newline_token + # 2) adds our header + # 3) remove timestamp in cython's header so that different timestamps do not + # confuse merger + result = ['/* %s */\n' % (banner)] + + with open(filename) as finput: + firstline = finput.readline() + + m = cython_header_re.match(firstline.strip()) + if m: + result.append('/* %s */' % m.group(1)) + else: + result.append(firstline) + + in_comment = False + for line in finput: + + if line.endswith('\n'): + line = line[:-1].rstrip() + '\n' + + if in_comment: + if '*/' in line: + in_comment = False + result.append(line) + else: + result.append(line.replace('\n', newline_token)) + else: + if line.lstrip().startswith('/* ') and '*/' not in line: + line = line.lstrip() # cython adds space before /* for some reason + line = line.replace('\n', newline_token) + result.append(line) + in_comment = True + else: + result.append(line) + return ''.join(result) + +def log(message, *args): + try: + string = message % args + except Exception: + try: + prefix = 'Traceback (most recent call last):\n' + lines = traceback.format_stack()[:-1] + error_lines = traceback.format_exc().replace(prefix, '') + last_length = len(lines[-1].strip().rsplit(' ', 1)[-1]) + last_length = min(80, last_length) + last_length = max(5, last_length) + msg = '%s%s %s\n%s' % (prefix, ''.join(lines), '^' * last_length, error_lines) + sys.stderr.write(msg) + except Exception: + traceback.print_exc() + try: + message = '%r %% %r\n\n' % (message, args) + except Exception: + pass + try: + sys.stderr.write(message) + except Exception: + traceback.print_exc() + else: + print(string, file=sys.stderr) + +if DEBUG: + dbg = log +else: + def dbg(*_): + return +if TRACE: + trace = log +else: + def trace(*_): + return + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--debug', action='store_true') + parser.add_argument('--list', action='store_true', help='Show the list of different conditions') + parser.add_argument('--list-cond', action='store_true') + parser.add_argument('--ignore-cond', action='store_true', help='Ignore conditional directives (only expand definitions)') + parser.add_argument('--write-intermediate', action='store_true', help='Save intermediate files produced by preprocessor and Cython') + parser.add_argument('-o', '--output-file', help='Specify name of generated C file') + # TODO: Derive the module name automatically from the input filename relative to the base + # dir. + parser.add_argument('--module-name', help="specify name of .pyx module") + parser.add_argument("input") + options = parser.parse_args() + filename = options.input + + if options.debug: + global DEBUG + DEBUG = True + + if options.write_intermediate: + global WRITE_OUTPUT + WRITE_OUTPUT = True + + run = True + + if options.list_cond: + run = False + for x in Configuration.get_configurations(filename): + print("* ", x) + + if options.list: + run = False + for x in Configuration.get_complete_configurations(filename): + print("* ", x) + + if options.ignore_cond: + run = False + + class FakeConfig(object): + def is_condition_true(*args): + return False + + sys.stdout.write(preprocess_filename(filename, FakeConfig())) + + if run: + process_filename(filename, options.output_file, options.module_name) + + +if __name__ == '__main__': + main() diff --git a/util/makedeb.sh b/util/makedeb.sh new file mode 100755 index 0000000..0329c9f --- /dev/null +++ b/util/makedeb.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +CWD=`pwd` +rm -fr /tmp/build_gevent_deb +set -x +mkdir /tmp/build_gevent_deb +#util/makedist.py --dest /tmp/build_gevent_deb/gevent.tar.gz --version dev +cd /tmp/build_gevent_deb +tar -xf $CWD/dist/gevent-1.0.tar.gz +fpm --no-python-dependencies -s python -t deb gevent*/setup.py +mkdir -p $CWD/build +mv *.deb $CWD/build/ diff --git a/util/makedist.py b/util/makedist.py new file mode 100755 index 0000000..abb9e00 --- /dev/null +++ b/util/makedist.py @@ -0,0 +1,91 @@ +#!/usr/bin/python +# Copyright (C) 2012 Denis Bilenko (http://denisbilenko.com) +""" +Create a source distribution of gevent. + +Does the following: + + - Clones the repo into a temporary location. + - Run set_version.py that will update gevent/__init__.py. + - Run 'python setup.py sdist'. +""" +from __future__ import print_function +import sys +import os +import glob +import argparse +from os.path import exists, join, abspath, basename +from pipes import quote + + +TMPDIR = '/tmp/gevent-make-dist' + + +def system(cmd, noisy=True): + if noisy: + print(cmd) + res = os.system(cmd) + if res: + sys.exit('%r failed with %s' % (cmd, res)) + + +def makedist(*args, **kwargs): + cwd = os.getcwd() + try: + return _makedist(*args, **kwargs) + finally: + os.chdir(cwd) + + +def _makedist(version=None, dest=None): + assert exists('gevent/__init__.py'), 'Where am I?' + basedir = abspath(os.getcwd()) + version = version or 'dev' + set_version_command = 'util/set_version.py --version %s ./gevent/__init__.py' % version + os.chdir('/tmp') + system('rm -fr ' + TMPDIR) + os.mkdir(TMPDIR) + os.chdir(TMPDIR) + + system('git clone %s gevent' % basedir) + + directory = os.listdir('.') + assert len(directory) == 1, directory + + os.chdir(directory[0]) + system('git branch') + system(set_version_command) + system('git diff', noisy=False) + system('python setup.py -q sdist') + + dist_filename = glob.glob('dist/gevent-*.tar.gz') + assert len(dist_filename) == 1, dist_filename + dist_path = abspath(dist_filename[0]) + dist_filename = basename(dist_path) + + if dest: + if os.path.isdir(dest): + dest = join(dest, dist_filename) + else: + if not exists(join(basedir, 'dist')): + os.mkdir(join(basedir, 'dist')) + dest = join(basedir, 'dist', dist_filename) + + copy(dist_path, dest) + return dist_path + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--dest') + parser.add_argument('--version') + options = parser.parse_args() + return makedist(options.version, dest=options.dest) + + +def copy(source, dest): + system('cp -a %s %s' % (quote(source), quote(dest))) + + +if __name__ == '__main__': + main() diff --git a/util/set_version.py b/util/set_version.py new file mode 100755 index 0000000..e724574 --- /dev/null +++ b/util/set_version.py @@ -0,0 +1,157 @@ +#!/usr/bin/python +"""Update __version__, version_info and add __changeset__. + +'dev' in version_info should be replaced with alpha|beta|candidate|final +'dev' in __version__ should be replaced with a|b|rc| +""" + +from __future__ import print_function +import sys +import os +import re +from argparse import ArgumentParser +from distutils.version import LooseVersion + + +version_re = re.compile("^__version__\s*=\s*'([^']+)'", re.M) +version_info_re = re.compile(r"^version_info\s*=\s*([^\n]+)", re.M) +strict_version_re = re.compile(r'^(\d+) \. (\d+) (\. (\d+))? ([ab](\d+))?$', re.VERBOSE) + + +def read(command): + popen = os.popen(command) + data = popen.read() + retcode = popen.close() + if retcode: + sys.exit('Failed (%s) to run %r' % (retcode, command)) + return data.strip() + + +def get_changeset(): + return read('git describe --tags --always --dirty --long') + + +def get_version_info(version): + """ + >>> get_version_info('0.13.6') + (0, 13, 6, 'final', 0) + >>> get_version_info('1.1') + (1, 1, 0, 'final', 0) + >>> get_version_info('1') + (1, 0, 0, 'final', 0) + >>> get_version_info('1.0dev1') + (1, 0, 0, 'dev', 1) + >>> get_version_info('1.0a3') + (1, 0, 0, 'alpha', 3) + >>> get_version_info('1.0rc1') + (1, 0, 0, 'candidate', 1) + """ + + repl = {'a': 'alpha', + 'b': 'beta', + 'rc': 'candidate', + 'dev': 'dev'} + + components = LooseVersion(version).version + result = [] + + for component in components: + if isinstance(component, int): + result.append(component) + else: + while len(result) < 3: + result.append(0) + component = repl[component] + result.append(component) + + while len(result) < 3: + result.append(0) + + if len(result) == 3: + result.append('final') + result.append(0) + + return tuple(result) + + +def modify_version(filename, new_version): + # return (current_contents, modified_contents, is_match) + original_data = open(filename).read() + assert '__changeset__' not in original_data, 'Must revert the old update first' + data = original_data + + if new_version: + new_version_info = get_version_info(new_version) + + def repl_version_info(m): + return 'version_info = %s' % (new_version_info, ) + + data, count = version_info_re.subn(repl_version_info, data) + if not count: + raise AssertionError('version_info not found in %s' % filename) + if count != 1: + raise AssertionError('version_info found more than once in %s' % filename) + + def repl_version(m): + result = m.group(0).replace(m.group(1), new_version or m.group(1)) + result += "\n__changeset__ = '%s'" % get_changeset() + return result + + data, count = version_re.subn(repl_version, data) + if not count: + raise AssertionError('__version__ not found in %s' % filename) + if count != 1: + raise AssertionError('__version__ found more than once in %s' % filename) + + return original_data, data + + +def unlink(path): + try: + os.unlink(path) + except OSError as ex: + if ex.errno == 2: # No such file or directory + return + raise + + +def write(filename, data): + # intentionally breaking links here so that util/makedist.py can use "cp --link" + tmpname = filename + '.tmp.%s' % os.getpid() + f = open(tmpname, 'w') + try: + f.write(data) + f.flush() + os.fsync(f.fileno()) + f.close() + os.rename(tmpname, filename) + except: + unlink(tmpname) + raise + + +def main(): + global options + parser = ArgumentParser() + parser.add_argument('--version', default='dev') + parser.add_argument('--dry-run', action='store_true') + parser.add_argument('filename') + options = parser.parse_args() + version = options.version + if version.lower() == 'dev': + version = '' + if version and strict_version_re.match(version) is None: + sys.stderr.write('WARNING: Not a strict version: %r (bdist_msi will fail)' % version) + original_content, new_content = modify_version(options.filename, version) + if options.dry_run: + tmpname = '/tmp/' + os.path.basename(options.filename) + '.set_version' + write(tmpname, new_content) + if not os.system('diff -u %s %s' % (options.filename, tmpname)): + sys.exit('No differences applied') + else: + write(options.filename, new_content) + print('Updated %s' % options.filename) + + +if __name__ == '__main__': + main() diff --git a/util/wintest.py b/util/wintest.py new file mode 100755 index 0000000..06af104 --- /dev/null +++ b/util/wintest.py @@ -0,0 +1,82 @@ +#!/usr/bin/python -u +""" +Unix utilities must be installed on target machine for this to work: http://unxutils.sourceforge.net/ +""" +import sys +import os +import argparse + + +def system(cmd, exit=True): + sys.stderr.write('+ %s\n' % cmd) + retcode = os.system(cmd) + if retcode: + if exit: + sys.exit('%r failed' % cmd) + return retcode + + +parser = argparse.ArgumentParser() +parser.add_argument('--host') +parser.add_argument('--username', default='Administrator') +parser.add_argument('--source') +parser.add_argument('--dist', action='store_true') +parser.add_argument('--python', default='27') +parser.add_argument('args', nargs='*') +options = parser.parse_args() +args = options.args + + +def prepare(): + source_name = args[1] + tar_name = source_name.rsplit('.', 1)[0] + dir_name = tar_name.rsplit('.', 1)[0] + system('rm -fr %s %s' % (tar_name, dir_name)) + system('gzip -d %s && tar -xf %s' % (source_name, tar_name)) + os.chdir(dir_name) + os.environ.setdefault('VS90COMNTOOLS', 'C:\\Program Files\\Microsoft Visual Studio 10.0\\Common7\Tools\\') + +if args[0:1] == ['test']: + prepare() + system('%s setup.py build' % sys.executable) + os.chdir('greentest') + os.environ['PYTHONPATH'] = '.;..;../..' + system('%s testrunner.py --config ../known_failures.py' % sys.executable) +elif args[0:1] == ['dist']: + prepare() + success = 0 + for command in ['bdist_egg', 'bdist_wininst', 'bdist_msi']: + cmd = sys.executable + ' setup.py ' + command + if not system(cmd, exit=False): + success += 1 + if not success: + sys.exit('bdist_egg bdist_wininst and bdist_msi all failed') +elif not args: + assert options.host + if not options.source: + import makedist + options.source = makedist.makedist() + + options.source_name = os.path.basename(options.source) + options.script_path = os.path.abspath(__file__) + options.script_name = os.path.basename(__file__) + + if options.python.isdigit(): + options.python = 'C:/Python' + options.python + '/python.exe' + + tar_name = options.source_name.rsplit('.', 1)[0] + dir_name = tar_name.rsplit('.', 1)[0] + options.dir_name = dir_name + + system('scp %(source)s %(script_path)s %(username)s@%(host)s:' % options.__dict__) + if options.dist: + system('ssh %(username)s@%(host)s %(python)s -u %(script_name)s dist %(source_name)s' % options.__dict__) + try: + os.mkdir('dist') + except OSError: + pass + system('scp -r %(username)s@%(host)s:%(dir_name)s/dist/ dist' % options.__dict__) + else: + system('ssh %(username)s@%(host)s C:/Python27/python.exe -u %(script_name)s test %(source_name)s' % options.__dict__) +else: + sys.exit('Invalid args: %r' % (args, ))