WASM Python integration milestone - game.py runs in browser

Major milestone for issue #158 (Emscripten/WebAssembly build target):
- Python 3.14 successfully initializes and runs in WASM
- mcrfpy module loads and works correctly
- Game scripts execute with full level generation
- Entities (boulders, rats, cyclops, spawn points) placed correctly

Key changes:
- CMakeLists.txt: Add 2MB stack, Emscripten link options, preload files
- platform.h: Add WASM-specific implementations for executable paths
- HeadlessTypes.h: Make Texture/Font/Sound stubs return success
- CommandLineParser.cpp: Guard filesystem operations for WASM
- McRFPy_API.cpp: Add WASM path configuration, debug output
- game.py: Make 'code' module import optional (not available in WASM)
- wasm_stdlib/: Add minimal Python stdlib for WASM (~4MB)

Build with: emmake make (from build-emscripten/)
Test with: node mcrogueface.js

Next steps:
- Integrate VRSFML for actual WebGL rendering
- Create HTML page to host WASM build
- Test in actual browsers

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
John McCardle 2026-01-31 05:15:11 -05:00
commit 8c3128e29c
222 changed files with 80639 additions and 25 deletions

5
.gitignore vendored
View file

@ -7,9 +7,12 @@ PCbuild
.vs
obj
build
lib
/lib
__pycache__
# WASM stdlib for Emscripten build
!wasm_stdlib/
.cache/
7DRL2025 Release/
CMakeFiles/

View file

@ -208,6 +208,23 @@ if(EMSCRIPTEN)
-sUSE_ZLIB=1
-sUSE_BZIP2=1
-sUSE_SQLITE3=1
-sALLOW_MEMORY_GROWTH=1
-sSTACK_SIZE=2097152
-sEXPORTED_RUNTIME_METHODS=ccall,cwrap
-sASSERTIONS=2
-sSTACK_OVERFLOW_CHECK=2
-fexceptions
-sNO_DISABLE_EXCEPTION_CATCHING
# Preload Python stdlib into virtual filesystem at /lib/python3.14
--preload-file=${CMAKE_SOURCE_DIR}/wasm_stdlib/lib@/lib
# Preload game scripts into /scripts
--preload-file=${CMAKE_SOURCE_DIR}/src/scripts@/scripts
# Preload assets
--preload-file=${CMAKE_SOURCE_DIR}/assets@/assets
)
# Set Python home for the embedded interpreter
target_compile_definitions(mcrogueface PRIVATE
MCRF_WASM_PYTHON_HOME="/lib/python3.14"
)
endif()

View file

@ -1,6 +1,54 @@
#ifndef __PLATFORM
#define __PLATFORM
#define __PLATFORM_SET_PYTHON_SEARCH_PATHS 1
#ifdef __EMSCRIPTEN__
// WASM/Emscripten platform - no /proc filesystem, limited std::filesystem support
std::wstring executable_path()
{
// In WASM, the executable is at the root of the virtual filesystem
return L"/";
}
std::wstring executable_filename()
{
// In WASM, we use a fixed executable name
return L"/mcrogueface";
}
std::wstring working_path()
{
// In WASM, working directory is root of virtual filesystem
return L"/";
}
std::string narrow_string(std::wstring convertme)
{
// Simple conversion for ASCII/UTF-8 compatible strings
std::string result;
result.reserve(convertme.size());
for (wchar_t wc : convertme) {
if (wc < 128) {
result.push_back(static_cast<char>(wc));
} else {
// For non-ASCII, use a simple UTF-8 encoding
if (wc < 0x800) {
result.push_back(static_cast<char>(0xC0 | (wc >> 6)));
result.push_back(static_cast<char>(0x80 | (wc & 0x3F)));
} else {
result.push_back(static_cast<char>(0xE0 | (wc >> 12)));
result.push_back(static_cast<char>(0x80 | ((wc >> 6) & 0x3F)));
result.push_back(static_cast<char>(0x80 | (wc & 0x3F)));
}
}
}
return result;
}
#else
// Native Linux platform
std::wstring executable_path()
{
/*
@ -12,7 +60,7 @@ std::wstring executable_path()
return exec_path.wstring();
//size_t path_index = exec_path.find_last_of('/');
//return exec_path.substr(0, path_index);
}
std::wstring executable_filename()
@ -37,4 +85,6 @@ std::string narrow_string(std::wstring convertme)
return converter.to_bytes(convertme);
}
#endif
#endif // __EMSCRIPTEN__
#endif // __PLATFORM

View file

@ -11,11 +11,13 @@ CommandLineParser::ParseResult CommandLineParser::parse(McRogueFaceConfig& confi
current_arg = 1; // Reset for each parse
// Detect if running as Python interpreter
#ifndef __EMSCRIPTEN__
std::filesystem::path exec_name = std::filesystem::path(argv[0]).filename();
if (exec_name.string().find("python") == 0) {
config.headless = true;
config.python_mode = true;
}
#endif
while (current_arg < argc) {
std::string arg = argv[current_arg];

View file

@ -698,32 +698,64 @@ PyObject* PyInit_mcrfpy()
// init_python - configure interpreter details here
PyStatus init_python(const char *program_name)
{
std::cerr << "[DEBUG] api_init: starting" << std::endl;
std::cerr.flush();
PyStatus status;
//**preconfig to establish locale**
//**preconfig to establish locale**
PyPreConfig preconfig;
PyPreConfig_InitIsolatedConfig(&preconfig);
preconfig.utf8_mode = 1;
std::cerr << "[DEBUG] api_init: Py_PreInitialize" << std::endl;
std::cerr.flush();
status = Py_PreInitialize(&preconfig);
if (PyStatus_Exception(status)) {
Py_ExitStatusException(status);
std::cerr << "[DEBUG] api_init: PreInit failed" << std::endl;
Py_ExitStatusException(status);
}
std::cerr << "[DEBUG] api_init: PyConfig setup" << std::endl;
std::cerr.flush();
PyConfig config;
PyConfig_InitIsolatedConfig(&config);
config.dev_mode = 0;
config.dev_mode = 0;
// Configure UTF-8 for stdio
PyConfig_SetString(&config, &config.stdio_encoding, L"UTF-8");
PyConfig_SetString(&config, &config.stdio_errors, L"surrogateescape");
config.configure_c_stdio = 1;
#ifdef __EMSCRIPTEN__
std::cerr << "[DEBUG] api_init: WASM path config" << std::endl;
std::cerr.flush();
// WASM: Use absolute paths in virtual filesystem
PyConfig_SetString(&config, &config.executable, L"/mcrogueface");
PyConfig_SetString(&config, &config.home, L"/lib/python3.14");
status = PyConfig_SetBytesString(&config, &config.program_name, "mcrogueface");
// Set up module search paths for WASM
config.module_search_paths_set = 1;
const wchar_t* wasm_paths[] = {
L"/scripts",
L"/lib/python3.14"
};
for (auto s : wasm_paths) {
status = PyWideStringList_Append(&config.module_search_paths, s);
if (PyStatus_Exception(status)) {
continue;
}
}
#else
// Set sys.executable to the McRogueFace binary path
auto exe_filename = executable_filename();
PyConfig_SetString(&config, &config.executable, exe_filename.c_str());
PyConfig_SetBytesString(&config, &config.home,
PyConfig_SetBytesString(&config, &config.home,
narrow_string(executable_path() + L"/lib/Python").c_str());
status = PyConfig_SetBytesString(&config, &config.program_name,
@ -770,6 +802,7 @@ PyStatus init_python(const char *program_name)
}
}
#endif
#endif // __EMSCRIPTEN__
status = Py_InitializeFromConfig(&config);
@ -780,11 +813,18 @@ PyStatus init_python(const char *program_name)
PyStatus McRFPy_API::init_python_with_config(const McRogueFaceConfig& config)
{
std::cerr << "[DEBUG] init_python_with_config: starting" << std::endl;
std::cerr.flush();
// If Python is already initialized, just return success
if (Py_IsInitialized()) {
std::cerr << "[DEBUG] init_python_with_config: already initialized" << std::endl;
return PyStatus_Ok();
}
std::cerr << "[DEBUG] init_python_with_config: PyConfig_InitIsolatedConfig" << std::endl;
std::cerr.flush();
PyStatus status;
PyConfig pyconfig;
PyConfig_InitIsolatedConfig(&pyconfig);
@ -849,7 +889,10 @@ PyStatus McRFPy_API::init_python_with_config(const McRogueFaceConfig& config)
return status;
}
// Set Python home to our bundled Python
#ifndef __EMSCRIPTEN__
// Check if we're in a virtual environment (symlinked into a venv)
// Skip for WASM builds - no filesystem access like this
auto exe_wpath = executable_filename();
auto exe_path_fs = std::filesystem::path(exe_wpath);
auto exe_dir = exe_path_fs.parent_path();
@ -880,8 +923,24 @@ PyStatus McRFPy_API::init_python_with_config(const McRogueFaceConfig& config)
pyconfig.module_search_paths_set = 1;
}
}
#endif // !__EMSCRIPTEN__
#ifdef __EMSCRIPTEN__
// WASM: Use absolute paths in virtual filesystem
PyConfig_SetString(&pyconfig, &pyconfig.home, L"/lib/python3.14");
// Set Python home to our bundled Python
// Set up module search paths for WASM
pyconfig.module_search_paths_set = 1;
const wchar_t* wasm_paths[] = {
L"/scripts",
L"/lib/python3.14"
};
for (auto s : wasm_paths) {
status = PyWideStringList_Append(&pyconfig.module_search_paths, s);
if (PyStatus_Exception(status)) {
continue;
}
}
#else
auto python_home = executable_path() + L"/lib/Python";
PyConfig_SetString(&pyconfig, &pyconfig.home, python_home.c_str());
@ -907,6 +966,7 @@ PyStatus McRFPy_API::init_python_with_config(const McRogueFaceConfig& config)
}
}
#endif
#endif // __EMSCRIPTEN__
// Register mcrfpy module before initialization
PyImport_AppendInittab("mcrfpy", &PyInit_mcrfpy);
@ -988,26 +1048,39 @@ void McRFPy_API::api_init(const McRogueFaceConfig& config) {
void McRFPy_API::executeScript(std::string filename)
{
std::string script_path_str;
#ifdef __EMSCRIPTEN__
// WASM: Scripts are at /scripts/ in virtual filesystem
if (filename.find('/') == std::string::npos) {
// Simple filename - look in /scripts/
script_path_str = "/scripts/" + filename;
} else {
script_path_str = filename;
}
#else
std::filesystem::path script_path(filename);
// If the path is relative and the file doesn't exist, try resolving it relative to the executable
if (script_path.is_relative() && !std::filesystem::exists(script_path)) {
// Get the directory where the executable is located using platform-specific function
std::wstring exe_dir_w = executable_path();
std::filesystem::path exe_dir(exe_dir_w);
// Try the script path relative to the executable directory
std::filesystem::path resolved_path = exe_dir / script_path;
if (std::filesystem::exists(resolved_path)) {
script_path = resolved_path;
}
}
script_path_str = script_path.string();
#endif
// Use std::ifstream + PyRun_SimpleString instead of PyRun_SimpleFile
// PyRun_SimpleFile has compatibility issues with MinGW-compiled code
std::ifstream file(script_path);
std::ifstream file(script_path_str);
if (!file.is_open()) {
std::cout << "Failed to open script: " << script_path.string() << std::endl;
std::cout << "Failed to open script: " << script_path_str << std::endl;
return;
}
@ -1018,7 +1091,7 @@ void McRFPy_API::executeScript(std::string filename)
// Set __file__ before execution
PyObject* main_module = PyImport_AddModule("__main__");
PyObject* main_dict = PyModule_GetDict(main_module);
PyObject* py_filename = PyUnicode_FromString(script_path.string().c_str());
PyObject* py_filename = PyUnicode_FromString(script_path_str.c_str());
PyDict_SetItemString(main_dict, "__file__", py_filename);
Py_DECREF(py_filename);

View file

@ -493,8 +493,16 @@ class Texture {
public:
Texture() = default;
bool create(unsigned int width, unsigned int height) { size_ = Vector2u(width, height); return true; }
bool loadFromFile(const std::string& filename) { return false; }
bool loadFromMemory(const void* data, size_t size) { return false; }
// In headless mode, pretend texture loading succeeded with dummy dimensions
// This allows game scripts to run without actual graphics
bool loadFromFile(const std::string& filename) {
size_ = Vector2u(256, 256); // Default size for headless textures
return true;
}
bool loadFromMemory(const void* data, size_t size) {
size_ = Vector2u(256, 256);
return true;
}
Vector2u getSize() const { return size_; }
void setSmooth(bool smooth) {}
bool isSmooth() const { return false; }
@ -545,8 +553,9 @@ public:
};
Font() = default;
bool loadFromFile(const std::string& filename) { return false; }
bool loadFromMemory(const void* data, size_t sizeInBytes) { return false; }
// In headless mode, pretend font loading succeeded
bool loadFromFile(const std::string& filename) { return true; }
bool loadFromMemory(const void* data, size_t sizeInBytes) { return true; }
const Info& getInfo() const { static Info info; return info; }
};
@ -723,8 +732,9 @@ public:
class SoundBuffer {
public:
SoundBuffer() = default;
bool loadFromFile(const std::string& filename) { return false; }
bool loadFromMemory(const void* data, size_t sizeInBytes) { return false; }
// In headless mode, pretend sound loading succeeded
bool loadFromFile(const std::string& filename) { return true; }
bool loadFromMemory(const void* data, size_t sizeInBytes) { return true; }
Time getDuration() const { return Time(); }
};
@ -752,7 +762,8 @@ public:
enum Status { Stopped, Paused, Playing };
Music() = default;
bool openFromFile(const std::string& filename) { return false; }
// In headless mode, pretend music loading succeeded
bool openFromFile(const std::string& filename) { return true; }
void play() {}
void pause() {}

View file

@ -1,5 +1,8 @@
import mcrfpy
import code
try:
import code
except ImportError:
code = None # Interactive console not available in WASM
#t = mcrfpy.Texture("assets/kenney_tinydungeon.png", 16, 16) # 12, 11)
t = mcrfpy.Texture("assets/kenney_TD_MR_IP.png", 16, 16) # 12, 11)
@ -326,7 +329,8 @@ class Crypt:
d = None
if state == "end": return
elif key == "Grave":
code.InteractiveConsole(locals=globals()).interact()
if code: # Only available in native builds, not WASM
code.InteractiveConsole(locals=globals()).interact()
return
elif key == "Z":
self.player.do_zap()

View file

@ -0,0 +1,147 @@
"""Record of phased-in incompatible language changes.
Each line is of the form:
FeatureName = "_Feature(" OptionalRelease "," MandatoryRelease ","
CompilerFlag ")"
where, normally, OptionalRelease < MandatoryRelease, and both are 5-tuples
of the same form as sys.version_info:
(PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int
PY_MINOR_VERSION, # the 1; an int
PY_MICRO_VERSION, # the 0; an int
PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string
PY_RELEASE_SERIAL # the 3; an int
)
OptionalRelease records the first release in which
from __future__ import FeatureName
was accepted.
In the case of MandatoryReleases that have not yet occurred,
MandatoryRelease predicts the release in which the feature will become part
of the language.
Else MandatoryRelease records when the feature became part of the language;
in releases at or after that, modules no longer need
from __future__ import FeatureName
to use the feature in question, but may continue to use such imports.
MandatoryRelease may also be None, meaning that a planned feature got
dropped or that the release version is undetermined.
Instances of class _Feature have two corresponding methods,
.getOptionalRelease() and .getMandatoryRelease().
CompilerFlag is the (bitfield) flag that should be passed in the fourth
argument to the builtin function compile() to enable the feature in
dynamically compiled code. This flag is stored in the .compiler_flag
attribute on _Future instances. These values must match the appropriate
#defines of CO_xxx flags in Include/cpython/compile.h.
No feature line is ever to be deleted from this file.
"""
all_feature_names = [
"nested_scopes",
"generators",
"division",
"absolute_import",
"with_statement",
"print_function",
"unicode_literals",
"barry_as_FLUFL",
"generator_stop",
"annotations",
]
__all__ = ["all_feature_names"] + all_feature_names
# The CO_xxx symbols are defined here under the same names defined in
# code.h and used by compile.h, so that an editor search will find them here.
# However, they're not exported in __all__, because they don't really belong to
# this module.
CO_NESTED = 0x0010 # nested_scopes
CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000)
CO_FUTURE_DIVISION = 0x20000 # division
CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default
CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement
CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function
CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals
CO_FUTURE_BARRY_AS_BDFL = 0x400000
CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators
CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime
class _Feature:
def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
self.optional = optionalRelease
self.mandatory = mandatoryRelease
self.compiler_flag = compiler_flag
def getOptionalRelease(self):
"""Return first release in which this feature was recognized.
This is a 5-tuple, of the same form as sys.version_info.
"""
return self.optional
def getMandatoryRelease(self):
"""Return release in which this feature will become mandatory.
This is a 5-tuple, of the same form as sys.version_info, or, if
the feature was dropped, or the release date is undetermined, is None.
"""
return self.mandatory
def __repr__(self):
return "_Feature" + repr((self.optional,
self.mandatory,
self.compiler_flag))
nested_scopes = _Feature((2, 1, 0, "beta", 1),
(2, 2, 0, "alpha", 0),
CO_NESTED)
generators = _Feature((2, 2, 0, "alpha", 1),
(2, 3, 0, "final", 0),
CO_GENERATOR_ALLOWED)
division = _Feature((2, 2, 0, "alpha", 2),
(3, 0, 0, "alpha", 0),
CO_FUTURE_DIVISION)
absolute_import = _Feature((2, 5, 0, "alpha", 1),
(3, 0, 0, "alpha", 0),
CO_FUTURE_ABSOLUTE_IMPORT)
with_statement = _Feature((2, 5, 0, "alpha", 1),
(2, 6, 0, "alpha", 0),
CO_FUTURE_WITH_STATEMENT)
print_function = _Feature((2, 6, 0, "alpha", 2),
(3, 0, 0, "alpha", 0),
CO_FUTURE_PRINT_FUNCTION)
unicode_literals = _Feature((2, 6, 0, "alpha", 2),
(3, 0, 0, "alpha", 0),
CO_FUTURE_UNICODE_LITERALS)
barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
(4, 0, 0, "alpha", 0),
CO_FUTURE_BARRY_AS_BDFL)
generator_stop = _Feature((3, 5, 0, "beta", 1),
(3, 7, 0, "alpha", 0),
CO_FUTURE_GENERATOR_STOP)
annotations = _Feature((3, 7, 0, "beta", 1),
None,
CO_FUTURE_ANNOTATIONS)

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,371 @@
# This file is generated by Tools/cases_generator/py_metadata_generator.py
# from:
# Python/bytecodes.c
# Do not edit!
_specializations = {
"RESUME": [
"RESUME_CHECK",
],
"LOAD_CONST": [
"LOAD_CONST_MORTAL",
"LOAD_CONST_IMMORTAL",
],
"TO_BOOL": [
"TO_BOOL_ALWAYS_TRUE",
"TO_BOOL_BOOL",
"TO_BOOL_INT",
"TO_BOOL_LIST",
"TO_BOOL_NONE",
"TO_BOOL_STR",
],
"BINARY_OP": [
"BINARY_OP_MULTIPLY_INT",
"BINARY_OP_ADD_INT",
"BINARY_OP_SUBTRACT_INT",
"BINARY_OP_MULTIPLY_FLOAT",
"BINARY_OP_ADD_FLOAT",
"BINARY_OP_SUBTRACT_FLOAT",
"BINARY_OP_ADD_UNICODE",
"BINARY_OP_SUBSCR_LIST_INT",
"BINARY_OP_SUBSCR_LIST_SLICE",
"BINARY_OP_SUBSCR_TUPLE_INT",
"BINARY_OP_SUBSCR_STR_INT",
"BINARY_OP_SUBSCR_DICT",
"BINARY_OP_SUBSCR_GETITEM",
"BINARY_OP_EXTEND",
"BINARY_OP_INPLACE_ADD_UNICODE",
],
"STORE_SUBSCR": [
"STORE_SUBSCR_DICT",
"STORE_SUBSCR_LIST_INT",
],
"SEND": [
"SEND_GEN",
],
"UNPACK_SEQUENCE": [
"UNPACK_SEQUENCE_TWO_TUPLE",
"UNPACK_SEQUENCE_TUPLE",
"UNPACK_SEQUENCE_LIST",
],
"STORE_ATTR": [
"STORE_ATTR_INSTANCE_VALUE",
"STORE_ATTR_SLOT",
"STORE_ATTR_WITH_HINT",
],
"LOAD_GLOBAL": [
"LOAD_GLOBAL_MODULE",
"LOAD_GLOBAL_BUILTIN",
],
"LOAD_SUPER_ATTR": [
"LOAD_SUPER_ATTR_ATTR",
"LOAD_SUPER_ATTR_METHOD",
],
"LOAD_ATTR": [
"LOAD_ATTR_INSTANCE_VALUE",
"LOAD_ATTR_MODULE",
"LOAD_ATTR_WITH_HINT",
"LOAD_ATTR_SLOT",
"LOAD_ATTR_CLASS",
"LOAD_ATTR_CLASS_WITH_METACLASS_CHECK",
"LOAD_ATTR_PROPERTY",
"LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
"LOAD_ATTR_METHOD_WITH_VALUES",
"LOAD_ATTR_METHOD_NO_DICT",
"LOAD_ATTR_METHOD_LAZY_DICT",
"LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES",
"LOAD_ATTR_NONDESCRIPTOR_NO_DICT",
],
"COMPARE_OP": [
"COMPARE_OP_FLOAT",
"COMPARE_OP_INT",
"COMPARE_OP_STR",
],
"CONTAINS_OP": [
"CONTAINS_OP_SET",
"CONTAINS_OP_DICT",
],
"JUMP_BACKWARD": [
"JUMP_BACKWARD_NO_JIT",
"JUMP_BACKWARD_JIT",
],
"FOR_ITER": [
"FOR_ITER_LIST",
"FOR_ITER_TUPLE",
"FOR_ITER_RANGE",
"FOR_ITER_GEN",
],
"CALL": [
"CALL_BOUND_METHOD_EXACT_ARGS",
"CALL_PY_EXACT_ARGS",
"CALL_TYPE_1",
"CALL_STR_1",
"CALL_TUPLE_1",
"CALL_BUILTIN_CLASS",
"CALL_BUILTIN_O",
"CALL_BUILTIN_FAST",
"CALL_BUILTIN_FAST_WITH_KEYWORDS",
"CALL_LEN",
"CALL_ISINSTANCE",
"CALL_LIST_APPEND",
"CALL_METHOD_DESCRIPTOR_O",
"CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS",
"CALL_METHOD_DESCRIPTOR_NOARGS",
"CALL_METHOD_DESCRIPTOR_FAST",
"CALL_ALLOC_AND_ENTER_INIT",
"CALL_PY_GENERAL",
"CALL_BOUND_METHOD_GENERAL",
"CALL_NON_PY_GENERAL",
],
"CALL_KW": [
"CALL_KW_BOUND_METHOD",
"CALL_KW_PY",
"CALL_KW_NON_PY",
],
}
_specialized_opmap = {
'BINARY_OP_ADD_FLOAT': 129,
'BINARY_OP_ADD_INT': 130,
'BINARY_OP_ADD_UNICODE': 131,
'BINARY_OP_EXTEND': 132,
'BINARY_OP_INPLACE_ADD_UNICODE': 3,
'BINARY_OP_MULTIPLY_FLOAT': 133,
'BINARY_OP_MULTIPLY_INT': 134,
'BINARY_OP_SUBSCR_DICT': 135,
'BINARY_OP_SUBSCR_GETITEM': 136,
'BINARY_OP_SUBSCR_LIST_INT': 137,
'BINARY_OP_SUBSCR_LIST_SLICE': 138,
'BINARY_OP_SUBSCR_STR_INT': 139,
'BINARY_OP_SUBSCR_TUPLE_INT': 140,
'BINARY_OP_SUBTRACT_FLOAT': 141,
'BINARY_OP_SUBTRACT_INT': 142,
'CALL_ALLOC_AND_ENTER_INIT': 143,
'CALL_BOUND_METHOD_EXACT_ARGS': 144,
'CALL_BOUND_METHOD_GENERAL': 145,
'CALL_BUILTIN_CLASS': 146,
'CALL_BUILTIN_FAST': 147,
'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148,
'CALL_BUILTIN_O': 149,
'CALL_ISINSTANCE': 150,
'CALL_KW_BOUND_METHOD': 151,
'CALL_KW_NON_PY': 152,
'CALL_KW_PY': 153,
'CALL_LEN': 154,
'CALL_LIST_APPEND': 155,
'CALL_METHOD_DESCRIPTOR_FAST': 156,
'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157,
'CALL_METHOD_DESCRIPTOR_NOARGS': 158,
'CALL_METHOD_DESCRIPTOR_O': 159,
'CALL_NON_PY_GENERAL': 160,
'CALL_PY_EXACT_ARGS': 161,
'CALL_PY_GENERAL': 162,
'CALL_STR_1': 163,
'CALL_TUPLE_1': 164,
'CALL_TYPE_1': 165,
'COMPARE_OP_FLOAT': 166,
'COMPARE_OP_INT': 167,
'COMPARE_OP_STR': 168,
'CONTAINS_OP_DICT': 169,
'CONTAINS_OP_SET': 170,
'FOR_ITER_GEN': 171,
'FOR_ITER_LIST': 172,
'FOR_ITER_RANGE': 173,
'FOR_ITER_TUPLE': 174,
'JUMP_BACKWARD_JIT': 175,
'JUMP_BACKWARD_NO_JIT': 176,
'LOAD_ATTR_CLASS': 177,
'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178,
'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179,
'LOAD_ATTR_INSTANCE_VALUE': 180,
'LOAD_ATTR_METHOD_LAZY_DICT': 181,
'LOAD_ATTR_METHOD_NO_DICT': 182,
'LOAD_ATTR_METHOD_WITH_VALUES': 183,
'LOAD_ATTR_MODULE': 184,
'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185,
'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186,
'LOAD_ATTR_PROPERTY': 187,
'LOAD_ATTR_SLOT': 188,
'LOAD_ATTR_WITH_HINT': 189,
'LOAD_CONST_IMMORTAL': 190,
'LOAD_CONST_MORTAL': 191,
'LOAD_GLOBAL_BUILTIN': 192,
'LOAD_GLOBAL_MODULE': 193,
'LOAD_SUPER_ATTR_ATTR': 194,
'LOAD_SUPER_ATTR_METHOD': 195,
'RESUME_CHECK': 196,
'SEND_GEN': 197,
'STORE_ATTR_INSTANCE_VALUE': 198,
'STORE_ATTR_SLOT': 199,
'STORE_ATTR_WITH_HINT': 200,
'STORE_SUBSCR_DICT': 201,
'STORE_SUBSCR_LIST_INT': 202,
'TO_BOOL_ALWAYS_TRUE': 203,
'TO_BOOL_BOOL': 204,
'TO_BOOL_INT': 205,
'TO_BOOL_LIST': 206,
'TO_BOOL_NONE': 207,
'TO_BOOL_STR': 208,
'UNPACK_SEQUENCE_LIST': 209,
'UNPACK_SEQUENCE_TUPLE': 210,
'UNPACK_SEQUENCE_TWO_TUPLE': 211,
}
opmap = {
'CACHE': 0,
'RESERVED': 17,
'RESUME': 128,
'INSTRUMENTED_LINE': 254,
'ENTER_EXECUTOR': 255,
'BINARY_SLICE': 1,
'BUILD_TEMPLATE': 2,
'CALL_FUNCTION_EX': 4,
'CHECK_EG_MATCH': 5,
'CHECK_EXC_MATCH': 6,
'CLEANUP_THROW': 7,
'DELETE_SUBSCR': 8,
'END_FOR': 9,
'END_SEND': 10,
'EXIT_INIT_CHECK': 11,
'FORMAT_SIMPLE': 12,
'FORMAT_WITH_SPEC': 13,
'GET_AITER': 14,
'GET_ANEXT': 15,
'GET_ITER': 16,
'GET_LEN': 18,
'GET_YIELD_FROM_ITER': 19,
'INTERPRETER_EXIT': 20,
'LOAD_BUILD_CLASS': 21,
'LOAD_LOCALS': 22,
'MAKE_FUNCTION': 23,
'MATCH_KEYS': 24,
'MATCH_MAPPING': 25,
'MATCH_SEQUENCE': 26,
'NOP': 27,
'NOT_TAKEN': 28,
'POP_EXCEPT': 29,
'POP_ITER': 30,
'POP_TOP': 31,
'PUSH_EXC_INFO': 32,
'PUSH_NULL': 33,
'RETURN_GENERATOR': 34,
'RETURN_VALUE': 35,
'SETUP_ANNOTATIONS': 36,
'STORE_SLICE': 37,
'STORE_SUBSCR': 38,
'TO_BOOL': 39,
'UNARY_INVERT': 40,
'UNARY_NEGATIVE': 41,
'UNARY_NOT': 42,
'WITH_EXCEPT_START': 43,
'BINARY_OP': 44,
'BUILD_INTERPOLATION': 45,
'BUILD_LIST': 46,
'BUILD_MAP': 47,
'BUILD_SET': 48,
'BUILD_SLICE': 49,
'BUILD_STRING': 50,
'BUILD_TUPLE': 51,
'CALL': 52,
'CALL_INTRINSIC_1': 53,
'CALL_INTRINSIC_2': 54,
'CALL_KW': 55,
'COMPARE_OP': 56,
'CONTAINS_OP': 57,
'CONVERT_VALUE': 58,
'COPY': 59,
'COPY_FREE_VARS': 60,
'DELETE_ATTR': 61,
'DELETE_DEREF': 62,
'DELETE_FAST': 63,
'DELETE_GLOBAL': 64,
'DELETE_NAME': 65,
'DICT_MERGE': 66,
'DICT_UPDATE': 67,
'END_ASYNC_FOR': 68,
'EXTENDED_ARG': 69,
'FOR_ITER': 70,
'GET_AWAITABLE': 71,
'IMPORT_FROM': 72,
'IMPORT_NAME': 73,
'IS_OP': 74,
'JUMP_BACKWARD': 75,
'JUMP_BACKWARD_NO_INTERRUPT': 76,
'JUMP_FORWARD': 77,
'LIST_APPEND': 78,
'LIST_EXTEND': 79,
'LOAD_ATTR': 80,
'LOAD_COMMON_CONSTANT': 81,
'LOAD_CONST': 82,
'LOAD_DEREF': 83,
'LOAD_FAST': 84,
'LOAD_FAST_AND_CLEAR': 85,
'LOAD_FAST_BORROW': 86,
'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87,
'LOAD_FAST_CHECK': 88,
'LOAD_FAST_LOAD_FAST': 89,
'LOAD_FROM_DICT_OR_DEREF': 90,
'LOAD_FROM_DICT_OR_GLOBALS': 91,
'LOAD_GLOBAL': 92,
'LOAD_NAME': 93,
'LOAD_SMALL_INT': 94,
'LOAD_SPECIAL': 95,
'LOAD_SUPER_ATTR': 96,
'MAKE_CELL': 97,
'MAP_ADD': 98,
'MATCH_CLASS': 99,
'POP_JUMP_IF_FALSE': 100,
'POP_JUMP_IF_NONE': 101,
'POP_JUMP_IF_NOT_NONE': 102,
'POP_JUMP_IF_TRUE': 103,
'RAISE_VARARGS': 104,
'RERAISE': 105,
'SEND': 106,
'SET_ADD': 107,
'SET_FUNCTION_ATTRIBUTE': 108,
'SET_UPDATE': 109,
'STORE_ATTR': 110,
'STORE_DEREF': 111,
'STORE_FAST': 112,
'STORE_FAST_LOAD_FAST': 113,
'STORE_FAST_STORE_FAST': 114,
'STORE_GLOBAL': 115,
'STORE_NAME': 116,
'SWAP': 117,
'UNPACK_EX': 118,
'UNPACK_SEQUENCE': 119,
'YIELD_VALUE': 120,
'INSTRUMENTED_END_FOR': 234,
'INSTRUMENTED_POP_ITER': 235,
'INSTRUMENTED_END_SEND': 236,
'INSTRUMENTED_FOR_ITER': 237,
'INSTRUMENTED_INSTRUCTION': 238,
'INSTRUMENTED_JUMP_FORWARD': 239,
'INSTRUMENTED_NOT_TAKEN': 240,
'INSTRUMENTED_POP_JUMP_IF_TRUE': 241,
'INSTRUMENTED_POP_JUMP_IF_FALSE': 242,
'INSTRUMENTED_POP_JUMP_IF_NONE': 243,
'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244,
'INSTRUMENTED_RESUME': 245,
'INSTRUMENTED_RETURN_VALUE': 246,
'INSTRUMENTED_YIELD_VALUE': 247,
'INSTRUMENTED_END_ASYNC_FOR': 248,
'INSTRUMENTED_LOAD_SUPER_ATTR': 249,
'INSTRUMENTED_CALL': 250,
'INSTRUMENTED_CALL_KW': 251,
'INSTRUMENTED_CALL_FUNCTION_EX': 252,
'INSTRUMENTED_JUMP_BACKWARD': 253,
'ANNOTATIONS_PLACEHOLDER': 256,
'JUMP': 257,
'JUMP_IF_FALSE': 258,
'JUMP_IF_TRUE': 259,
'JUMP_NO_INTERRUPT': 260,
'LOAD_CLOSURE': 261,
'POP_BLOCK': 262,
'SETUP_CLEANUP': 263,
'SETUP_FINALLY': 264,
'SETUP_WITH': 265,
'STORE_FAST_MAYBE_NULL': 266,
}
HAVE_ARGUMENT = 43
MIN_INSTRUMENTED_OPCODE = 234

View file

@ -0,0 +1,869 @@
"""Python part of the warnings subsystem."""
import sys
import _contextvars
import _thread
__all__ = ["warn", "warn_explicit", "showwarning",
"formatwarning", "filterwarnings", "simplefilter",
"resetwarnings", "catch_warnings", "deprecated"]
# Normally '_wm' is sys.modules['warnings'] but for unit tests it can be
# a different module. User code is allowed to reassign global attributes
# of the 'warnings' module, commonly 'filters' or 'showwarning'. So we
# need to lookup these global attributes dynamically on the '_wm' object,
# rather than binding them earlier. The code in this module consistently uses
# '_wm.<something>' rather than using the globals of this module. If the
# '_warnings' C extension is in use, some globals are replaced by functions
# and variables defined in that extension.
_wm = None
def _set_module(module):
global _wm
_wm = module
# filters contains a sequence of filter 5-tuples
# The components of the 5-tuple are:
# - an action: error, ignore, always, all, default, module, or once
# - a compiled regex that must match the warning message
# - a class representing the warning category
# - a compiled regex that must match the module that is being warned
# - a line number for the line being warning, or 0 to mean any line
# If either if the compiled regexs are None, match anything.
filters = []
defaultaction = "default"
onceregistry = {}
_lock = _thread.RLock()
_filters_version = 1
# If true, catch_warnings() will use a context var to hold the modified
# filters list. Otherwise, catch_warnings() will operate on the 'filters'
# global of the warnings module.
_use_context = sys.flags.context_aware_warnings
class _Context:
def __init__(self, filters):
self._filters = filters
self.log = None # if set to a list, logging is enabled
def copy(self):
context = _Context(self._filters[:])
if self.log is not None:
context.log = self.log
return context
def _record_warning(self, msg):
self.log.append(msg)
class _GlobalContext(_Context):
def __init__(self):
self.log = None
@property
def _filters(self):
# Since there is quite a lot of code that assigns to
# warnings.filters, this needs to return the current value of
# the module global.
try:
return _wm.filters
except AttributeError:
# 'filters' global was deleted. Do we need to actually handle this case?
return []
_global_context = _GlobalContext()
_warnings_context = _contextvars.ContextVar('warnings_context')
def _get_context():
if not _use_context:
return _global_context
try:
return _wm._warnings_context.get()
except LookupError:
return _global_context
def _set_context(context):
assert _use_context
_wm._warnings_context.set(context)
def _new_context():
assert _use_context
old_context = _wm._get_context()
new_context = old_context.copy()
_wm._set_context(new_context)
return old_context, new_context
def _get_filters():
"""Return the current list of filters. This is a non-public API used by
module functions and by the unit tests."""
return _wm._get_context()._filters
def _filters_mutated_lock_held():
_wm._filters_version += 1
def showwarning(message, category, filename, lineno, file=None, line=None):
"""Hook to write a warning to a file; replace if you like."""
msg = _wm.WarningMessage(message, category, filename, lineno, file, line)
_wm._showwarnmsg_impl(msg)
def formatwarning(message, category, filename, lineno, line=None):
"""Function to format a warning the standard way."""
msg = _wm.WarningMessage(message, category, filename, lineno, None, line)
return _wm._formatwarnmsg_impl(msg)
def _showwarnmsg_impl(msg):
context = _wm._get_context()
if context.log is not None:
context._record_warning(msg)
return
file = msg.file
if file is None:
file = sys.stderr
if file is None:
# sys.stderr is None when run with pythonw.exe:
# warnings get lost
return
text = _wm._formatwarnmsg(msg)
try:
file.write(text)
except OSError:
# the file (probably stderr) is invalid - this warning gets lost.
pass
def _formatwarnmsg_impl(msg):
category = msg.category.__name__
s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n"
if msg.line is None:
try:
import linecache
line = linecache.getline(msg.filename, msg.lineno)
except Exception:
# When a warning is logged during Python shutdown, linecache
# and the import machinery don't work anymore
line = None
linecache = None
else:
line = msg.line
if line:
line = line.strip()
s += " %s\n" % line
if msg.source is not None:
try:
import tracemalloc
# Logging a warning should not raise a new exception:
# catch Exception, not only ImportError and RecursionError.
except Exception:
# don't suggest to enable tracemalloc if it's not available
suggest_tracemalloc = False
tb = None
else:
try:
suggest_tracemalloc = not tracemalloc.is_tracing()
tb = tracemalloc.get_object_traceback(msg.source)
except Exception:
# When a warning is logged during Python shutdown, tracemalloc
# and the import machinery don't work anymore
suggest_tracemalloc = False
tb = None
if tb is not None:
s += 'Object allocated at (most recent call last):\n'
for frame in tb:
s += (' File "%s", lineno %s\n'
% (frame.filename, frame.lineno))
try:
if linecache is not None:
line = linecache.getline(frame.filename, frame.lineno)
else:
line = None
except Exception:
line = None
if line:
line = line.strip()
s += ' %s\n' % line
elif suggest_tracemalloc:
s += (f'{category}: Enable tracemalloc to get the object '
f'allocation traceback\n')
return s
# Keep a reference to check if the function was replaced
_showwarning_orig = showwarning
def _showwarnmsg(msg):
"""Hook to write a warning to a file; replace if you like."""
try:
sw = _wm.showwarning
except AttributeError:
pass
else:
if sw is not _showwarning_orig:
# warnings.showwarning() was replaced
if not callable(sw):
raise TypeError("warnings.showwarning() must be set to a "
"function or method")
sw(msg.message, msg.category, msg.filename, msg.lineno,
msg.file, msg.line)
return
_wm._showwarnmsg_impl(msg)
# Keep a reference to check if the function was replaced
_formatwarning_orig = formatwarning
def _formatwarnmsg(msg):
"""Function to format a warning the standard way."""
try:
fw = _wm.formatwarning
except AttributeError:
pass
else:
if fw is not _formatwarning_orig:
# warnings.formatwarning() was replaced
return fw(msg.message, msg.category,
msg.filename, msg.lineno, msg.line)
return _wm._formatwarnmsg_impl(msg)
def filterwarnings(action, message="", category=Warning, module="", lineno=0,
append=False):
"""Insert an entry into the list of warnings filters (at the front).
'action' -- one of "error", "ignore", "always", "all", "default", "module",
or "once"
'message' -- a regex that the warning message must match
'category' -- a class that the warning must be a subclass of
'module' -- a regex that the module name must match
'lineno' -- an integer line number, 0 matches all warnings
'append' -- if true, append to the list of filters
"""
if action not in {"error", "ignore", "always", "all", "default", "module", "once"}:
raise ValueError(f"invalid action: {action!r}")
if not isinstance(message, str):
raise TypeError("message must be a string")
if not isinstance(category, type) or not issubclass(category, Warning):
raise TypeError("category must be a Warning subclass")
if not isinstance(module, str):
raise TypeError("module must be a string")
if not isinstance(lineno, int):
raise TypeError("lineno must be an int")
if lineno < 0:
raise ValueError("lineno must be an int >= 0")
if message or module:
import re
if message:
message = re.compile(message, re.I)
else:
message = None
if module:
module = re.compile(module)
else:
module = None
_wm._add_filter(action, message, category, module, lineno, append=append)
def simplefilter(action, category=Warning, lineno=0, append=False):
"""Insert a simple entry into the list of warnings filters (at the front).
A simple filter matches all modules and messages.
'action' -- one of "error", "ignore", "always", "all", "default", "module",
or "once"
'category' -- a class that the warning must be a subclass of
'lineno' -- an integer line number, 0 matches all warnings
'append' -- if true, append to the list of filters
"""
if action not in {"error", "ignore", "always", "all", "default", "module", "once"}:
raise ValueError(f"invalid action: {action!r}")
if not isinstance(lineno, int):
raise TypeError("lineno must be an int")
if lineno < 0:
raise ValueError("lineno must be an int >= 0")
_wm._add_filter(action, None, category, None, lineno, append=append)
def _filters_mutated():
# Even though this function is not part of the public API, it's used by
# a fair amount of user code.
with _wm._lock:
_wm._filters_mutated_lock_held()
def _add_filter(*item, append):
with _wm._lock:
filters = _wm._get_filters()
if not append:
# Remove possible duplicate filters, so new one will be placed
# in correct place. If append=True and duplicate exists, do nothing.
try:
filters.remove(item)
except ValueError:
pass
filters.insert(0, item)
else:
if item not in filters:
filters.append(item)
_wm._filters_mutated_lock_held()
def resetwarnings():
"""Clear the list of warning filters, so that no filters are active."""
with _wm._lock:
del _wm._get_filters()[:]
_wm._filters_mutated_lock_held()
class _OptionError(Exception):
"""Exception used by option processing helpers."""
pass
# Helper to process -W options passed via sys.warnoptions
def _processoptions(args):
for arg in args:
try:
_wm._setoption(arg)
except _wm._OptionError as msg:
print("Invalid -W option ignored:", msg, file=sys.stderr)
# Helper for _processoptions()
def _setoption(arg):
parts = arg.split(':')
if len(parts) > 5:
raise _wm._OptionError("too many fields (max 5): %r" % (arg,))
while len(parts) < 5:
parts.append('')
action, message, category, module, lineno = [s.strip()
for s in parts]
action = _wm._getaction(action)
category = _wm._getcategory(category)
if message or module:
import re
if message:
message = re.escape(message)
if module:
module = re.escape(module) + r'\z'
if lineno:
try:
lineno = int(lineno)
if lineno < 0:
raise ValueError
except (ValueError, OverflowError):
raise _wm._OptionError("invalid lineno %r" % (lineno,)) from None
else:
lineno = 0
_wm.filterwarnings(action, message, category, module, lineno)
# Helper for _setoption()
def _getaction(action):
if not action:
return "default"
for a in ('default', 'always', 'all', 'ignore', 'module', 'once', 'error'):
if a.startswith(action):
return a
raise _wm._OptionError("invalid action: %r" % (action,))
# Helper for _setoption()
def _getcategory(category):
if not category:
return Warning
if '.' not in category:
import builtins as m
klass = category
else:
module, _, klass = category.rpartition('.')
try:
m = __import__(module, None, None, [klass])
except ImportError:
raise _wm._OptionError("invalid module name: %r" % (module,)) from None
try:
cat = getattr(m, klass)
except AttributeError:
raise _wm._OptionError("unknown warning category: %r" % (category,)) from None
if not issubclass(cat, Warning):
raise _wm._OptionError("invalid warning category: %r" % (category,))
return cat
def _is_internal_filename(filename):
return 'importlib' in filename and '_bootstrap' in filename
def _is_filename_to_skip(filename, skip_file_prefixes):
return any(filename.startswith(prefix) for prefix in skip_file_prefixes)
def _is_internal_frame(frame):
"""Signal whether the frame is an internal CPython implementation detail."""
return _is_internal_filename(frame.f_code.co_filename)
def _next_external_frame(frame, skip_file_prefixes):
"""Find the next frame that doesn't involve Python or user internals."""
frame = frame.f_back
while frame is not None and (
_is_internal_filename(filename := frame.f_code.co_filename) or
_is_filename_to_skip(filename, skip_file_prefixes)):
frame = frame.f_back
return frame
# Code typically replaced by _warnings
def warn(message, category=None, stacklevel=1, source=None,
*, skip_file_prefixes=()):
"""Issue a warning, or maybe ignore it or raise an exception."""
# Check if message is already a Warning object
if isinstance(message, Warning):
category = message.__class__
# Check category argument
if category is None:
category = UserWarning
if not (isinstance(category, type) and issubclass(category, Warning)):
raise TypeError("category must be a Warning subclass, "
"not '{:s}'".format(type(category).__name__))
if not isinstance(skip_file_prefixes, tuple):
# The C version demands a tuple for implementation performance.
raise TypeError('skip_file_prefixes must be a tuple of strs.')
if skip_file_prefixes:
stacklevel = max(2, stacklevel)
# Get context information
try:
if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)):
# If frame is too small to care or if the warning originated in
# internal code, then do not try to hide any frames.
frame = sys._getframe(stacklevel)
else:
frame = sys._getframe(1)
# Look for one frame less since the above line starts us off.
for x in range(stacklevel-1):
frame = _next_external_frame(frame, skip_file_prefixes)
if frame is None:
raise ValueError
except ValueError:
globals = sys.__dict__
filename = "<sys>"
lineno = 0
else:
globals = frame.f_globals
filename = frame.f_code.co_filename
lineno = frame.f_lineno
if '__name__' in globals:
module = globals['__name__']
else:
module = "<string>"
registry = globals.setdefault("__warningregistry__", {})
_wm.warn_explicit(
message,
category,
filename,
lineno,
module,
registry,
globals,
source=source,
)
def warn_explicit(message, category, filename, lineno,
module=None, registry=None, module_globals=None,
source=None):
lineno = int(lineno)
if module is None:
module = filename or "<unknown>"
if module[-3:].lower() == ".py":
module = module[:-3] # XXX What about leading pathname?
if isinstance(message, Warning):
text = str(message)
category = message.__class__
else:
text = message
message = category(message)
key = (text, category, lineno)
with _wm._lock:
if registry is None:
registry = {}
if registry.get('version', 0) != _wm._filters_version:
registry.clear()
registry['version'] = _wm._filters_version
# Quick test for common case
if registry.get(key):
return
# Search the filters
for item in _wm._get_filters():
action, msg, cat, mod, ln = item
if ((msg is None or msg.match(text)) and
issubclass(category, cat) and
(mod is None or mod.match(module)) and
(ln == 0 or lineno == ln)):
break
else:
action = _wm.defaultaction
# Early exit actions
if action == "ignore":
return
if action == "error":
raise message
# Other actions
if action == "once":
registry[key] = 1
oncekey = (text, category)
if _wm.onceregistry.get(oncekey):
return
_wm.onceregistry[oncekey] = 1
elif action in {"always", "all"}:
pass
elif action == "module":
registry[key] = 1
altkey = (text, category, 0)
if registry.get(altkey):
return
registry[altkey] = 1
elif action == "default":
registry[key] = 1
else:
# Unrecognized actions are errors
raise RuntimeError(
"Unrecognized action (%r) in warnings.filters:\n %s" %
(action, item))
# Prime the linecache for formatting, in case the
# "file" is actually in a zipfile or something.
import linecache
linecache.getlines(filename, module_globals)
# Print message and context
msg = _wm.WarningMessage(message, category, filename, lineno, source=source)
_wm._showwarnmsg(msg)
class WarningMessage(object):
_WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
"line", "source")
def __init__(self, message, category, filename, lineno, file=None,
line=None, source=None):
self.message = message
self.category = category
self.filename = filename
self.lineno = lineno
self.file = file
self.line = line
self.source = source
self._category_name = category.__name__ if category else None
def __str__(self):
return ("{message : %r, category : %r, filename : %r, lineno : %s, "
"line : %r}" % (self.message, self._category_name,
self.filename, self.lineno, self.line))
def __repr__(self):
return f'<{type(self).__qualname__} {self}>'
class catch_warnings(object):
"""A context manager that copies and restores the warnings filter upon
exiting the context.
The 'record' argument specifies whether warnings should be captured by a
custom implementation of warnings.showwarning() and be appended to a list
returned by the context manager. Otherwise None is returned by the context
manager. The objects appended to the list are arguments whose attributes
mirror the arguments to showwarning().
The 'module' argument is to specify an alternative module to the module
named 'warnings' and imported under that name. This argument is only useful
when testing the warnings module itself.
If the 'action' argument is not None, the remaining arguments are passed
to warnings.simplefilter() as if it were called immediately on entering the
context.
"""
def __init__(self, *, record=False, module=None,
action=None, category=Warning, lineno=0, append=False):
"""Specify whether to record warnings and if an alternative module
should be used other than sys.modules['warnings'].
"""
self._record = record
self._module = sys.modules['warnings'] if module is None else module
self._entered = False
if action is None:
self._filter = None
else:
self._filter = (action, category, lineno, append)
def __repr__(self):
args = []
if self._record:
args.append("record=True")
if self._module is not sys.modules['warnings']:
args.append("module=%r" % self._module)
name = type(self).__name__
return "%s(%s)" % (name, ", ".join(args))
def __enter__(self):
if self._entered:
raise RuntimeError("Cannot enter %r twice" % self)
self._entered = True
with _wm._lock:
if _use_context:
self._saved_context, context = self._module._new_context()
else:
context = None
self._filters = self._module.filters
self._module.filters = self._filters[:]
self._showwarning = self._module.showwarning
self._showwarnmsg_impl = self._module._showwarnmsg_impl
self._module._filters_mutated_lock_held()
if self._record:
if _use_context:
context.log = log = []
else:
log = []
self._module._showwarnmsg_impl = log.append
# Reset showwarning() to the default implementation to make sure
# that _showwarnmsg() calls _showwarnmsg_impl()
self._module.showwarning = self._module._showwarning_orig
else:
log = None
if self._filter is not None:
self._module.simplefilter(*self._filter)
return log
def __exit__(self, *exc_info):
if not self._entered:
raise RuntimeError("Cannot exit %r without entering first" % self)
with _wm._lock:
if _use_context:
self._module._warnings_context.set(self._saved_context)
else:
self._module.filters = self._filters
self._module.showwarning = self._showwarning
self._module._showwarnmsg_impl = self._showwarnmsg_impl
self._module._filters_mutated_lock_held()
class deprecated:
"""Indicate that a class, function or overload is deprecated.
When this decorator is applied to an object, the type checker
will generate a diagnostic on usage of the deprecated object.
Usage:
@deprecated("Use B instead")
class A:
pass
@deprecated("Use g instead")
def f():
pass
@overload
@deprecated("int support is deprecated")
def g(x: int) -> int: ...
@overload
def g(x: str) -> int: ...
The warning specified by *category* will be emitted at runtime
on use of deprecated objects. For functions, that happens on calls;
for classes, on instantiation and on creation of subclasses.
If the *category* is ``None``, no warning is emitted at runtime.
The *stacklevel* determines where the
warning is emitted. If it is ``1`` (the default), the warning
is emitted at the direct caller of the deprecated object; if it
is higher, it is emitted further up the stack.
Static type checker behavior is not affected by the *category*
and *stacklevel* arguments.
The deprecation message passed to the decorator is saved in the
``__deprecated__`` attribute on the decorated object.
If applied to an overload, the decorator
must be after the ``@overload`` decorator for the attribute to
exist on the overload as returned by ``get_overloads()``.
See PEP 702 for details.
"""
def __init__(
self,
message: str,
/,
*,
category: type[Warning] | None = DeprecationWarning,
stacklevel: int = 1,
) -> None:
if not isinstance(message, str):
raise TypeError(
f"Expected an object of type str for 'message', not {type(message).__name__!r}"
)
self.message = message
self.category = category
self.stacklevel = stacklevel
def __call__(self, arg, /):
# Make sure the inner functions created below don't
# retain a reference to self.
msg = self.message
category = self.category
stacklevel = self.stacklevel
if category is None:
arg.__deprecated__ = msg
return arg
elif isinstance(arg, type):
import functools
from types import MethodType
original_new = arg.__new__
@functools.wraps(original_new)
def __new__(cls, /, *args, **kwargs):
if cls is arg:
_wm.warn(msg, category=category, stacklevel=stacklevel + 1)
if original_new is not object.__new__:
return original_new(cls, *args, **kwargs)
# Mirrors a similar check in object.__new__.
elif cls.__init__ is object.__init__ and (args or kwargs):
raise TypeError(f"{cls.__name__}() takes no arguments")
else:
return original_new(cls)
arg.__new__ = staticmethod(__new__)
if "__init_subclass__" in arg.__dict__:
# __init_subclass__ is directly present on the decorated class.
# Synthesize a wrapper that calls this method directly.
original_init_subclass = arg.__init_subclass__
# We need slightly different behavior if __init_subclass__
# is a bound method (likely if it was implemented in Python).
# Otherwise, it likely means it's a builtin such as
# object's implementation of __init_subclass__.
if isinstance(original_init_subclass, MethodType):
original_init_subclass = original_init_subclass.__func__
@functools.wraps(original_init_subclass)
def __init_subclass__(*args, **kwargs):
_wm.warn(msg, category=category, stacklevel=stacklevel + 1)
return original_init_subclass(*args, **kwargs)
else:
def __init_subclass__(cls, *args, **kwargs):
_wm.warn(msg, category=category, stacklevel=stacklevel + 1)
return super(arg, cls).__init_subclass__(*args, **kwargs)
arg.__init_subclass__ = classmethod(__init_subclass__)
arg.__deprecated__ = __new__.__deprecated__ = msg
__init_subclass__.__deprecated__ = msg
return arg
elif callable(arg):
import functools
import inspect
@functools.wraps(arg)
def wrapper(*args, **kwargs):
_wm.warn(msg, category=category, stacklevel=stacklevel + 1)
return arg(*args, **kwargs)
if inspect.iscoroutinefunction(arg):
wrapper = inspect.markcoroutinefunction(wrapper)
arg.__deprecated__ = wrapper.__deprecated__ = msg
return wrapper
else:
raise TypeError(
"@deprecated decorator with non-None category must be applied to "
f"a class or callable, not {arg!r}"
)
_DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}"
def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info):
"""Warn that *name* is deprecated or should be removed.
RuntimeError is raised if *remove* specifies a major/minor tuple older than
the current Python version or the same version but past the alpha.
The *message* argument is formatted with *name* and *remove* as a Python
version tuple (e.g. (3, 11)).
"""
remove_formatted = f"{remove[0]}.{remove[1]}"
if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"):
msg = f"{name!r} was slated for removal after Python {remove_formatted} alpha"
raise RuntimeError(msg)
else:
msg = message.format(name=name, remove=remove_formatted)
_wm.warn(msg, DeprecationWarning, stacklevel=3)
# Private utility function called by _PyErr_WarnUnawaitedCoroutine
def _warn_unawaited_coroutine(coro):
msg_lines = [
f"coroutine '{coro.__qualname__}' was never awaited\n"
]
if coro.cr_origin is not None:
import linecache, traceback
def extract():
for filename, lineno, funcname in reversed(coro.cr_origin):
line = linecache.getline(filename, lineno)
yield (filename, lineno, funcname, line)
msg_lines.append("Coroutine created at (most recent call last)\n")
msg_lines += traceback.format_list(list(extract()))
msg = "".join(msg_lines).rstrip("\n")
# Passing source= here means that if the user happens to have tracemalloc
# enabled and tracking where the coroutine was created, the warning will
# contain that traceback. This does mean that if they have *both*
# coroutine origin tracking *and* tracemalloc enabled, they'll get two
# partially-redundant tracebacks. If we wanted to be clever we could
# probably detect this case and avoid it, but for now we don't bother.
_wm.warn(
msg, category=RuntimeWarning, stacklevel=2, source=coro
)
def _setup_defaults():
# Several warning categories are ignored by default in regular builds
if hasattr(sys, 'gettotalrefcount'):
return
_wm.filterwarnings("default", category=DeprecationWarning, module="__main__", append=1)
_wm.simplefilter("ignore", category=DeprecationWarning, append=1)
_wm.simplefilter("ignore", category=PendingDeprecationWarning, append=1)
_wm.simplefilter("ignore", category=ImportWarning, append=1)
_wm.simplefilter("ignore", category=ResourceWarning, append=1)

View file

@ -0,0 +1,103 @@
"""
The objects used by the site module to add custom builtins.
"""
# Those objects are almost immortal and they keep a reference to their module
# globals. Defining them in the site module would keep too many references
# alive.
# Note this means this module should also avoid keep things alive in its
# globals.
import sys
class Quitter(object):
def __init__(self, name, eof):
self.name = name
self.eof = eof
def __repr__(self):
return 'Use %s() or %s to exit' % (self.name, self.eof)
def __call__(self, code=None):
# Shells like IDLE catch the SystemExit, but listen when their
# stdin wrapper is closed.
try:
sys.stdin.close()
except:
pass
raise SystemExit(code)
class _Printer(object):
"""interactive prompt objects for printing the license text, a list of
contributors and the copyright notice."""
MAXLINES = 23
def __init__(self, name, data, files=(), dirs=()):
import os
self.__name = name
self.__data = data
self.__lines = None
self.__filenames = [os.path.join(dir, filename)
for dir in dirs
for filename in files]
def __setup(self):
if self.__lines:
return
data = None
for filename in self.__filenames:
try:
with open(filename, encoding='utf-8') as fp:
data = fp.read()
break
except OSError:
pass
if not data:
data = self.__data
self.__lines = data.split('\n')
self.__linecnt = len(self.__lines)
def __repr__(self):
self.__setup()
if len(self.__lines) <= self.MAXLINES:
return "\n".join(self.__lines)
else:
return "Type %s() to see the full %s text" % ((self.__name,)*2)
def __call__(self):
self.__setup()
prompt = 'Hit Return for more, or q (and Return) to quit: '
lineno = 0
while 1:
try:
for i in range(lineno, lineno + self.MAXLINES):
print(self.__lines[i])
except IndexError:
break
else:
lineno += self.MAXLINES
key = None
while key is None:
key = input(prompt)
if key not in ('', 'q'):
key = None
if key == 'q':
break
class _Helper(object):
"""Define the builtin 'help'.
This is a wrapper around pydoc.help that provides a helpful message
when 'help' is typed at the Python interactive prompt.
Calling help() at the Python prompt starts an interactive help session.
Calling help(thing) prints help for the python object 'thing'.
"""
def __repr__(self):
return "Type help() for interactive help, " \
"or help(object) for help about object."
def __call__(self, *args, **kwds):
import pydoc
return pydoc.help(*args, **kwds)

View file

@ -0,0 +1,147 @@
# Access WeakSet through the weakref module.
# This code is separated-out because it is needed
# by abc.py to load everything else at startup.
from _weakref import ref
from types import GenericAlias
__all__ = ['WeakSet']
class WeakSet:
def __init__(self, data=None):
self.data = set()
def _remove(item, selfref=ref(self)):
self = selfref()
if self is not None:
self.data.discard(item)
self._remove = _remove
if data is not None:
self.update(data)
def __iter__(self):
for itemref in self.data.copy():
item = itemref()
if item is not None:
# Caveat: the iterator will keep a strong reference to
# `item` until it is resumed or closed.
yield item
def __len__(self):
return len(self.data)
def __contains__(self, item):
try:
wr = ref(item)
except TypeError:
return False
return wr in self.data
def __reduce__(self):
return self.__class__, (list(self),), self.__getstate__()
def add(self, item):
self.data.add(ref(item, self._remove))
def clear(self):
self.data.clear()
def copy(self):
return self.__class__(self)
def pop(self):
while True:
try:
itemref = self.data.pop()
except KeyError:
raise KeyError('pop from empty WeakSet') from None
item = itemref()
if item is not None:
return item
def remove(self, item):
self.data.remove(ref(item))
def discard(self, item):
self.data.discard(ref(item))
def update(self, other):
for element in other:
self.add(element)
def __ior__(self, other):
self.update(other)
return self
def difference(self, other):
newset = self.copy()
newset.difference_update(other)
return newset
__sub__ = difference
def difference_update(self, other):
self.__isub__(other)
def __isub__(self, other):
if self is other:
self.data.clear()
else:
self.data.difference_update(ref(item) for item in other)
return self
def intersection(self, other):
return self.__class__(item for item in other if item in self)
__and__ = intersection
def intersection_update(self, other):
self.__iand__(other)
def __iand__(self, other):
self.data.intersection_update(ref(item) for item in other)
return self
def issubset(self, other):
return self.data.issubset(ref(item) for item in other)
__le__ = issubset
def __lt__(self, other):
return self.data < set(map(ref, other))
def issuperset(self, other):
return self.data.issuperset(ref(item) for item in other)
__ge__ = issuperset
def __gt__(self, other):
return self.data > set(map(ref, other))
def __eq__(self, other):
if not isinstance(other, self.__class__):
return NotImplemented
return self.data == set(map(ref, other))
def symmetric_difference(self, other):
newset = self.copy()
newset.symmetric_difference_update(other)
return newset
__xor__ = symmetric_difference
def symmetric_difference_update(self, other):
self.__ixor__(other)
def __ixor__(self, other):
if self is other:
self.data.clear()
else:
self.data.symmetric_difference_update(ref(item, self._remove) for item in other)
return self
def union(self, other):
return self.__class__(e for s in (self, other) for e in s)
__or__ = union
def isdisjoint(self, other):
return len(self.intersection(other)) == 0
def __repr__(self):
return repr(self.data)
__class_getitem__ = classmethod(GenericAlias)

View file

@ -0,0 +1,188 @@
# Copyright 2007 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""Abstract Base Classes (ABCs) according to PEP 3119."""
def abstractmethod(funcobj):
"""A decorator indicating abstract methods.
Requires that the metaclass is ABCMeta or derived from it. A
class that has a metaclass derived from ABCMeta cannot be
instantiated unless all of its abstract methods are overridden.
The abstract methods can be called using any of the normal
'super' call mechanisms. abstractmethod() may be used to declare
abstract methods for properties and descriptors.
Usage:
class C(metaclass=ABCMeta):
@abstractmethod
def my_abstract_method(self, arg1, arg2, argN):
...
"""
funcobj.__isabstractmethod__ = True
return funcobj
class abstractclassmethod(classmethod):
"""A decorator indicating abstract classmethods.
Deprecated, use 'classmethod' with 'abstractmethod' instead:
class C(ABC):
@classmethod
@abstractmethod
def my_abstract_classmethod(cls, ...):
...
"""
__isabstractmethod__ = True
def __init__(self, callable):
callable.__isabstractmethod__ = True
super().__init__(callable)
class abstractstaticmethod(staticmethod):
"""A decorator indicating abstract staticmethods.
Deprecated, use 'staticmethod' with 'abstractmethod' instead:
class C(ABC):
@staticmethod
@abstractmethod
def my_abstract_staticmethod(...):
...
"""
__isabstractmethod__ = True
def __init__(self, callable):
callable.__isabstractmethod__ = True
super().__init__(callable)
class abstractproperty(property):
"""A decorator indicating abstract properties.
Deprecated, use 'property' with 'abstractmethod' instead:
class C(ABC):
@property
@abstractmethod
def my_abstract_property(self):
...
"""
__isabstractmethod__ = True
try:
from _abc import (get_cache_token, _abc_init, _abc_register,
_abc_instancecheck, _abc_subclasscheck, _get_dump,
_reset_registry, _reset_caches)
except ImportError:
from _py_abc import ABCMeta, get_cache_token
ABCMeta.__module__ = 'abc'
else:
class ABCMeta(type):
"""Metaclass for defining Abstract Base Classes (ABCs).
Use this metaclass to create an ABC. An ABC can be subclassed
directly, and then acts as a mix-in class. You can also register
unrelated concrete classes (even built-in classes) and unrelated
ABCs as 'virtual subclasses' -- these and their descendants will
be considered subclasses of the registering ABC by the built-in
issubclass() function, but the registering ABC won't show up in
their MRO (Method Resolution Order) nor will method
implementations defined by the registering ABC be callable (not
even via super()).
"""
def __new__(mcls, name, bases, namespace, /, **kwargs):
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
_abc_init(cls)
return cls
def register(cls, subclass):
"""Register a virtual subclass of an ABC.
Returns the subclass, to allow usage as a class decorator.
"""
return _abc_register(cls, subclass)
def __instancecheck__(cls, instance):
"""Override for isinstance(instance, cls)."""
return _abc_instancecheck(cls, instance)
def __subclasscheck__(cls, subclass):
"""Override for issubclass(subclass, cls)."""
return _abc_subclasscheck(cls, subclass)
def _dump_registry(cls, file=None):
"""Debug helper to print the ABC registry."""
print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file)
print(f"Inv. counter: {get_cache_token()}", file=file)
(_abc_registry, _abc_cache, _abc_negative_cache,
_abc_negative_cache_version) = _get_dump(cls)
print(f"_abc_registry: {_abc_registry!r}", file=file)
print(f"_abc_cache: {_abc_cache!r}", file=file)
print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file)
print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}",
file=file)
def _abc_registry_clear(cls):
"""Clear the registry (for debugging or testing)."""
_reset_registry(cls)
def _abc_caches_clear(cls):
"""Clear the caches (for debugging or testing)."""
_reset_caches(cls)
def update_abstractmethods(cls):
"""Recalculate the set of abstract methods of an abstract class.
If a class has had one of its abstract methods implemented after the
class was created, the method will not be considered implemented until
this function is called. Alternatively, if a new abstract method has been
added to the class, it will only be considered an abstract method of the
class after this function is called.
This function should be called before any use is made of the class,
usually in class decorators that add methods to the subject class.
Returns cls, to allow usage as a class decorator.
If cls is not an instance of ABCMeta, does nothing.
"""
if not hasattr(cls, '__abstractmethods__'):
# We check for __abstractmethods__ here because cls might by a C
# implementation or a python implementation (especially during
# testing), and we want to handle both cases.
return cls
abstracts = set()
# Check the existing abstract methods of the parents, keep only the ones
# that are not implemented.
for scls in cls.__bases__:
for name in getattr(scls, '__abstractmethods__', ()):
value = getattr(cls, name, None)
if getattr(value, "__isabstractmethod__", False):
abstracts.add(name)
# Also add any other newly added abstract methods.
for name, value in cls.__dict__.items():
if getattr(value, "__isabstractmethod__", False):
abstracts.add(name)
cls.__abstractmethods__ = frozenset(abstracts)
return cls
class ABC(metaclass=ABCMeta):
"""Helper class that provides a standard way to create an ABC using
inheritance.
"""
__slots__ = ()

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,680 @@
"""
The `ast` module helps Python applications to process trees of the Python
abstract syntax grammar. The abstract syntax itself might change with
each Python release; this module helps to find out programmatically what
the current grammar looks like and allows modifications of it.
An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
a flag to the `compile()` builtin function or by using the `parse()`
function from this module. The result will be a tree of objects whose
classes all inherit from `ast.AST`.
A modified abstract syntax tree can be compiled into a Python code object
using the built-in `compile()` function.
Additionally various helper functions are provided that make working with
the trees simpler. The main intention of the helper functions and this
module in general is to provide an easy to use interface for libraries
that work tightly with the python syntax (template engines for example).
:copyright: Copyright 2008 by Armin Ronacher.
:license: Python License.
"""
from _ast import *
def parse(source, filename='<unknown>', mode='exec', *,
type_comments=False, feature_version=None, optimize=-1):
"""
Parse the source into an AST node.
Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
Pass type_comments=True to get back type comments where the syntax allows.
"""
flags = PyCF_ONLY_AST
if optimize > 0:
flags |= PyCF_OPTIMIZED_AST
if type_comments:
flags |= PyCF_TYPE_COMMENTS
if feature_version is None:
feature_version = -1
elif isinstance(feature_version, tuple):
major, minor = feature_version # Should be a 2-tuple.
if major != 3:
raise ValueError(f"Unsupported major version: {major}")
feature_version = minor
# Else it should be an int giving the minor version for 3.x.
return compile(source, filename, mode, flags,
_feature_version=feature_version, optimize=optimize)
def literal_eval(node_or_string):
"""
Evaluate an expression node or a string containing only a Python
expression. The string or node provided may only consist of the following
Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
sets, booleans, and None.
Caution: A complex expression can overflow the C stack and cause a crash.
"""
if isinstance(node_or_string, str):
node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval')
if isinstance(node_or_string, Expression):
node_or_string = node_or_string.body
def _raise_malformed_node(node):
msg = "malformed node or string"
if lno := getattr(node, 'lineno', None):
msg += f' on line {lno}'
raise ValueError(msg + f': {node!r}')
def _convert_num(node):
if not isinstance(node, Constant) or type(node.value) not in (int, float, complex):
_raise_malformed_node(node)
return node.value
def _convert_signed_num(node):
if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)):
operand = _convert_num(node.operand)
if isinstance(node.op, UAdd):
return + operand
else:
return - operand
return _convert_num(node)
def _convert(node):
if isinstance(node, Constant):
return node.value
elif isinstance(node, Tuple):
return tuple(map(_convert, node.elts))
elif isinstance(node, List):
return list(map(_convert, node.elts))
elif isinstance(node, Set):
return set(map(_convert, node.elts))
elif (isinstance(node, Call) and isinstance(node.func, Name) and
node.func.id == 'set' and node.args == node.keywords == []):
return set()
elif isinstance(node, Dict):
if len(node.keys) != len(node.values):
_raise_malformed_node(node)
return dict(zip(map(_convert, node.keys),
map(_convert, node.values)))
elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)):
left = _convert_signed_num(node.left)
right = _convert_num(node.right)
if isinstance(left, (int, float)) and isinstance(right, complex):
if isinstance(node.op, Add):
return left + right
else:
return left - right
return _convert_signed_num(node)
return _convert(node_or_string)
def dump(
node, annotate_fields=True, include_attributes=False,
*,
indent=None, show_empty=False,
):
"""
Return a formatted dump of the tree in node. This is mainly useful for
debugging purposes. If annotate_fields is true (by default),
the returned string will show the names and the values for fields.
If annotate_fields is false, the result string will be more compact by
omitting unambiguous field names. Attributes such as line
numbers and column offsets are not dumped by default. If this is wanted,
include_attributes can be set to true. If indent is a non-negative
integer or string, then the tree will be pretty-printed with that indent
level. None (the default) selects the single line representation.
If show_empty is False, then empty lists and fields that are None
will be omitted from the output for better readability.
"""
def _format(node, level=0):
if indent is not None:
level += 1
prefix = '\n' + indent * level
sep = ',\n' + indent * level
else:
prefix = ''
sep = ', '
if isinstance(node, AST):
cls = type(node)
args = []
args_buffer = []
allsimple = True
keywords = annotate_fields
for name in node._fields:
try:
value = getattr(node, name)
except AttributeError:
keywords = True
continue
if value is None and getattr(cls, name, ...) is None:
keywords = True
continue
if not show_empty:
if value == []:
field_type = cls._field_types.get(name, object)
if getattr(field_type, '__origin__', ...) is list:
if not keywords:
args_buffer.append(repr(value))
continue
if not keywords:
args.extend(args_buffer)
args_buffer = []
value, simple = _format(value, level)
allsimple = allsimple and simple
if keywords:
args.append('%s=%s' % (name, value))
else:
args.append(value)
if include_attributes and node._attributes:
for name in node._attributes:
try:
value = getattr(node, name)
except AttributeError:
continue
if value is None and getattr(cls, name, ...) is None:
continue
value, simple = _format(value, level)
allsimple = allsimple and simple
args.append('%s=%s' % (name, value))
if allsimple and len(args) <= 3:
return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args
return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False
elif isinstance(node, list):
if not node:
return '[]', True
return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False
return repr(node), True
if not isinstance(node, AST):
raise TypeError('expected AST, got %r' % node.__class__.__name__)
if indent is not None and not isinstance(indent, str):
indent = ' ' * indent
return _format(node)[0]
def copy_location(new_node, old_node):
"""
Copy source location (`lineno`, `col_offset`, `end_lineno`, and `end_col_offset`
attributes) from *old_node* to *new_node* if possible, and return *new_node*.
"""
for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset':
if attr in old_node._attributes and attr in new_node._attributes:
value = getattr(old_node, attr, None)
# end_lineno and end_col_offset are optional attributes, and they
# should be copied whether the value is None or not.
if value is not None or (
hasattr(old_node, attr) and attr.startswith("end_")
):
setattr(new_node, attr, value)
return new_node
def fix_missing_locations(node):
"""
When you compile a node tree with compile(), the compiler expects lineno and
col_offset attributes for every node that supports them. This is rather
tedious to fill in for generated nodes, so this helper adds these attributes
recursively where not already set, by setting them to the values of the
parent node. It works recursively starting at *node*.
"""
def _fix(node, lineno, col_offset, end_lineno, end_col_offset):
if 'lineno' in node._attributes:
if not hasattr(node, 'lineno'):
node.lineno = lineno
else:
lineno = node.lineno
if 'end_lineno' in node._attributes:
if getattr(node, 'end_lineno', None) is None:
node.end_lineno = end_lineno
else:
end_lineno = node.end_lineno
if 'col_offset' in node._attributes:
if not hasattr(node, 'col_offset'):
node.col_offset = col_offset
else:
col_offset = node.col_offset
if 'end_col_offset' in node._attributes:
if getattr(node, 'end_col_offset', None) is None:
node.end_col_offset = end_col_offset
else:
end_col_offset = node.end_col_offset
for child in iter_child_nodes(node):
_fix(child, lineno, col_offset, end_lineno, end_col_offset)
_fix(node, 1, 0, 1, 0)
return node
def increment_lineno(node, n=1):
"""
Increment the line number and end line number of each node in the tree
starting at *node* by *n*. This is useful to "move code" to a different
location in a file.
"""
for child in walk(node):
# TypeIgnore is a special case where lineno is not an attribute
# but rather a field of the node itself.
if isinstance(child, TypeIgnore):
child.lineno = getattr(child, 'lineno', 0) + n
continue
if 'lineno' in child._attributes:
child.lineno = getattr(child, 'lineno', 0) + n
if (
"end_lineno" in child._attributes
and (end_lineno := getattr(child, "end_lineno", 0)) is not None
):
child.end_lineno = end_lineno + n
return node
def iter_fields(node):
"""
Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields``
that is present on *node*.
"""
for field in node._fields:
try:
yield field, getattr(node, field)
except AttributeError:
pass
def iter_child_nodes(node):
"""
Yield all direct child nodes of *node*, that is, all fields that are nodes
and all items of fields that are lists of nodes.
"""
for name, field in iter_fields(node):
if isinstance(field, AST):
yield field
elif isinstance(field, list):
for item in field:
if isinstance(item, AST):
yield item
def get_docstring(node, clean=True):
"""
Return the docstring for the given node or None if no docstring can
be found. If the node provided does not have docstrings a TypeError
will be raised.
If *clean* is `True`, all tabs are expanded to spaces and any whitespace
that can be uniformly removed from the second line onwards is removed.
"""
if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
raise TypeError("%r can't have docstrings" % node.__class__.__name__)
if not(node.body and isinstance(node.body[0], Expr)):
return None
node = node.body[0].value
if isinstance(node, Constant) and isinstance(node.value, str):
text = node.value
else:
return None
if clean:
import inspect
text = inspect.cleandoc(text)
return text
_line_pattern = None
def _splitlines_no_ff(source, maxlines=None):
"""Split a string into lines ignoring form feed and other chars.
This mimics how the Python parser splits source code.
"""
global _line_pattern
if _line_pattern is None:
# lazily computed to speedup import time of `ast`
import re
_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))")
lines = []
for lineno, match in enumerate(_line_pattern.finditer(source), 1):
if maxlines is not None and lineno > maxlines:
break
lines.append(match[0])
return lines
def _pad_whitespace(source):
r"""Replace all chars except '\f\t' in a line with spaces."""
result = ''
for c in source:
if c in '\f\t':
result += c
else:
result += ' '
return result
def get_source_segment(source, node, *, padded=False):
"""Get source code segment of the *source* that generated *node*.
If some location information (`lineno`, `end_lineno`, `col_offset`,
or `end_col_offset`) is missing, return None.
If *padded* is `True`, the first line of a multi-line statement will
be padded with spaces to match its original position.
"""
try:
if node.end_lineno is None or node.end_col_offset is None:
return None
lineno = node.lineno - 1
end_lineno = node.end_lineno - 1
col_offset = node.col_offset
end_col_offset = node.end_col_offset
except AttributeError:
return None
lines = _splitlines_no_ff(source, maxlines=end_lineno+1)
if end_lineno == lineno:
return lines[lineno].encode()[col_offset:end_col_offset].decode()
if padded:
padding = _pad_whitespace(lines[lineno].encode()[:col_offset].decode())
else:
padding = ''
first = padding + lines[lineno].encode()[col_offset:].decode()
last = lines[end_lineno].encode()[:end_col_offset].decode()
lines = lines[lineno+1:end_lineno]
lines.insert(0, first)
lines.append(last)
return ''.join(lines)
def walk(node):
"""
Recursively yield all descendant nodes in the tree starting at *node*
(including *node* itself), in no specified order. This is useful if you
only want to modify nodes in place and don't care about the context.
"""
from collections import deque
todo = deque([node])
while todo:
node = todo.popleft()
todo.extend(iter_child_nodes(node))
yield node
def compare(
a,
b,
/,
*,
compare_attributes=False,
):
"""Recursively compares two ASTs.
compare_attributes affects whether AST attributes are considered
in the comparison. If compare_attributes is False (default), then
attributes are ignored. Otherwise they must all be equal. This
option is useful to check whether the ASTs are structurally equal but
might differ in whitespace or similar details.
"""
sentinel = object() # handle the possibility of a missing attribute/field
def _compare(a, b):
# Compare two fields on an AST object, which may themselves be
# AST objects, lists of AST objects, or primitive ASDL types
# like identifiers and constants.
if isinstance(a, AST):
return compare(
a,
b,
compare_attributes=compare_attributes,
)
elif isinstance(a, list):
# If a field is repeated, then both objects will represent
# the value as a list.
if len(a) != len(b):
return False
for a_item, b_item in zip(a, b):
if not _compare(a_item, b_item):
return False
else:
return True
else:
return type(a) is type(b) and a == b
def _compare_fields(a, b):
if a._fields != b._fields:
return False
for field in a._fields:
a_field = getattr(a, field, sentinel)
b_field = getattr(b, field, sentinel)
if a_field is sentinel and b_field is sentinel:
# both nodes are missing a field at runtime
continue
if a_field is sentinel or b_field is sentinel:
# one of the node is missing a field
return False
if not _compare(a_field, b_field):
return False
else:
return True
def _compare_attributes(a, b):
if a._attributes != b._attributes:
return False
# Attributes are always ints.
for attr in a._attributes:
a_attr = getattr(a, attr, sentinel)
b_attr = getattr(b, attr, sentinel)
if a_attr is sentinel and b_attr is sentinel:
# both nodes are missing an attribute at runtime
continue
if a_attr != b_attr:
return False
else:
return True
if type(a) is not type(b):
return False
if not _compare_fields(a, b):
return False
if compare_attributes and not _compare_attributes(a, b):
return False
return True
class NodeVisitor(object):
"""
A node visitor base class that walks the abstract syntax tree and calls a
visitor function for every node found. This function may return a value
which is forwarded by the `visit` method.
This class is meant to be subclassed, with the subclass adding visitor
methods.
Per default the visitor functions for the nodes are ``'visit_'`` +
class name of the node. So a `TryFinally` node visit function would
be `visit_TryFinally`. This behavior can be changed by overriding
the `visit` method. If no visitor function exists for a node
(return value `None`) the `generic_visit` visitor is used instead.
Don't use the `NodeVisitor` if you want to apply changes to nodes during
traversing. For this a special visitor exists (`NodeTransformer`) that
allows modifications.
"""
def visit(self, node):
"""Visit a node."""
method = 'visit_' + node.__class__.__name__
visitor = getattr(self, method, self.generic_visit)
return visitor(node)
def generic_visit(self, node):
"""Called if no explicit visitor function exists for a node."""
for field, value in iter_fields(node):
if isinstance(value, list):
for item in value:
if isinstance(item, AST):
self.visit(item)
elif isinstance(value, AST):
self.visit(value)
class NodeTransformer(NodeVisitor):
"""
A :class:`NodeVisitor` subclass that walks the abstract syntax tree and
allows modification of nodes.
The `NodeTransformer` will walk the AST and use the return value of the
visitor methods to replace or remove the old node. If the return value of
the visitor method is ``None``, the node will be removed from its location,
otherwise it is replaced with the return value. The return value may be the
original node in which case no replacement takes place.
Here is an example transformer that rewrites all occurrences of name lookups
(``foo``) to ``data['foo']``::
class RewriteName(NodeTransformer):
def visit_Name(self, node):
return Subscript(
value=Name(id='data', ctx=Load()),
slice=Constant(value=node.id),
ctx=node.ctx
)
Keep in mind that if the node you're operating on has child nodes you must
either transform the child nodes yourself or call the :meth:`generic_visit`
method for the node first.
For nodes that were part of a collection of statements (that applies to all
statement nodes), the visitor may also return a list of nodes rather than
just a single node.
Usually you use the transformer like this::
node = YourTransformer().visit(node)
"""
def generic_visit(self, node):
for field, old_value in iter_fields(node):
if isinstance(old_value, list):
new_values = []
for value in old_value:
if isinstance(value, AST):
value = self.visit(value)
if value is None:
continue
elif not isinstance(value, AST):
new_values.extend(value)
continue
new_values.append(value)
old_value[:] = new_values
elif isinstance(old_value, AST):
new_node = self.visit(old_value)
if new_node is None:
delattr(node, field)
else:
setattr(node, field, new_node)
return node
class slice(AST):
"""Deprecated AST node class."""
class Index(slice):
"""Deprecated AST node class. Use the index value directly instead."""
def __new__(cls, value, **kwargs):
return value
class ExtSlice(slice):
"""Deprecated AST node class. Use ast.Tuple instead."""
def __new__(cls, dims=(), **kwargs):
return Tuple(list(dims), Load(), **kwargs)
# If the ast module is loaded more than once, only add deprecated methods once
if not hasattr(Tuple, 'dims'):
# The following code is for backward compatibility.
# It will be removed in future.
def _dims_getter(self):
"""Deprecated. Use elts instead."""
return self.elts
def _dims_setter(self, value):
self.elts = value
Tuple.dims = property(_dims_getter, _dims_setter)
class Suite(mod):
"""Deprecated AST node class. Unused in Python 3."""
class AugLoad(expr_context):
"""Deprecated AST node class. Unused in Python 3."""
class AugStore(expr_context):
"""Deprecated AST node class. Unused in Python 3."""
class Param(expr_context):
"""Deprecated AST node class. Unused in Python 3."""
def unparse(ast_obj):
global _Unparser
try:
unparser = _Unparser()
except NameError:
from _ast_unparse import Unparser as _Unparser
unparser = _Unparser()
return unparser.visit(ast_obj)
def main(args=None):
import argparse
import sys
parser = argparse.ArgumentParser(color=True)
parser.add_argument('infile', nargs='?', default='-',
help='the file to parse; defaults to stdin')
parser.add_argument('-m', '--mode', default='exec',
choices=('exec', 'single', 'eval', 'func_type'),
help='specify what kind of code must be parsed')
parser.add_argument('--no-type-comments', default=True, action='store_false',
help="don't add information about type comments")
parser.add_argument('-a', '--include-attributes', action='store_true',
help='include attributes such as line numbers and '
'column offsets')
parser.add_argument('-i', '--indent', type=int, default=3,
help='indentation of nodes (number of spaces)')
parser.add_argument('--feature-version',
type=str, default=None, metavar='VERSION',
help='Python version in the format 3.x '
'(for example, 3.10)')
parser.add_argument('-O', '--optimize',
type=int, default=-1, metavar='LEVEL',
help='optimization level for parser (default -1)')
parser.add_argument('--show-empty', default=False, action='store_true',
help='show empty lists and fields in dump output')
args = parser.parse_args(args)
if args.infile == '-':
name = '<stdin>'
source = sys.stdin.buffer.read()
else:
name = args.infile
with open(args.infile, 'rb') as infile:
source = infile.read()
# Process feature_version
feature_version = None
if args.feature_version:
try:
major, minor = map(int, args.feature_version.split('.', 1))
except ValueError:
parser.error('Invalid format for --feature-version; '
'expected format 3.x (for example, 3.10)')
feature_version = (major, minor)
tree = parse(source, name, args.mode, type_comments=args.no_type_comments,
feature_version=feature_version, optimize=args.optimize)
print(dump(tree, include_attributes=args.include_attributes,
indent=args.indent, show_empty=args.show_empty))
if __name__ == '__main__':
main()

View file

@ -0,0 +1,618 @@
"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
# Modified 04-Oct-1995 by Jack Jansen to use binascii module
# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere
import struct
import binascii
__all__ = [
# Legacy interface exports traditional RFC 2045 Base64 encodings
'encode', 'decode', 'encodebytes', 'decodebytes',
# Generalized interface for other encodings
'b64encode', 'b64decode', 'b32encode', 'b32decode',
'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode',
# Base85 and Ascii85 encodings
'b85encode', 'b85decode', 'a85encode', 'a85decode', 'z85encode', 'z85decode',
# Standard Base64 encoding
'standard_b64encode', 'standard_b64decode',
# Some common Base64 alternatives. As referenced by RFC 3458, see thread
# starting at:
#
# http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html
'urlsafe_b64encode', 'urlsafe_b64decode',
]
bytes_types = (bytes, bytearray) # Types acceptable as binary data
def _bytes_from_decode_data(s):
if isinstance(s, str):
try:
return s.encode('ascii')
except UnicodeEncodeError:
raise ValueError('string argument should contain only ASCII characters')
if isinstance(s, bytes_types):
return s
try:
return memoryview(s).tobytes()
except TypeError:
raise TypeError("argument should be a bytes-like object or ASCII "
"string, not %r" % s.__class__.__name__) from None
# Base64 encoding/decoding uses binascii
def b64encode(s, altchars=None):
"""Encode the bytes-like object s using Base64 and return a bytes object.
Optional altchars should be a byte string of length 2 which specifies an
alternative alphabet for the '+' and '/' characters. This allows an
application to e.g. generate url or filesystem safe Base64 strings.
"""
encoded = binascii.b2a_base64(s, newline=False)
if altchars is not None:
assert len(altchars) == 2, repr(altchars)
return encoded.translate(bytes.maketrans(b'+/', altchars))
return encoded
def b64decode(s, altchars=None, validate=False):
"""Decode the Base64 encoded bytes-like object or ASCII string s.
Optional altchars must be a bytes-like object or ASCII string of length 2
which specifies the alternative alphabet used instead of the '+' and '/'
characters.
The result is returned as a bytes object. A binascii.Error is raised if
s is incorrectly padded.
If validate is False (the default), characters that are neither in the
normal base-64 alphabet nor the alternative alphabet are discarded prior
to the padding check. If validate is True, these non-alphabet characters
in the input result in a binascii.Error.
For more information about the strict base64 check, see:
https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64
"""
s = _bytes_from_decode_data(s)
if altchars is not None:
altchars = _bytes_from_decode_data(altchars)
assert len(altchars) == 2, repr(altchars)
s = s.translate(bytes.maketrans(altchars, b'+/'))
return binascii.a2b_base64(s, strict_mode=validate)
def standard_b64encode(s):
"""Encode bytes-like object s using the standard Base64 alphabet.
The result is returned as a bytes object.
"""
return b64encode(s)
def standard_b64decode(s):
"""Decode bytes encoded with the standard Base64 alphabet.
Argument s is a bytes-like object or ASCII string to decode. The result
is returned as a bytes object. A binascii.Error is raised if the input
is incorrectly padded. Characters that are not in the standard alphabet
are discarded prior to the padding check.
"""
return b64decode(s)
_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_')
_urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/')
def urlsafe_b64encode(s):
"""Encode bytes using the URL- and filesystem-safe Base64 alphabet.
Argument s is a bytes-like object to encode. The result is returned as a
bytes object. The alphabet uses '-' instead of '+' and '_' instead of
'/'.
"""
return b64encode(s).translate(_urlsafe_encode_translation)
def urlsafe_b64decode(s):
"""Decode bytes using the URL- and filesystem-safe Base64 alphabet.
Argument s is a bytes-like object or ASCII string to decode. The result
is returned as a bytes object. A binascii.Error is raised if the input
is incorrectly padded. Characters that are not in the URL-safe base-64
alphabet, and are not a plus '+' or slash '/', are discarded prior to the
padding check.
The alphabet uses '-' instead of '+' and '_' instead of '/'.
"""
s = _bytes_from_decode_data(s)
s = s.translate(_urlsafe_decode_translation)
return b64decode(s)
# Base32 encoding/decoding must be done in Python
_B32_ENCODE_DOCSTRING = '''
Encode the bytes-like objects using {encoding} and return a bytes object.
'''
_B32_DECODE_DOCSTRING = '''
Decode the {encoding} encoded bytes-like object or ASCII string s.
Optional casefold is a flag specifying whether a lowercase alphabet is
acceptable as input. For security purposes, the default is False.
{extra_args}
The result is returned as a bytes object. A binascii.Error is raised if
the input is incorrectly padded or if there are non-alphabet
characters present in the input.
'''
_B32_DECODE_MAP01_DOCSTRING = '''
RFC 3548 allows for optional mapping of the digit 0 (zero) to the
letter O (oh), and for optional mapping of the digit 1 (one) to
either the letter I (eye) or letter L (el). The optional argument
map01 when not None, specifies which letter the digit 1 should be
mapped to (when map01 is not None, the digit 0 is always mapped to
the letter O). For security purposes the default is None, so that
0 and 1 are not allowed in the input.
'''
_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV'
_b32tab2 = {}
_b32rev = {}
def _b32encode(alphabet, s):
# Delay the initialization of the table to not waste memory
# if the function is never called
if alphabet not in _b32tab2:
b32tab = [bytes((i,)) for i in alphabet]
_b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab]
b32tab = None
if not isinstance(s, bytes_types):
s = memoryview(s).tobytes()
leftover = len(s) % 5
# Pad the last quantum with zero bits if necessary
if leftover:
s = s + b'\0' * (5 - leftover) # Don't use += !
encoded = bytearray()
from_bytes = int.from_bytes
b32tab2 = _b32tab2[alphabet]
for i in range(0, len(s), 5):
c = from_bytes(s[i: i + 5]) # big endian
encoded += (b32tab2[c >> 30] + # bits 1 - 10
b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
b32tab2[c & 0x3ff] # bits 31 - 40
)
# Adjust for any leftover partial quanta
if leftover == 1:
encoded[-6:] = b'======'
elif leftover == 2:
encoded[-4:] = b'===='
elif leftover == 3:
encoded[-3:] = b'==='
elif leftover == 4:
encoded[-1:] = b'='
return bytes(encoded)
def _b32decode(alphabet, s, casefold=False, map01=None):
# Delay the initialization of the table to not waste memory
# if the function is never called
if alphabet not in _b32rev:
_b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)}
s = _bytes_from_decode_data(s)
if len(s) % 8:
raise binascii.Error('Incorrect padding')
# Handle section 2.4 zero and one mapping. The flag map01 will be either
# False, or the character to map the digit 1 (one) to. It should be
# either L (el) or I (eye).
if map01 is not None:
map01 = _bytes_from_decode_data(map01)
assert len(map01) == 1, repr(map01)
s = s.translate(bytes.maketrans(b'01', b'O' + map01))
if casefold:
s = s.upper()
# Strip off pad characters from the right. We need to count the pad
# characters because this will tell us how many null bytes to remove from
# the end of the decoded string.
l = len(s)
s = s.rstrip(b'=')
padchars = l - len(s)
# Now decode the full quanta
decoded = bytearray()
b32rev = _b32rev[alphabet]
for i in range(0, len(s), 8):
quanta = s[i: i + 8]
acc = 0
try:
for c in quanta:
acc = (acc << 5) + b32rev[c]
except KeyError:
raise binascii.Error('Non-base32 digit found') from None
decoded += acc.to_bytes(5) # big endian
# Process the last, partial quanta
if l % 8 or padchars not in {0, 1, 3, 4, 6}:
raise binascii.Error('Incorrect padding')
if padchars and decoded:
acc <<= 5 * padchars
last = acc.to_bytes(5) # big endian
leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1
decoded[-5:] = last[:leftover]
return bytes(decoded)
def b32encode(s):
return _b32encode(_b32alphabet, s)
b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32')
def b32decode(s, casefold=False, map01=None):
return _b32decode(_b32alphabet, s, casefold, map01)
b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32',
extra_args=_B32_DECODE_MAP01_DOCSTRING)
def b32hexencode(s):
return _b32encode(_b32hexalphabet, s)
b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex')
def b32hexdecode(s, casefold=False):
# base32hex does not have the 01 mapping
return _b32decode(_b32hexalphabet, s, casefold)
b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex',
extra_args='')
# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
# lowercase. The RFC also recommends against accepting input case
# insensitively.
def b16encode(s):
"""Encode the bytes-like object s using Base16 and return a bytes object.
"""
return binascii.hexlify(s).upper()
def b16decode(s, casefold=False):
"""Decode the Base16 encoded bytes-like object or ASCII string s.
Optional casefold is a flag specifying whether a lowercase alphabet is
acceptable as input. For security purposes, the default is False.
The result is returned as a bytes object. A binascii.Error is raised if
s is incorrectly padded or if there are non-alphabet characters present
in the input.
"""
s = _bytes_from_decode_data(s)
if casefold:
s = s.upper()
if s.translate(None, delete=b'0123456789ABCDEF'):
raise binascii.Error('Non-base16 digit found')
return binascii.unhexlify(s)
#
# Ascii85 encoding/decoding
#
_a85chars = None
_a85chars2 = None
_A85START = b"<~"
_A85END = b"~>"
def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False):
# Helper function for a85encode and b85encode
if not isinstance(b, bytes_types):
b = memoryview(b).tobytes()
padding = (-len(b)) % 4
if padding:
b = b + b'\0' * padding
words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b)
chunks = [b'z' if foldnuls and not word else
b'y' if foldspaces and word == 0x20202020 else
(chars2[word // 614125] +
chars2[word // 85 % 7225] +
chars[word % 85])
for word in words]
if padding and not pad:
if chunks[-1] == b'z':
chunks[-1] = chars[0] * 5
chunks[-1] = chunks[-1][:-padding]
return b''.join(chunks)
def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
"""Encode bytes-like object b using Ascii85 and return a bytes object.
foldspaces is an optional flag that uses the special short sequence 'y'
instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This
feature is not supported by the "standard" Adobe encoding.
wrapcol controls whether the output should have newline (b'\\n') characters
added to it. If this is non-zero, each output line will be at most this
many characters long, excluding the trailing newline.
pad controls whether the input is padded to a multiple of 4 before
encoding. Note that the btoa implementation always pads.
adobe controls whether the encoded byte sequence is framed with <~ and ~>,
which is used by the Adobe implementation.
"""
global _a85chars, _a85chars2
# Delay the initialization of tables to not waste memory
# if the function is never called
if _a85chars2 is None:
_a85chars = [bytes((i,)) for i in range(33, 118)]
_a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces)
if adobe:
result = _A85START + result
if wrapcol:
wrapcol = max(2 if adobe else 1, wrapcol)
chunks = [result[i: i + wrapcol]
for i in range(0, len(result), wrapcol)]
if adobe:
if len(chunks[-1]) + 2 > wrapcol:
chunks.append(b'')
result = b'\n'.join(chunks)
if adobe:
result += _A85END
return result
def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'):
"""Decode the Ascii85 encoded bytes-like object or ASCII string b.
foldspaces is a flag that specifies whether the 'y' short sequence should be
accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is
not supported by the "standard" Adobe encoding.
adobe controls whether the input sequence is in Adobe Ascii85 format (i.e.
is framed with <~ and ~>).
ignorechars should be a byte string containing characters to ignore from the
input. This should only contain whitespace characters, and by default
contains all whitespace characters in ASCII.
The result is returned as a bytes object.
"""
b = _bytes_from_decode_data(b)
if adobe:
if not b.endswith(_A85END):
raise ValueError(
"Ascii85 encoded byte sequences must end "
"with {!r}".format(_A85END)
)
if b.startswith(_A85START):
b = b[2:-2] # Strip off start/end markers
else:
b = b[:-2]
#
# We have to go through this stepwise, so as to ignore spaces and handle
# special short sequences
#
packI = struct.Struct('!I').pack
decoded = []
decoded_append = decoded.append
curr = []
curr_append = curr.append
curr_clear = curr.clear
for x in b + b'u' * 4:
if b'!'[0] <= x <= b'u'[0]:
curr_append(x)
if len(curr) == 5:
acc = 0
for x in curr:
acc = 85 * acc + (x - 33)
try:
decoded_append(packI(acc))
except struct.error:
raise ValueError('Ascii85 overflow') from None
curr_clear()
elif x == b'z'[0]:
if curr:
raise ValueError('z inside Ascii85 5-tuple')
decoded_append(b'\0\0\0\0')
elif foldspaces and x == b'y'[0]:
if curr:
raise ValueError('y inside Ascii85 5-tuple')
decoded_append(b'\x20\x20\x20\x20')
elif x in ignorechars:
# Skip whitespace
continue
else:
raise ValueError('Non-Ascii85 digit found: %c' % x)
result = b''.join(decoded)
padding = 4 - len(curr)
if padding:
# Throw away the extra padding
result = result[:-padding]
return result
# The following code is originally taken (with permission) from Mercurial
_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~")
_b85chars = None
_b85chars2 = None
_b85dec = None
def b85encode(b, pad=False):
"""Encode bytes-like object b in base85 format and return a bytes object.
If pad is true, the input is padded with b'\\0' so its length is a multiple of
4 bytes before encoding.
"""
global _b85chars, _b85chars2
# Delay the initialization of tables to not waste memory
# if the function is never called
if _b85chars2 is None:
_b85chars = [bytes((i,)) for i in _b85alphabet]
_b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
return _85encode(b, _b85chars, _b85chars2, pad)
def b85decode(b):
"""Decode the base85-encoded bytes-like object or ASCII string b
The result is returned as a bytes object.
"""
global _b85dec
# Delay the initialization of tables to not waste memory
# if the function is never called
if _b85dec is None:
# we don't assign to _b85dec directly to avoid issues when
# multiple threads call this function simultaneously
b85dec_tmp = [None] * 256
for i, c in enumerate(_b85alphabet):
b85dec_tmp[c] = i
_b85dec = b85dec_tmp
b = _bytes_from_decode_data(b)
padding = (-len(b)) % 5
b = b + b'~' * padding
out = []
packI = struct.Struct('!I').pack
for i in range(0, len(b), 5):
chunk = b[i:i + 5]
acc = 0
try:
for c in chunk:
acc = acc * 85 + _b85dec[c]
except TypeError:
for j, c in enumerate(chunk):
if _b85dec[c] is None:
raise ValueError('bad base85 character at position %d'
% (i + j)) from None
raise
try:
out.append(packI(acc))
except struct.error:
raise ValueError('base85 overflow in hunk starting at byte %d'
% i) from None
result = b''.join(out)
if padding:
result = result[:-padding]
return result
_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz'
b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#')
# Translating b85 valid but z85 invalid chars to b'\x00' is required
# to prevent them from being decoded as b85 valid chars.
_z85_b85_decode_diff = b';_`|~'
_z85_decode_translation = bytes.maketrans(
_z85alphabet + _z85_b85_decode_diff,
_b85alphabet + b'\x00' * len(_z85_b85_decode_diff)
)
_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet)
def z85encode(s):
"""Encode bytes-like object b in z85 format and return a bytes object."""
return b85encode(s).translate(_z85_encode_translation)
def z85decode(s):
"""Decode the z85-encoded bytes-like object or ASCII string b
The result is returned as a bytes object.
"""
s = _bytes_from_decode_data(s)
s = s.translate(_z85_decode_translation)
try:
return b85decode(s)
except ValueError as e:
raise ValueError(e.args[0].replace('base85', 'z85')) from None
# Legacy interface. This code could be cleaned up since I don't believe
# binascii has any line length limitations. It just doesn't seem worth it
# though. The files should be opened in binary mode.
MAXLINESIZE = 76 # Excluding the CRLF
MAXBINSIZE = (MAXLINESIZE//4)*3
def encode(input, output):
"""Encode a file; input and output are binary files."""
while s := input.read(MAXBINSIZE):
while len(s) < MAXBINSIZE and (ns := input.read(MAXBINSIZE-len(s))):
s += ns
line = binascii.b2a_base64(s)
output.write(line)
def decode(input, output):
"""Decode a file; input and output are binary files."""
while line := input.readline():
s = binascii.a2b_base64(line)
output.write(s)
def _input_type_check(s):
try:
m = memoryview(s)
except TypeError as err:
msg = "expected bytes-like object, not %s" % s.__class__.__name__
raise TypeError(msg) from err
if m.format not in ('c', 'b', 'B'):
msg = ("expected single byte elements, not %r from %s" %
(m.format, s.__class__.__name__))
raise TypeError(msg)
if m.ndim != 1:
msg = ("expected 1-D data, not %d-D data from %s" %
(m.ndim, s.__class__.__name__))
raise TypeError(msg)
def encodebytes(s):
"""Encode a bytestring into a bytes object containing multiple lines
of base-64 data."""
_input_type_check(s)
pieces = []
for i in range(0, len(s), MAXBINSIZE):
chunk = s[i : i + MAXBINSIZE]
pieces.append(binascii.b2a_base64(chunk))
return b"".join(pieces)
def decodebytes(s):
"""Decode a bytestring of base-64 data into a bytes object."""
_input_type_check(s)
return binascii.a2b_base64(s)
# Usable as a script...
def main():
"""Small main program"""
import sys, getopt
usage = f"""usage: {sys.argv[0]} [-h|-d|-e|-u] [file|-]
-h: print this help message and exit
-d, -u: decode
-e: encode (default)"""
try:
opts, args = getopt.getopt(sys.argv[1:], 'hdeu')
except getopt.error as msg:
sys.stdout = sys.stderr
print(msg)
print(usage)
sys.exit(2)
func = encode
for o, a in opts:
if o == '-e': func = encode
if o == '-d': func = decode
if o == '-u': func = decode
if o == '-h': print(usage); return
if args and args[0] != '-':
with open(args[0], 'rb') as f:
func(f, sys.stdout.buffer)
else:
if sys.stdin.isatty():
# gh-138775: read terminal input data all at once to detect EOF
import io
data = sys.stdin.buffer.read()
buffer = io.BytesIO(data)
else:
buffer = sys.stdin.buffer
func(buffer, sys.stdout.buffer)
if __name__ == '__main__':
main()

View file

@ -0,0 +1,118 @@
"""Bisection algorithms."""
def insort_right(a, x, lo=0, hi=None, *, key=None):
"""Insert item x in list a, and keep it sorted assuming a is sorted.
If x is already in a, insert it to the right of the rightmost x.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
A custom key function can be supplied to customize the sort order.
"""
if key is None:
lo = bisect_right(a, x, lo, hi)
else:
lo = bisect_right(a, key(x), lo, hi, key=key)
a.insert(lo, x)
def bisect_right(a, x, lo=0, hi=None, *, key=None):
"""Return the index where to insert item x in list a, assuming a is sorted.
The return value i is such that all e in a[:i] have e <= x, and all e in
a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will
insert just after the rightmost x already there.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
A custom key function can be supplied to customize the sort order.
"""
if lo < 0:
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
# Note, the comparison uses "<" to match the
# __lt__() logic in list.sort() and in heapq.
if key is None:
while lo < hi:
mid = (lo + hi) // 2
if x < a[mid]:
hi = mid
else:
lo = mid + 1
else:
while lo < hi:
mid = (lo + hi) // 2
if x < key(a[mid]):
hi = mid
else:
lo = mid + 1
return lo
def insort_left(a, x, lo=0, hi=None, *, key=None):
"""Insert item x in list a, and keep it sorted assuming a is sorted.
If x is already in a, insert it to the left of the leftmost x.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
A custom key function can be supplied to customize the sort order.
"""
if key is None:
lo = bisect_left(a, x, lo, hi)
else:
lo = bisect_left(a, key(x), lo, hi, key=key)
a.insert(lo, x)
def bisect_left(a, x, lo=0, hi=None, *, key=None):
"""Return the index where to insert item x in list a, assuming a is sorted.
The return value i is such that all e in a[:i] have e < x, and all e in
a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will
insert just before the leftmost x already there.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
A custom key function can be supplied to customize the sort order.
"""
if lo < 0:
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
# Note, the comparison uses "<" to match the
# __lt__() logic in list.sort() and in heapq.
if key is None:
while lo < hi:
mid = (lo + hi) // 2
if a[mid] < x:
lo = mid + 1
else:
hi = mid
else:
while lo < hi:
mid = (lo + hi) // 2
if key(a[mid]) < x:
lo = mid + 1
else:
hi = mid
return lo
# Overwrite above definitions with a fast C implementation
try:
from _bisect import *
except ImportError:
pass
# Create aliases
bisect = bisect_right
insort = insort_right

View file

@ -0,0 +1,396 @@
"""Utilities needed to emulate Python's interactive interpreter.
"""
# Inspired by similar code by Jeff Epler and Fredrik Lundh.
import builtins
import sys
import traceback
from codeop import CommandCompiler, compile_command
__all__ = ["InteractiveInterpreter", "InteractiveConsole", "interact",
"compile_command"]
class InteractiveInterpreter:
"""Base class for InteractiveConsole.
This class deals with parsing and interpreter state (the user's
namespace); it doesn't deal with input buffering or prompting or
input file naming (the filename is always passed in explicitly).
"""
def __init__(self, locals=None):
"""Constructor.
The optional 'locals' argument specifies a mapping to use as the
namespace in which code will be executed; it defaults to a newly
created dictionary with key "__name__" set to "__console__" and
key "__doc__" set to None.
"""
if locals is None:
locals = {"__name__": "__console__", "__doc__": None}
self.locals = locals
self.compile = CommandCompiler()
def runsource(self, source, filename="<input>", symbol="single"):
"""Compile and run some source in the interpreter.
Arguments are as for compile_command().
One of several things can happen:
1) The input is incorrect; compile_command() raised an
exception (SyntaxError or OverflowError). A syntax traceback
will be printed by calling the showsyntaxerror() method.
2) The input is incomplete, and more input is required;
compile_command() returned None. Nothing happens.
3) The input is complete; compile_command() returned a code
object. The code is executed by calling self.runcode() (which
also handles run-time exceptions, except for SystemExit).
The return value is True in case 2, False in the other cases (unless
an exception is raised). The return value can be used to
decide whether to use sys.ps1 or sys.ps2 to prompt the next
line.
"""
try:
code = self.compile(source, filename, symbol)
except (OverflowError, SyntaxError, ValueError):
# Case 1
self.showsyntaxerror(filename, source=source)
return False
if code is None:
# Case 2
return True
# Case 3
self.runcode(code)
return False
def runcode(self, code):
"""Execute a code object.
When an exception occurs, self.showtraceback() is called to
display a traceback. All exceptions are caught except
SystemExit, which is reraised.
A note about KeyboardInterrupt: this exception may occur
elsewhere in this code, and may not always be caught. The
caller should be prepared to deal with it.
"""
try:
exec(code, self.locals)
except SystemExit:
raise
except:
self.showtraceback()
def showsyntaxerror(self, filename=None, **kwargs):
"""Display the syntax error that just occurred.
This doesn't display a stack trace because there isn't one.
If a filename is given, it is stuffed in the exception instead
of what was there before (because Python's parser always uses
"<string>" when reading from a string).
The output is written by self.write(), below.
"""
try:
typ, value, tb = sys.exc_info()
if filename and issubclass(typ, SyntaxError):
value.filename = filename
source = kwargs.pop('source', "")
self._showtraceback(typ, value, None, source)
finally:
typ = value = tb = None
def showtraceback(self):
"""Display the exception that just occurred.
We remove the first stack item because it is our own code.
The output is written by self.write(), below.
"""
try:
typ, value, tb = sys.exc_info()
self._showtraceback(typ, value, tb.tb_next, "")
finally:
typ = value = tb = None
def _showtraceback(self, typ, value, tb, source):
sys.last_type = typ
sys.last_traceback = tb
value = value.with_traceback(tb)
# Set the line of text that the exception refers to
lines = source.splitlines()
if (source and typ is SyntaxError
and not value.text and value.lineno is not None
and len(lines) >= value.lineno):
value.text = lines[value.lineno - 1]
sys.last_exc = sys.last_value = value
if sys.excepthook is sys.__excepthook__:
self._excepthook(typ, value, tb)
else:
# If someone has set sys.excepthook, we let that take precedence
# over self.write
try:
sys.excepthook(typ, value, tb)
except SystemExit:
raise
except BaseException as e:
e.__context__ = None
e = e.with_traceback(e.__traceback__.tb_next)
print('Error in sys.excepthook:', file=sys.stderr)
sys.__excepthook__(type(e), e, e.__traceback__)
print(file=sys.stderr)
print('Original exception was:', file=sys.stderr)
sys.__excepthook__(typ, value, tb)
def _excepthook(self, typ, value, tb):
# This method is being overwritten in
# _pyrepl.console.InteractiveColoredConsole
lines = traceback.format_exception(typ, value, tb)
self.write(''.join(lines))
def write(self, data):
"""Write a string.
The base implementation writes to sys.stderr; a subclass may
replace this with a different implementation.
"""
sys.stderr.write(data)
class InteractiveConsole(InteractiveInterpreter):
"""Closely emulate the behavior of the interactive Python interpreter.
This class builds on InteractiveInterpreter and adds prompting
using the familiar sys.ps1 and sys.ps2, and input buffering.
"""
def __init__(self, locals=None, filename="<console>", *, local_exit=False):
"""Constructor.
The optional locals argument will be passed to the
InteractiveInterpreter base class.
The optional filename argument should specify the (file)name
of the input stream; it will show up in tracebacks.
"""
InteractiveInterpreter.__init__(self, locals)
self.filename = filename
self.local_exit = local_exit
self.resetbuffer()
def resetbuffer(self):
"""Reset the input buffer."""
self.buffer = []
def interact(self, banner=None, exitmsg=None):
"""Closely emulate the interactive Python console.
The optional banner argument specifies the banner to print
before the first interaction; by default it prints a banner
similar to the one printed by the real Python interpreter,
followed by the current class name in parentheses (so as not
to confuse this with the real interpreter -- since it's so
close!).
The optional exitmsg argument specifies the exit message
printed when exiting. Pass the empty string to suppress
printing an exit message. If exitmsg is not given or None,
a default message is printed.
"""
try:
sys.ps1
delete_ps1_after = False
except AttributeError:
sys.ps1 = ">>> "
delete_ps1_after = True
try:
_ps2 = sys.ps2
delete_ps2_after = False
except AttributeError:
sys.ps2 = "... "
delete_ps2_after = True
cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
if banner is None:
self.write("Python %s on %s\n%s\n(%s)\n" %
(sys.version, sys.platform, cprt,
self.__class__.__name__))
elif banner:
self.write("%s\n" % str(banner))
more = 0
# When the user uses exit() or quit() in their interactive shell
# they probably just want to exit the created shell, not the whole
# process. exit and quit in builtins closes sys.stdin which makes
# it super difficult to restore
#
# When self.local_exit is True, we overwrite the builtins so
# exit() and quit() only raises SystemExit and we can catch that
# to only exit the interactive shell
_exit = None
_quit = None
if self.local_exit:
if hasattr(builtins, "exit"):
_exit = builtins.exit
builtins.exit = Quitter("exit")
if hasattr(builtins, "quit"):
_quit = builtins.quit
builtins.quit = Quitter("quit")
try:
while True:
try:
if more:
prompt = sys.ps2
else:
prompt = sys.ps1
try:
line = self.raw_input(prompt)
except EOFError:
self.write("\n")
break
else:
more = self.push(line)
except KeyboardInterrupt:
self.write("\nKeyboardInterrupt\n")
self.resetbuffer()
more = 0
except SystemExit as e:
if self.local_exit:
self.write("\n")
break
else:
raise e
finally:
# restore exit and quit in builtins if they were modified
if _exit is not None:
builtins.exit = _exit
if _quit is not None:
builtins.quit = _quit
if delete_ps1_after:
del sys.ps1
if delete_ps2_after:
del sys.ps2
if exitmsg is None:
self.write('now exiting %s...\n' % self.__class__.__name__)
elif exitmsg != '':
self.write('%s\n' % exitmsg)
def push(self, line, filename=None, _symbol="single"):
"""Push a line to the interpreter.
The line should not have a trailing newline; it may have
internal newlines. The line is appended to a buffer and the
interpreter's runsource() method is called with the
concatenated contents of the buffer as source. If this
indicates that the command was executed or invalid, the buffer
is reset; otherwise, the command is incomplete, and the buffer
is left as it was after the line was appended. The return
value is 1 if more input is required, 0 if the line was dealt
with in some way (this is the same as runsource()).
"""
self.buffer.append(line)
source = "\n".join(self.buffer)
if filename is None:
filename = self.filename
more = self.runsource(source, filename, symbol=_symbol)
if not more:
self.resetbuffer()
return more
def raw_input(self, prompt=""):
"""Write a prompt and read a line.
The returned line does not include the trailing newline.
When the user enters the EOF key sequence, EOFError is raised.
The base implementation uses the built-in function
input(); a subclass may replace this with a different
implementation.
"""
return input(prompt)
class Quitter:
def __init__(self, name):
self.name = name
if sys.platform == "win32":
self.eof = 'Ctrl-Z plus Return'
else:
self.eof = 'Ctrl-D (i.e. EOF)'
def __repr__(self):
return f'Use {self.name} or {self.eof} to exit'
def __call__(self, code=None):
raise SystemExit(code)
def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False):
"""Closely emulate the interactive Python interpreter.
This is a backwards compatible interface to the InteractiveConsole
class. When readfunc is not specified, it attempts to import the
readline module to enable GNU readline if it is available.
Arguments (all optional, all default to None):
banner -- passed to InteractiveConsole.interact()
readfunc -- if not None, replaces InteractiveConsole.raw_input()
local -- passed to InteractiveInterpreter.__init__()
exitmsg -- passed to InteractiveConsole.interact()
local_exit -- passed to InteractiveConsole.__init__()
"""
console = InteractiveConsole(local, local_exit=local_exit)
if readfunc is not None:
console.raw_input = readfunc
else:
try:
import readline # noqa: F401
except ImportError:
pass
console.interact(banner, exitmsg)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(color=True)
parser.add_argument('-q', action='store_true',
help="don't print version and copyright messages")
args = parser.parse_args()
if args.q or sys.flags.quiet:
banner = ''
else:
banner = None
interact(banner)

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,154 @@
r"""Utilities to compile possibly incomplete Python source code.
This module provides two interfaces, broadly similar to the builtin
function compile(), which take program text, a filename and a 'mode'
and:
- Return code object if the command is complete and valid
- Return None if the command is incomplete
- Raise SyntaxError, ValueError or OverflowError if the command is a
syntax error (OverflowError and ValueError can be produced by
malformed literals).
The two interfaces are:
compile_command(source, filename, symbol):
Compiles a single command in the manner described above.
CommandCompiler():
Instances of this class have __call__ methods identical in
signature to compile_command; the difference is that if the
instance compiles program text containing a __future__ statement,
the instance 'remembers' and compiles all subsequent program texts
with the statement in force.
The module also provides another class:
Compile():
Instances of this class act like the built-in function compile,
but with 'memory' in the sense described above.
"""
import __future__
import warnings
_features = [getattr(__future__, fname)
for fname in __future__.all_feature_names]
__all__ = ["compile_command", "Compile", "CommandCompiler"]
# The following flags match the values from Include/cpython/compile.h
# Caveat emptor: These flags are undocumented on purpose and depending
# on their effect outside the standard library is **unsupported**.
PyCF_DONT_IMPLY_DEDENT = 0x200
PyCF_ONLY_AST = 0x400
PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
def _maybe_compile(compiler, source, filename, symbol, flags):
# Check for source consisting of only blank lines and comments.
for line in source.split("\n"):
line = line.strip()
if line and line[0] != '#':
break # Leave it alone.
else:
if symbol != "eval":
source = "pass" # Replace it with a 'pass' statement
# Disable compiler warnings when checking for incomplete input.
with warnings.catch_warnings():
warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
try:
compiler(source, filename, symbol, flags=flags)
except SyntaxError: # Let other compile() errors propagate.
try:
compiler(source + "\n", filename, symbol, flags=flags)
return None
except _IncompleteInputError as e:
return None
except SyntaxError as e:
pass
# fallthrough
return compiler(source, filename, symbol, incomplete_input=False)
def _compile(source, filename, symbol, incomplete_input=True, *, flags=0):
if incomplete_input:
flags |= PyCF_ALLOW_INCOMPLETE_INPUT
flags |= PyCF_DONT_IMPLY_DEDENT
return compile(source, filename, symbol, flags)
def compile_command(source, filename="<input>", symbol="single", flags=0):
r"""Compile a command and determine whether it is incomplete.
Arguments:
source -- the source string; may contain \n characters
filename -- optional filename from which source was read; default
"<input>"
symbol -- optional grammar start symbol; "single" (default), "exec"
or "eval"
Return value / exceptions raised:
- Return a code object if the command is complete and valid
- Return None if the command is incomplete
- Raise SyntaxError, ValueError or OverflowError if the command is a
syntax error (OverflowError and ValueError can be produced by
malformed literals).
"""
return _maybe_compile(_compile, source, filename, symbol, flags)
class Compile:
"""Instances of this class behave much like the built-in compile
function, but if one is used to compile text containing a future
statement, it "remembers" and compiles all subsequent program texts
with the statement in force."""
def __init__(self):
self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
def __call__(self, source, filename, symbol, flags=0, **kwargs):
flags |= self.flags
if kwargs.get('incomplete_input', True) is False:
flags &= ~PyCF_DONT_IMPLY_DEDENT
flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT
codeob = compile(source, filename, symbol, flags, True)
if flags & PyCF_ONLY_AST:
return codeob # this is an ast.Module in this case
for feature in _features:
if codeob.co_flags & feature.compiler_flag:
self.flags |= feature.compiler_flag
return codeob
class CommandCompiler:
"""Instances of this class have __call__ methods identical in
signature to compile_command; the difference is that if the
instance compiles program text containing a __future__ statement,
the instance 'remembers' and compiles all subsequent program texts
with the statement in force."""
def __init__(self,):
self.compiler = Compile()
def __call__(self, source, filename="<input>", symbol="single"):
r"""Compile a command and determine whether it is incomplete.
Arguments:
source -- the source string; may contain \n characters
filename -- optional filename from which source was read;
default "<input>"
symbol -- optional grammar start symbol; "single" (default) or
"eval"
Return value / exceptions raised:
- Return a code object if the command is complete and valid
- Return None if the command is incomplete
- Raise SyntaxError, ValueError or OverflowError if the command is a
syntax error (OverflowError and ValueError can be produced by
malformed literals).
"""
return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags)

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,166 @@
"""Conversion functions between RGB and other color systems.
This modules provides two functions for each color system ABC:
rgb_to_abc(r, g, b) --> a, b, c
abc_to_rgb(a, b, c) --> r, g, b
All inputs and outputs are triples of floats in the range [0.0...1.0]
(with the exception of I and Q, which covers a slightly larger range).
Inputs outside the valid range may cause exceptions or invalid outputs.
Supported color systems:
RGB: Red, Green, Blue components
YIQ: Luminance, Chrominance (used by composite video signals)
HLS: Hue, Luminance, Saturation
HSV: Hue, Saturation, Value
"""
# References:
# http://en.wikipedia.org/wiki/YIQ
# http://en.wikipedia.org/wiki/HLS_color_space
# http://en.wikipedia.org/wiki/HSV_color_space
__all__ = ["rgb_to_yiq","yiq_to_rgb","rgb_to_hls","hls_to_rgb",
"rgb_to_hsv","hsv_to_rgb"]
# Some floating-point constants
ONE_THIRD = 1.0/3.0
ONE_SIXTH = 1.0/6.0
TWO_THIRD = 2.0/3.0
# YIQ: used by composite video signals (linear combinations of RGB)
# Y: perceived grey level (0.0 == black, 1.0 == white)
# I, Q: color components
#
# There are a great many versions of the constants used in these formulae.
# The ones in this library uses constants from the FCC version of NTSC.
def rgb_to_yiq(r, g, b):
y = 0.30*r + 0.59*g + 0.11*b
i = 0.74*(r-y) - 0.27*(b-y)
q = 0.48*(r-y) + 0.41*(b-y)
return (y, i, q)
def yiq_to_rgb(y, i, q):
# r = y + (0.27*q + 0.41*i) / (0.74*0.41 + 0.27*0.48)
# b = y + (0.74*q - 0.48*i) / (0.74*0.41 + 0.27*0.48)
# g = y - (0.30*(r-y) + 0.11*(b-y)) / 0.59
r = y + 0.9468822170900693*i + 0.6235565819861433*q
g = y - 0.27478764629897834*i - 0.6356910791873801*q
b = y - 1.1085450346420322*i + 1.7090069284064666*q
if r < 0.0:
r = 0.0
if g < 0.0:
g = 0.0
if b < 0.0:
b = 0.0
if r > 1.0:
r = 1.0
if g > 1.0:
g = 1.0
if b > 1.0:
b = 1.0
return (r, g, b)
# HLS: Hue, Luminance, Saturation
# H: position in the spectrum
# L: color lightness
# S: color saturation
def rgb_to_hls(r, g, b):
maxc = max(r, g, b)
minc = min(r, g, b)
sumc = (maxc+minc)
rangec = (maxc-minc)
l = sumc/2.0
if minc == maxc:
return 0.0, l, 0.0
if l <= 0.5:
s = rangec / sumc
else:
s = rangec / (2.0-maxc-minc) # Not always 2.0-sumc: gh-106498.
rc = (maxc-r) / rangec
gc = (maxc-g) / rangec
bc = (maxc-b) / rangec
if r == maxc:
h = bc-gc
elif g == maxc:
h = 2.0+rc-bc
else:
h = 4.0+gc-rc
h = (h/6.0) % 1.0
return h, l, s
def hls_to_rgb(h, l, s):
if s == 0.0:
return l, l, l
if l <= 0.5:
m2 = l * (1.0+s)
else:
m2 = l+s-(l*s)
m1 = 2.0*l - m2
return (_v(m1, m2, h+ONE_THIRD), _v(m1, m2, h), _v(m1, m2, h-ONE_THIRD))
def _v(m1, m2, hue):
hue = hue % 1.0
if hue < ONE_SIXTH:
return m1 + (m2-m1)*hue*6.0
if hue < 0.5:
return m2
if hue < TWO_THIRD:
return m1 + (m2-m1)*(TWO_THIRD-hue)*6.0
return m1
# HSV: Hue, Saturation, Value
# H: position in the spectrum
# S: color saturation ("purity")
# V: color brightness
def rgb_to_hsv(r, g, b):
maxc = max(r, g, b)
minc = min(r, g, b)
rangec = (maxc-minc)
v = maxc
if minc == maxc:
return 0.0, 0.0, v
s = rangec / maxc
rc = (maxc-r) / rangec
gc = (maxc-g) / rangec
bc = (maxc-b) / rangec
if r == maxc:
h = bc-gc
elif g == maxc:
h = 2.0+rc-bc
else:
h = 4.0+gc-rc
h = (h/6.0) % 1.0
return h, s, v
def hsv_to_rgb(h, s, v):
if s == 0.0:
return v, v, v
i = int(h*6.0) # XXX assume int() truncates!
f = (h*6.0) - i
p = v*(1.0 - s)
q = v*(1.0 - s*f)
t = v*(1.0 - s*(1.0-f))
i = i%6
if i == 0:
return v, t, p
if i == 1:
return q, v, p
if i == 2:
return p, v, t
if i == 3:
return p, q, v
if i == 4:
return t, p, v
if i == 5:
return v, p, q
# Cannot get here

View file

@ -0,0 +1,814 @@
"""Utilities for with-statement contexts. See PEP 343."""
import abc
import os
import sys
import _collections_abc
from collections import deque
from functools import wraps
from types import MethodType, GenericAlias
__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext",
"AbstractContextManager", "AbstractAsyncContextManager",
"AsyncExitStack", "ContextDecorator", "ExitStack",
"redirect_stdout", "redirect_stderr", "suppress", "aclosing",
"chdir"]
class AbstractContextManager(abc.ABC):
"""An abstract base class for context managers."""
__class_getitem__ = classmethod(GenericAlias)
__slots__ = ()
def __enter__(self):
"""Return `self` upon entering the runtime context."""
return self
@abc.abstractmethod
def __exit__(self, exc_type, exc_value, traceback):
"""Raise any exception triggered within the runtime context."""
return None
@classmethod
def __subclasshook__(cls, C):
if cls is AbstractContextManager:
return _collections_abc._check_methods(C, "__enter__", "__exit__")
return NotImplemented
class AbstractAsyncContextManager(abc.ABC):
"""An abstract base class for asynchronous context managers."""
__class_getitem__ = classmethod(GenericAlias)
__slots__ = ()
async def __aenter__(self):
"""Return `self` upon entering the runtime context."""
return self
@abc.abstractmethod
async def __aexit__(self, exc_type, exc_value, traceback):
"""Raise any exception triggered within the runtime context."""
return None
@classmethod
def __subclasshook__(cls, C):
if cls is AbstractAsyncContextManager:
return _collections_abc._check_methods(C, "__aenter__",
"__aexit__")
return NotImplemented
class ContextDecorator(object):
"A base class or mixin that enables context managers to work as decorators."
def _recreate_cm(self):
"""Return a recreated instance of self.
Allows an otherwise one-shot context manager like
_GeneratorContextManager to support use as
a decorator via implicit recreation.
This is a private interface just for _GeneratorContextManager.
See issue #11647 for details.
"""
return self
def __call__(self, func):
@wraps(func)
def inner(*args, **kwds):
with self._recreate_cm():
return func(*args, **kwds)
return inner
class AsyncContextDecorator(object):
"A base class or mixin that enables async context managers to work as decorators."
def _recreate_cm(self):
"""Return a recreated instance of self.
"""
return self
def __call__(self, func):
@wraps(func)
async def inner(*args, **kwds):
async with self._recreate_cm():
return await func(*args, **kwds)
return inner
class _GeneratorContextManagerBase:
"""Shared functionality for @contextmanager and @asynccontextmanager."""
def __init__(self, func, args, kwds):
self.gen = func(*args, **kwds)
self.func, self.args, self.kwds = func, args, kwds
# Issue 19330: ensure context manager instances have good docstrings
doc = getattr(func, "__doc__", None)
if doc is None:
doc = type(self).__doc__
self.__doc__ = doc
# Unfortunately, this still doesn't provide good help output when
# inspecting the created context manager instances, since pydoc
# currently bypasses the instance docstring and shows the docstring
# for the class instead.
# See http://bugs.python.org/issue19404 for more details.
def _recreate_cm(self):
# _GCMB instances are one-shot context managers, so the
# CM must be recreated each time a decorated function is
# called
return self.__class__(self.func, self.args, self.kwds)
class _GeneratorContextManager(
_GeneratorContextManagerBase,
AbstractContextManager,
ContextDecorator,
):
"""Helper for @contextmanager decorator."""
def __enter__(self):
# do not keep args and kwds alive unnecessarily
# they are only needed for recreation, which is not possible anymore
del self.args, self.kwds, self.func
try:
return next(self.gen)
except StopIteration:
raise RuntimeError("generator didn't yield") from None
def __exit__(self, typ, value, traceback):
if typ is None:
try:
next(self.gen)
except StopIteration:
return False
else:
try:
raise RuntimeError("generator didn't stop")
finally:
self.gen.close()
else:
if value is None:
# Need to force instantiation so we can reliably
# tell if we get the same exception back
value = typ()
try:
self.gen.throw(value)
except StopIteration as exc:
# Suppress StopIteration *unless* it's the same exception that
# was passed to throw(). This prevents a StopIteration
# raised inside the "with" statement from being suppressed.
return exc is not value
except RuntimeError as exc:
# Don't re-raise the passed in exception. (issue27122)
if exc is value:
exc.__traceback__ = traceback
return False
# Avoid suppressing if a StopIteration exception
# was passed to throw() and later wrapped into a RuntimeError
# (see PEP 479 for sync generators; async generators also
# have this behavior). But do this only if the exception wrapped
# by the RuntimeError is actually Stop(Async)Iteration (see
# issue29692).
if (
isinstance(value, StopIteration)
and exc.__cause__ is value
):
value.__traceback__ = traceback
return False
raise
except BaseException as exc:
# only re-raise if it's *not* the exception that was
# passed to throw(), because __exit__() must not raise
# an exception unless __exit__() itself failed. But throw()
# has to raise the exception to signal propagation, so this
# fixes the impedance mismatch between the throw() protocol
# and the __exit__() protocol.
if exc is not value:
raise
exc.__traceback__ = traceback
return False
try:
raise RuntimeError("generator didn't stop after throw()")
finally:
self.gen.close()
class _AsyncGeneratorContextManager(
_GeneratorContextManagerBase,
AbstractAsyncContextManager,
AsyncContextDecorator,
):
"""Helper for @asynccontextmanager decorator."""
async def __aenter__(self):
# do not keep args and kwds alive unnecessarily
# they are only needed for recreation, which is not possible anymore
del self.args, self.kwds, self.func
try:
return await anext(self.gen)
except StopAsyncIteration:
raise RuntimeError("generator didn't yield") from None
async def __aexit__(self, typ, value, traceback):
if typ is None:
try:
await anext(self.gen)
except StopAsyncIteration:
return False
else:
try:
raise RuntimeError("generator didn't stop")
finally:
await self.gen.aclose()
else:
if value is None:
# Need to force instantiation so we can reliably
# tell if we get the same exception back
value = typ()
try:
await self.gen.athrow(value)
except StopAsyncIteration as exc:
# Suppress StopIteration *unless* it's the same exception that
# was passed to throw(). This prevents a StopIteration
# raised inside the "with" statement from being suppressed.
return exc is not value
except RuntimeError as exc:
# Don't re-raise the passed in exception. (issue27122)
if exc is value:
exc.__traceback__ = traceback
return False
# Avoid suppressing if a Stop(Async)Iteration exception
# was passed to athrow() and later wrapped into a RuntimeError
# (see PEP 479 for sync generators; async generators also
# have this behavior). But do this only if the exception wrapped
# by the RuntimeError is actually Stop(Async)Iteration (see
# issue29692).
if (
isinstance(value, (StopIteration, StopAsyncIteration))
and exc.__cause__ is value
):
value.__traceback__ = traceback
return False
raise
except BaseException as exc:
# only re-raise if it's *not* the exception that was
# passed to throw(), because __exit__() must not raise
# an exception unless __exit__() itself failed. But throw()
# has to raise the exception to signal propagation, so this
# fixes the impedance mismatch between the throw() protocol
# and the __exit__() protocol.
if exc is not value:
raise
exc.__traceback__ = traceback
return False
try:
raise RuntimeError("generator didn't stop after athrow()")
finally:
await self.gen.aclose()
def contextmanager(func):
"""@contextmanager decorator.
Typical usage:
@contextmanager
def some_generator(<arguments>):
<setup>
try:
yield <value>
finally:
<cleanup>
This makes this:
with some_generator(<arguments>) as <variable>:
<body>
equivalent to this:
<setup>
try:
<variable> = <value>
<body>
finally:
<cleanup>
"""
@wraps(func)
def helper(*args, **kwds):
return _GeneratorContextManager(func, args, kwds)
return helper
def asynccontextmanager(func):
"""@asynccontextmanager decorator.
Typical usage:
@asynccontextmanager
async def some_async_generator(<arguments>):
<setup>
try:
yield <value>
finally:
<cleanup>
This makes this:
async with some_async_generator(<arguments>) as <variable>:
<body>
equivalent to this:
<setup>
try:
<variable> = <value>
<body>
finally:
<cleanup>
"""
@wraps(func)
def helper(*args, **kwds):
return _AsyncGeneratorContextManager(func, args, kwds)
return helper
class closing(AbstractContextManager):
"""Context to automatically close something at the end of a block.
Code like this:
with closing(<module>.open(<arguments>)) as f:
<block>
is equivalent to this:
f = <module>.open(<arguments>)
try:
<block>
finally:
f.close()
"""
def __init__(self, thing):
self.thing = thing
def __enter__(self):
return self.thing
def __exit__(self, *exc_info):
self.thing.close()
class aclosing(AbstractAsyncContextManager):
"""Async context manager for safely finalizing an asynchronously cleaned-up
resource such as an async generator, calling its ``aclose()`` method.
Code like this:
async with aclosing(<module>.fetch(<arguments>)) as agen:
<block>
is equivalent to this:
agen = <module>.fetch(<arguments>)
try:
<block>
finally:
await agen.aclose()
"""
def __init__(self, thing):
self.thing = thing
async def __aenter__(self):
return self.thing
async def __aexit__(self, *exc_info):
await self.thing.aclose()
class _RedirectStream(AbstractContextManager):
_stream = None
def __init__(self, new_target):
self._new_target = new_target
# We use a list of old targets to make this CM re-entrant
self._old_targets = []
def __enter__(self):
self._old_targets.append(getattr(sys, self._stream))
setattr(sys, self._stream, self._new_target)
return self._new_target
def __exit__(self, exctype, excinst, exctb):
setattr(sys, self._stream, self._old_targets.pop())
class redirect_stdout(_RedirectStream):
"""Context manager for temporarily redirecting stdout to another file.
# How to send help() to stderr
with redirect_stdout(sys.stderr):
help(dir)
# How to write help() to a file
with open('help.txt', 'w') as f:
with redirect_stdout(f):
help(pow)
"""
_stream = "stdout"
class redirect_stderr(_RedirectStream):
"""Context manager for temporarily redirecting stderr to another file."""
_stream = "stderr"
class suppress(AbstractContextManager):
"""Context manager to suppress specified exceptions
After the exception is suppressed, execution proceeds with the next
statement following the with statement.
with suppress(FileNotFoundError):
os.remove(somefile)
# Execution still resumes here if the file was already removed
"""
def __init__(self, *exceptions):
self._exceptions = exceptions
def __enter__(self):
pass
def __exit__(self, exctype, excinst, exctb):
# Unlike isinstance and issubclass, CPython exception handling
# currently only looks at the concrete type hierarchy (ignoring
# the instance and subclass checking hooks). While Guido considers
# that a bug rather than a feature, it's a fairly hard one to fix
# due to various internal implementation details. suppress provides
# the simpler issubclass based semantics, rather than trying to
# exactly reproduce the limitations of the CPython interpreter.
#
# See http://bugs.python.org/issue12029 for more details
if exctype is None:
return
if issubclass(exctype, self._exceptions):
return True
if issubclass(exctype, BaseExceptionGroup):
match, rest = excinst.split(self._exceptions)
if rest is None:
return True
raise rest
return False
class _BaseExitStack:
"""A base class for ExitStack and AsyncExitStack."""
@staticmethod
def _create_exit_wrapper(cm, cm_exit):
return MethodType(cm_exit, cm)
@staticmethod
def _create_cb_wrapper(callback, /, *args, **kwds):
def _exit_wrapper(exc_type, exc, tb):
callback(*args, **kwds)
return _exit_wrapper
def __init__(self):
self._exit_callbacks = deque()
def pop_all(self):
"""Preserve the context stack by transferring it to a new instance."""
new_stack = type(self)()
new_stack._exit_callbacks = self._exit_callbacks
self._exit_callbacks = deque()
return new_stack
def push(self, exit):
"""Registers a callback with the standard __exit__ method signature.
Can suppress exceptions the same way __exit__ method can.
Also accepts any object with an __exit__ method (registering a call
to the method instead of the object itself).
"""
# We use an unbound method rather than a bound method to follow
# the standard lookup behaviour for special methods.
_cb_type = type(exit)
try:
exit_method = _cb_type.__exit__
except AttributeError:
# Not a context manager, so assume it's a callable.
self._push_exit_callback(exit)
else:
self._push_cm_exit(exit, exit_method)
return exit # Allow use as a decorator.
def enter_context(self, cm):
"""Enters the supplied context manager.
If successful, also pushes its __exit__ method as a callback and
returns the result of the __enter__ method.
"""
# We look up the special methods on the type to match the with
# statement.
cls = type(cm)
try:
_enter = cls.__enter__
_exit = cls.__exit__
except AttributeError:
raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
f"not support the context manager protocol") from None
result = _enter(cm)
self._push_cm_exit(cm, _exit)
return result
def callback(self, callback, /, *args, **kwds):
"""Registers an arbitrary callback and arguments.
Cannot suppress exceptions.
"""
_exit_wrapper = self._create_cb_wrapper(callback, *args, **kwds)
# We changed the signature, so using @wraps is not appropriate, but
# setting __wrapped__ may still help with introspection.
_exit_wrapper.__wrapped__ = callback
self._push_exit_callback(_exit_wrapper)
return callback # Allow use as a decorator
def _push_cm_exit(self, cm, cm_exit):
"""Helper to correctly register callbacks to __exit__ methods."""
_exit_wrapper = self._create_exit_wrapper(cm, cm_exit)
self._push_exit_callback(_exit_wrapper, True)
def _push_exit_callback(self, callback, is_sync=True):
self._exit_callbacks.append((is_sync, callback))
# Inspired by discussions on http://bugs.python.org/issue13585
class ExitStack(_BaseExitStack, AbstractContextManager):
"""Context manager for dynamic management of a stack of exit callbacks.
For example:
with ExitStack() as stack:
files = [stack.enter_context(open(fname)) for fname in filenames]
# All opened files will automatically be closed at the end of
# the with statement, even if attempts to open files later
# in the list raise an exception.
"""
def __enter__(self):
return self
def __exit__(self, *exc_details):
exc = exc_details[1]
received_exc = exc is not None
# We manipulate the exception state so it behaves as though
# we were actually nesting multiple with statements
frame_exc = sys.exception()
def _fix_exception_context(new_exc, old_exc):
# Context may not be correct, so find the end of the chain
while 1:
exc_context = new_exc.__context__
if exc_context is None or exc_context is old_exc:
# Context is already set correctly (see issue 20317)
return
if exc_context is frame_exc:
break
new_exc = exc_context
# Change the end of the chain to point to the exception
# we expect it to reference
new_exc.__context__ = old_exc
# Callbacks are invoked in LIFO order to match the behaviour of
# nested context managers
suppressed_exc = False
pending_raise = False
while self._exit_callbacks:
is_sync, cb = self._exit_callbacks.pop()
assert is_sync
try:
if exc is None:
exc_details = None, None, None
else:
exc_details = type(exc), exc, exc.__traceback__
if cb(*exc_details):
suppressed_exc = True
pending_raise = False
exc = None
except BaseException as new_exc:
# simulate the stack of exceptions by setting the context
_fix_exception_context(new_exc, exc)
pending_raise = True
exc = new_exc
if pending_raise:
try:
# bare "raise exc" replaces our carefully
# set-up context
fixed_ctx = exc.__context__
raise exc
except BaseException:
exc.__context__ = fixed_ctx
raise
return received_exc and suppressed_exc
def close(self):
"""Immediately unwind the context stack."""
self.__exit__(None, None, None)
# Inspired by discussions on https://bugs.python.org/issue29302
class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
"""Async context manager for dynamic management of a stack of exit
callbacks.
For example:
async with AsyncExitStack() as stack:
connections = [await stack.enter_async_context(get_connection())
for i in range(5)]
# All opened connections will automatically be released at the
# end of the async with statement, even if attempts to open a
# connection later in the list raise an exception.
"""
@staticmethod
def _create_async_exit_wrapper(cm, cm_exit):
return MethodType(cm_exit, cm)
@staticmethod
def _create_async_cb_wrapper(callback, /, *args, **kwds):
async def _exit_wrapper(exc_type, exc, tb):
await callback(*args, **kwds)
return _exit_wrapper
async def enter_async_context(self, cm):
"""Enters the supplied async context manager.
If successful, also pushes its __aexit__ method as a callback and
returns the result of the __aenter__ method.
"""
cls = type(cm)
try:
_enter = cls.__aenter__
_exit = cls.__aexit__
except AttributeError:
raise TypeError(f"'{cls.__module__}.{cls.__qualname__}' object does "
f"not support the asynchronous context manager protocol"
) from None
result = await _enter(cm)
self._push_async_cm_exit(cm, _exit)
return result
def push_async_exit(self, exit):
"""Registers a coroutine function with the standard __aexit__ method
signature.
Can suppress exceptions the same way __aexit__ method can.
Also accepts any object with an __aexit__ method (registering a call
to the method instead of the object itself).
"""
_cb_type = type(exit)
try:
exit_method = _cb_type.__aexit__
except AttributeError:
# Not an async context manager, so assume it's a coroutine function
self._push_exit_callback(exit, False)
else:
self._push_async_cm_exit(exit, exit_method)
return exit # Allow use as a decorator
def push_async_callback(self, callback, /, *args, **kwds):
"""Registers an arbitrary coroutine function and arguments.
Cannot suppress exceptions.
"""
_exit_wrapper = self._create_async_cb_wrapper(callback, *args, **kwds)
# We changed the signature, so using @wraps is not appropriate, but
# setting __wrapped__ may still help with introspection.
_exit_wrapper.__wrapped__ = callback
self._push_exit_callback(_exit_wrapper, False)
return callback # Allow use as a decorator
async def aclose(self):
"""Immediately unwind the context stack."""
await self.__aexit__(None, None, None)
def _push_async_cm_exit(self, cm, cm_exit):
"""Helper to correctly register coroutine function to __aexit__
method."""
_exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit)
self._push_exit_callback(_exit_wrapper, False)
async def __aenter__(self):
return self
async def __aexit__(self, *exc_details):
exc = exc_details[1]
received_exc = exc is not None
# We manipulate the exception state so it behaves as though
# we were actually nesting multiple with statements
frame_exc = sys.exception()
def _fix_exception_context(new_exc, old_exc):
# Context may not be correct, so find the end of the chain
while 1:
exc_context = new_exc.__context__
if exc_context is None or exc_context is old_exc:
# Context is already set correctly (see issue 20317)
return
if exc_context is frame_exc:
break
new_exc = exc_context
# Change the end of the chain to point to the exception
# we expect it to reference
new_exc.__context__ = old_exc
# Callbacks are invoked in LIFO order to match the behaviour of
# nested context managers
suppressed_exc = False
pending_raise = False
while self._exit_callbacks:
is_sync, cb = self._exit_callbacks.pop()
try:
if exc is None:
exc_details = None, None, None
else:
exc_details = type(exc), exc, exc.__traceback__
if is_sync:
cb_suppress = cb(*exc_details)
else:
cb_suppress = await cb(*exc_details)
if cb_suppress:
suppressed_exc = True
pending_raise = False
exc = None
except BaseException as new_exc:
# simulate the stack of exceptions by setting the context
_fix_exception_context(new_exc, exc)
pending_raise = True
exc = new_exc
if pending_raise:
try:
# bare "raise exc" replaces our carefully
# set-up context
fixed_ctx = exc.__context__
raise exc
except BaseException:
exc.__context__ = fixed_ctx
raise
return received_exc and suppressed_exc
class nullcontext(AbstractContextManager, AbstractAsyncContextManager):
"""Context manager that does no additional processing.
Used as a stand-in for a normal context manager, when a particular
block of code is only sometimes used with a normal context manager:
cm = optional_cm if condition else nullcontext()
with cm:
# Perform operation, using optional_cm if condition is True
"""
def __init__(self, enter_result=None):
self.enter_result = enter_result
def __enter__(self):
return self.enter_result
def __exit__(self, *excinfo):
pass
async def __aenter__(self):
return self.enter_result
async def __aexit__(self, *excinfo):
pass
class chdir(AbstractContextManager):
"""Non thread-safe context manager to change the current working directory."""
def __init__(self, path):
self.path = path
self._old_cwd = []
def __enter__(self):
self._old_cwd.append(os.getcwd())
os.chdir(self.path)
def __exit__(self, *excinfo):
os.chdir(self._old_cwd.pop())

View file

@ -0,0 +1,286 @@
"""Generic (shallow and deep) copying operations.
Interface summary:
import copy
x = copy.copy(y) # make a shallow copy of y
x = copy.deepcopy(y) # make a deep copy of y
x = copy.replace(y, a=1, b=2) # new object with fields replaced, as defined by `__replace__`
For module specific errors, copy.Error is raised.
The difference between shallow and deep copying is only relevant for
compound objects (objects that contain other objects, like lists or
class instances).
- A shallow copy constructs a new compound object and then (to the
extent possible) inserts *the same objects* into it that the
original contains.
- A deep copy constructs a new compound object and then, recursively,
inserts *copies* into it of the objects found in the original.
Two problems often exist with deep copy operations that don't exist
with shallow copy operations:
a) recursive objects (compound objects that, directly or indirectly,
contain a reference to themselves) may cause a recursive loop
b) because deep copy copies *everything* it may copy too much, e.g.
administrative data structures that should be shared even between
copies
Python's deep copy operation avoids these problems by:
a) keeping a table of objects already copied during the current
copying pass
b) letting user-defined classes override the copying operation or the
set of components copied
This version does not copy types like module, class, function, method,
nor stack trace, stack frame, nor file, socket, window, nor any
similar types.
Classes can use the same interfaces to control copying that they use
to control pickling: they can define methods called __getinitargs__(),
__getstate__() and __setstate__(). See the documentation for module
"pickle" for information on these methods.
"""
import types
import weakref
from copyreg import dispatch_table
class Error(Exception):
pass
error = Error # backward compatibility
__all__ = ["Error", "copy", "deepcopy", "replace"]
def copy(x):
"""Shallow copy operation on arbitrary Python objects.
See the module's __doc__ string for more info.
"""
cls = type(x)
if cls in _copy_atomic_types:
return x
if cls in _copy_builtin_containers:
return cls.copy(x)
if issubclass(cls, type):
# treat it as a regular class:
return x
copier = getattr(cls, "__copy__", None)
if copier is not None:
return copier(x)
reductor = dispatch_table.get(cls)
if reductor is not None:
rv = reductor(x)
else:
reductor = getattr(x, "__reduce_ex__", None)
if reductor is not None:
rv = reductor(4)
else:
reductor = getattr(x, "__reduce__", None)
if reductor:
rv = reductor()
else:
raise Error("un(shallow)copyable object of type %s" % cls)
if isinstance(rv, str):
return x
return _reconstruct(x, None, *rv)
_copy_atomic_types = {types.NoneType, int, float, bool, complex, str, tuple,
bytes, frozenset, type, range, slice, property,
types.BuiltinFunctionType, types.EllipsisType,
types.NotImplementedType, types.FunctionType, types.CodeType,
weakref.ref, super}
_copy_builtin_containers = {list, dict, set, bytearray}
def deepcopy(x, memo=None, _nil=[]):
"""Deep copy operation on arbitrary Python objects.
See the module's __doc__ string for more info.
"""
cls = type(x)
if cls in _atomic_types:
return x
d = id(x)
if memo is None:
memo = {}
else:
y = memo.get(d, _nil)
if y is not _nil:
return y
copier = _deepcopy_dispatch.get(cls)
if copier is not None:
y = copier(x, memo)
else:
if issubclass(cls, type):
y = x # atomic copy
else:
copier = getattr(x, "__deepcopy__", None)
if copier is not None:
y = copier(memo)
else:
reductor = dispatch_table.get(cls)
if reductor:
rv = reductor(x)
else:
reductor = getattr(x, "__reduce_ex__", None)
if reductor is not None:
rv = reductor(4)
else:
reductor = getattr(x, "__reduce__", None)
if reductor:
rv = reductor()
else:
raise Error(
"un(deep)copyable object of type %s" % cls)
if isinstance(rv, str):
y = x
else:
y = _reconstruct(x, memo, *rv)
# If is its own copy, don't memoize.
if y is not x:
memo[d] = y
_keep_alive(x, memo) # Make sure x lives at least as long as d
return y
_atomic_types = {types.NoneType, types.EllipsisType, types.NotImplementedType,
int, float, bool, complex, bytes, str, types.CodeType, type, range,
types.BuiltinFunctionType, types.FunctionType, weakref.ref, property}
_deepcopy_dispatch = d = {}
def _deepcopy_list(x, memo, deepcopy=deepcopy):
y = []
memo[id(x)] = y
append = y.append
for a in x:
append(deepcopy(a, memo))
return y
d[list] = _deepcopy_list
def _deepcopy_tuple(x, memo, deepcopy=deepcopy):
y = [deepcopy(a, memo) for a in x]
# We're not going to put the tuple in the memo, but it's still important we
# check for it, in case the tuple contains recursive mutable structures.
try:
return memo[id(x)]
except KeyError:
pass
for k, j in zip(x, y):
if k is not j:
y = tuple(y)
break
else:
y = x
return y
d[tuple] = _deepcopy_tuple
def _deepcopy_dict(x, memo, deepcopy=deepcopy):
y = {}
memo[id(x)] = y
for key, value in x.items():
y[deepcopy(key, memo)] = deepcopy(value, memo)
return y
d[dict] = _deepcopy_dict
def _deepcopy_method(x, memo): # Copy instance methods
return type(x)(x.__func__, deepcopy(x.__self__, memo))
d[types.MethodType] = _deepcopy_method
del d
def _keep_alive(x, memo):
"""Keeps a reference to the object x in the memo.
Because we remember objects by their id, we have
to assure that possibly temporary objects are kept
alive by referencing them.
We store a reference at the id of the memo, which should
normally not be used unless someone tries to deepcopy
the memo itself...
"""
try:
memo[id(memo)].append(x)
except KeyError:
# aha, this is the first one :-)
memo[id(memo)]=[x]
def _reconstruct(x, memo, func, args,
state=None, listiter=None, dictiter=None,
*, deepcopy=deepcopy):
deep = memo is not None
if deep and args:
args = (deepcopy(arg, memo) for arg in args)
y = func(*args)
if deep:
memo[id(x)] = y
if state is not None:
if deep:
state = deepcopy(state, memo)
if hasattr(y, '__setstate__'):
y.__setstate__(state)
else:
if isinstance(state, tuple) and len(state) == 2:
state, slotstate = state
else:
slotstate = None
if state is not None:
y.__dict__.update(state)
if slotstate is not None:
for key, value in slotstate.items():
setattr(y, key, value)
if listiter is not None:
if deep:
for item in listiter:
item = deepcopy(item, memo)
y.append(item)
else:
for item in listiter:
y.append(item)
if dictiter is not None:
if deep:
for key, value in dictiter:
key = deepcopy(key, memo)
value = deepcopy(value, memo)
y[key] = value
else:
for key, value in dictiter:
y[key] = value
return y
del types, weakref
def replace(obj, /, **changes):
"""Return a new object replacing specified fields with new values.
This is especially useful for immutable objects, like named tuples or
frozen dataclasses.
"""
cls = obj.__class__
func = getattr(cls, '__replace__', None)
if func is None:
raise TypeError(f"replace() does not support {cls.__name__} objects")
return func(obj, **changes)

View file

@ -0,0 +1,222 @@
"""Helper to provide extensibility for pickle.
This is only useful to add pickle support for extension types defined in
C, not for instances of user-defined classes.
"""
__all__ = ["pickle", "constructor",
"add_extension", "remove_extension", "clear_extension_cache"]
dispatch_table = {}
def pickle(ob_type, pickle_function, constructor_ob=None):
if not callable(pickle_function):
raise TypeError("reduction functions must be callable")
dispatch_table[ob_type] = pickle_function
# The constructor_ob function is a vestige of safe for unpickling.
# There is no reason for the caller to pass it anymore.
if constructor_ob is not None:
constructor(constructor_ob)
def constructor(object):
if not callable(object):
raise TypeError("constructors must be callable")
# Example: provide pickling support for complex numbers.
def pickle_complex(c):
return complex, (c.real, c.imag)
pickle(complex, pickle_complex, complex)
def pickle_union(obj):
import typing, operator
return operator.getitem, (typing.Union, obj.__args__)
pickle(type(int | str), pickle_union)
def pickle_super(obj):
return super, (obj.__thisclass__, obj.__self__)
pickle(super, pickle_super)
# Support for pickling new-style objects
def _reconstructor(cls, base, state):
if base is object:
obj = object.__new__(cls)
else:
obj = base.__new__(cls, state)
if base.__init__ != object.__init__:
base.__init__(obj, state)
return obj
_HEAPTYPE = 1<<9
_new_type = type(int.__new__)
# Python code for object.__reduce_ex__ for protocols 0 and 1
def _reduce_ex(self, proto):
assert proto < 2
cls = self.__class__
for base in cls.__mro__:
if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE:
break
new = base.__new__
if isinstance(new, _new_type) and new.__self__ is base:
break
else:
base = object # not really reachable
if base is object:
state = None
else:
if base is cls:
raise TypeError(f"cannot pickle {cls.__name__!r} object")
state = base(self)
args = (cls, base, state)
try:
getstate = self.__getstate__
except AttributeError:
if getattr(self, "__slots__", None):
raise TypeError(f"cannot pickle {cls.__name__!r} object: "
f"a class that defines __slots__ without "
f"defining __getstate__ cannot be pickled "
f"with protocol {proto}") from None
try:
dict = self.__dict__
except AttributeError:
dict = None
else:
if (type(self).__getstate__ is object.__getstate__ and
getattr(self, "__slots__", None)):
raise TypeError("a class that defines __slots__ without "
"defining __getstate__ cannot be pickled")
dict = getstate()
if dict:
return _reconstructor, args, dict
else:
return _reconstructor, args
# Helper for __reduce_ex__ protocol 2
def __newobj__(cls, *args):
return cls.__new__(cls, *args)
def __newobj_ex__(cls, args, kwargs):
"""Used by pickle protocol 4, instead of __newobj__ to allow classes with
keyword-only arguments to be pickled correctly.
"""
return cls.__new__(cls, *args, **kwargs)
def _slotnames(cls):
"""Return a list of slot names for a given class.
This needs to find slots defined by the class and its bases, so we
can't simply return the __slots__ attribute. We must walk down
the Method Resolution Order and concatenate the __slots__ of each
class found there. (This assumes classes don't modify their
__slots__ attribute to misrepresent their slots after the class is
defined.)
"""
# Get the value from a cache in the class if possible
names = cls.__dict__.get("__slotnames__")
if names is not None:
return names
# Not cached -- calculate the value
names = []
if not hasattr(cls, "__slots__"):
# This class has no slots
pass
else:
# Slots found -- gather slot names from all base classes
for c in cls.__mro__:
if "__slots__" in c.__dict__:
slots = c.__dict__['__slots__']
# if class has a single slot, it can be given as a string
if isinstance(slots, str):
slots = (slots,)
for name in slots:
# special descriptors
if name in ("__dict__", "__weakref__"):
continue
# mangled names
elif name.startswith('__') and not name.endswith('__'):
stripped = c.__name__.lstrip('_')
if stripped:
names.append('_%s%s' % (stripped, name))
else:
names.append(name)
else:
names.append(name)
# Cache the outcome in the class if at all possible
try:
cls.__slotnames__ = names
except:
pass # But don't die if we can't
return names
# A registry of extension codes. This is an ad-hoc compression
# mechanism. Whenever a global reference to <module>, <name> is about
# to be pickled, the (<module>, <name>) tuple is looked up here to see
# if it is a registered extension code for it. Extension codes are
# universal, so that the meaning of a pickle does not depend on
# context. (There are also some codes reserved for local use that
# don't have this restriction.) Codes are positive ints; 0 is
# reserved.
_extension_registry = {} # key -> code
_inverted_registry = {} # code -> key
_extension_cache = {} # code -> object
# Don't ever rebind those names: pickling grabs a reference to them when
# it's initialized, and won't see a rebinding.
def add_extension(module, name, code):
"""Register an extension code."""
code = int(code)
if not 1 <= code <= 0x7fffffff:
raise ValueError("code out of range")
key = (module, name)
if (_extension_registry.get(key) == code and
_inverted_registry.get(code) == key):
return # Redundant registrations are benign
if key in _extension_registry:
raise ValueError("key %s is already registered with code %s" %
(key, _extension_registry[key]))
if code in _inverted_registry:
raise ValueError("code %s is already in use for key %s" %
(code, _inverted_registry[code]))
_extension_registry[key] = code
_inverted_registry[code] = key
def remove_extension(module, name, code):
"""Unregister an extension code. For testing only."""
key = (module, name)
if (_extension_registry.get(key) != code or
_inverted_registry.get(code) != key):
raise ValueError("key %s is not registered with code %s" %
(key, code))
del _extension_registry[key]
del _inverted_registry[code]
if code in _extension_cache:
del _extension_cache[code]
def clear_extension_cache():
_extension_cache.clear()
# Standard extension code assignments
# Reserved ranges
# First Last Count Purpose
# 1 127 127 Reserved for Python standard library
# 128 191 64 Reserved for Zope
# 192 239 48 Reserved for 3rd parties
# 240 255 16 Reserved for private use (will never be assigned)
# 256 Inf Inf Reserved for future assignment
# Extension codes are assigned by the Python Software Foundation.

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,13 @@
"""Specific date/time and related types.
See https://data.iana.org/time-zones/tz-link.html for
time zone and DST data sources.
"""
try:
from _datetime import *
except ImportError:
from _pydatetime import *
__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo",
"MINYEAR", "MAXYEAR", "UTC")

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,177 @@
""" Standard "encodings" Package
Standard Python encoding modules are stored in this package
directory.
Codec modules must have names corresponding to normalized encoding
names as defined in the normalize_encoding() function below, e.g.
'utf-8' must be implemented by the module 'utf_8.py'.
Each codec module must export the following interface:
* getregentry() -> codecs.CodecInfo object
The getregentry() API must return a CodecInfo object with encoder, decoder,
incrementalencoder, incrementaldecoder, streamwriter and streamreader
attributes which adhere to the Python Codec Interface Standard.
In addition, a module may optionally also define the following
APIs which are then used by the package's codec search function:
* getaliases() -> sequence of encoding name strings to use as aliases
Alias names returned by getaliases() must be normalized encoding
names as defined by normalize_encoding().
Written by Marc-Andre Lemburg (mal@lemburg.com).
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
"""#"
import codecs
import sys
from . import aliases
_cache = {}
_unknown = '--unknown--'
_import_tail = ['*']
_aliases = aliases.aliases
class CodecRegistryError(LookupError, SystemError):
pass
def normalize_encoding(encoding):
""" Normalize an encoding name.
Normalization works as follows: all non-alphanumeric
characters except the dot used for Python package names are
collapsed and replaced with a single underscore, e.g. ' -;#'
becomes '_'. Leading and trailing underscores are removed.
Note that encoding names should be ASCII only.
"""
if isinstance(encoding, bytes):
encoding = str(encoding, "ascii")
chars = []
punct = False
for c in encoding:
if c.isalnum() or c == '.':
if punct and chars:
chars.append('_')
if c.isascii():
chars.append(c)
punct = False
else:
punct = True
return ''.join(chars)
def search_function(encoding):
# Cache lookup
entry = _cache.get(encoding, _unknown)
if entry is not _unknown:
return entry
# Import the module:
#
# First try to find an alias for the normalized encoding
# name and lookup the module using the aliased name, then try to
# lookup the module using the standard import scheme, i.e. first
# try in the encodings package, then at top-level.
#
norm_encoding = normalize_encoding(encoding)
aliased_encoding = _aliases.get(norm_encoding) or \
_aliases.get(norm_encoding.replace('.', '_'))
if aliased_encoding is not None:
modnames = [aliased_encoding,
norm_encoding]
else:
modnames = [norm_encoding]
for modname in modnames:
if not modname or '.' in modname:
continue
try:
# Import is absolute to prevent the possibly malicious import of a
# module with side-effects that is not in the 'encodings' package.
mod = __import__('encodings.' + modname, fromlist=_import_tail,
level=0)
except ImportError:
# ImportError may occur because 'encodings.(modname)' does not exist,
# or because it imports a name that does not exist (see mbcs and oem)
pass
else:
break
else:
mod = None
try:
getregentry = mod.getregentry
except AttributeError:
# Not a codec module
mod = None
if mod is None:
# Cache misses
_cache[encoding] = None
return None
# Now ask the module for the registry entry
entry = getregentry()
if not isinstance(entry, codecs.CodecInfo):
if not 4 <= len(entry) <= 7:
raise CodecRegistryError('module "%s" (%s) failed to register'
% (mod.__name__, mod.__file__))
if not callable(entry[0]) or not callable(entry[1]) or \
(entry[2] is not None and not callable(entry[2])) or \
(entry[3] is not None and not callable(entry[3])) or \
(len(entry) > 4 and entry[4] is not None and not callable(entry[4])) or \
(len(entry) > 5 and entry[5] is not None and not callable(entry[5])):
raise CodecRegistryError('incompatible codecs in module "%s" (%s)'
% (mod.__name__, mod.__file__))
if len(entry)<7 or entry[6] is None:
entry += (None,)*(6-len(entry)) + (mod.__name__.split(".", 1)[1],)
entry = codecs.CodecInfo(*entry)
# Cache the codec registry entry
_cache[encoding] = entry
# Register its aliases (without overwriting previously registered
# aliases)
try:
codecaliases = mod.getaliases()
except AttributeError:
pass
else:
for alias in codecaliases:
if alias not in _aliases:
_aliases[alias] = modname
# Return the registry entry
return entry
# Register the search_function in the Python codec registry
codecs.register(search_function)
if sys.platform == 'win32':
from ._win_cp_codecs import create_win32_code_page_codec
def win32_code_page_search_function(encoding):
encoding = encoding.lower()
if not encoding.startswith('cp'):
return None
try:
cp = int(encoding[2:])
except ValueError:
return None
# Test if the code page is supported
try:
codecs.code_page_encode(cp, 'x')
except (OverflowError, OSError):
return None
return create_win32_code_page_codec(cp)
codecs.register(win32_code_page_search_function)

View file

@ -0,0 +1,36 @@
import codecs
def create_win32_code_page_codec(cp):
from codecs import code_page_encode, code_page_decode
def encode(input, errors='strict'):
return code_page_encode(cp, input, errors)
def decode(input, errors='strict'):
return code_page_decode(cp, input, errors, True)
class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
return code_page_encode(cp, input, self.errors)[0]
class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
def _buffer_decode(self, input, errors, final):
return code_page_decode(cp, input, errors, final)
class StreamWriter(codecs.StreamWriter):
def encode(self, input, errors='strict'):
return code_page_encode(cp, input, errors)
class StreamReader(codecs.StreamReader):
def decode(self, input, errors, final):
return code_page_decode(cp, input, errors, final)
return codecs.CodecInfo(
name=f'cp{cp}',
encode=encode,
decode=decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamreader=StreamReader,
streamwriter=StreamWriter,
)

View file

@ -0,0 +1,560 @@
""" Encoding Aliases Support
This module is used by the encodings package search function to
map encodings names to module names.
Note that the search function normalizes the encoding names before
doing the lookup, so the mapping will have to map normalized
encoding names to module names.
Contents:
The following aliases dictionary contains mappings of all IANA
character set names for which the Python core library provides
codecs. In addition to these, a few Python specific codec
aliases have also been added.
"""
aliases = {
# Please keep this list sorted alphabetically by value !
# ascii codec
'646' : 'ascii',
'ansi_x3.4_1968' : 'ascii',
'ansi_x3_4_1968' : 'ascii', # some email headers use this non-standard name
'ansi_x3.4_1986' : 'ascii',
'cp367' : 'ascii',
'csascii' : 'ascii',
'ibm367' : 'ascii',
'iso646_us' : 'ascii',
'iso_646.irv_1991' : 'ascii',
'iso_ir_6' : 'ascii',
'us' : 'ascii',
'us_ascii' : 'ascii',
# base64_codec codec
'base64' : 'base64_codec',
'base_64' : 'base64_codec',
# big5 codec
'big5_tw' : 'big5',
'csbig5' : 'big5',
# big5hkscs codec
'big5_hkscs' : 'big5hkscs',
'hkscs' : 'big5hkscs',
# bz2_codec codec
'bz2' : 'bz2_codec',
# cp037 codec
'037' : 'cp037',
'csibm037' : 'cp037',
'ebcdic_cp_ca' : 'cp037',
'ebcdic_cp_nl' : 'cp037',
'ebcdic_cp_us' : 'cp037',
'ebcdic_cp_wt' : 'cp037',
'ibm037' : 'cp037',
'ibm039' : 'cp037',
# cp1026 codec
'1026' : 'cp1026',
'csibm1026' : 'cp1026',
'ibm1026' : 'cp1026',
# cp1125 codec
'1125' : 'cp1125',
'ibm1125' : 'cp1125',
'cp866u' : 'cp1125',
'ruscii' : 'cp1125',
# cp1140 codec
'1140' : 'cp1140',
'ibm1140' : 'cp1140',
# cp1250 codec
'1250' : 'cp1250',
'windows_1250' : 'cp1250',
# cp1251 codec
'1251' : 'cp1251',
'windows_1251' : 'cp1251',
# cp1252 codec
'1252' : 'cp1252',
'windows_1252' : 'cp1252',
# cp1253 codec
'1253' : 'cp1253',
'windows_1253' : 'cp1253',
# cp1254 codec
'1254' : 'cp1254',
'windows_1254' : 'cp1254',
# cp1255 codec
'1255' : 'cp1255',
'windows_1255' : 'cp1255',
# cp1256 codec
'1256' : 'cp1256',
'windows_1256' : 'cp1256',
# cp1257 codec
'1257' : 'cp1257',
'windows_1257' : 'cp1257',
# cp1258 codec
'1258' : 'cp1258',
'windows_1258' : 'cp1258',
# cp273 codec
'273' : 'cp273',
'ibm273' : 'cp273',
'csibm273' : 'cp273',
# cp424 codec
'424' : 'cp424',
'csibm424' : 'cp424',
'ebcdic_cp_he' : 'cp424',
'ibm424' : 'cp424',
# cp437 codec
'437' : 'cp437',
'cspc8codepage437' : 'cp437',
'ibm437' : 'cp437',
# cp500 codec
'500' : 'cp500',
'csibm500' : 'cp500',
'ebcdic_cp_be' : 'cp500',
'ebcdic_cp_ch' : 'cp500',
'ibm500' : 'cp500',
# cp775 codec
'775' : 'cp775',
'cspc775baltic' : 'cp775',
'ibm775' : 'cp775',
# cp850 codec
'850' : 'cp850',
'cspc850multilingual' : 'cp850',
'ibm850' : 'cp850',
# cp852 codec
'852' : 'cp852',
'cspcp852' : 'cp852',
'ibm852' : 'cp852',
# cp855 codec
'855' : 'cp855',
'csibm855' : 'cp855',
'ibm855' : 'cp855',
# cp857 codec
'857' : 'cp857',
'csibm857' : 'cp857',
'ibm857' : 'cp857',
# cp858 codec
'858' : 'cp858',
'csibm858' : 'cp858',
'ibm858' : 'cp858',
# cp860 codec
'860' : 'cp860',
'csibm860' : 'cp860',
'ibm860' : 'cp860',
# cp861 codec
'861' : 'cp861',
'cp_is' : 'cp861',
'csibm861' : 'cp861',
'ibm861' : 'cp861',
# cp862 codec
'862' : 'cp862',
'cspc862latinhebrew' : 'cp862',
'ibm862' : 'cp862',
# cp863 codec
'863' : 'cp863',
'csibm863' : 'cp863',
'ibm863' : 'cp863',
# cp864 codec
'864' : 'cp864',
'csibm864' : 'cp864',
'ibm864' : 'cp864',
# cp865 codec
'865' : 'cp865',
'csibm865' : 'cp865',
'ibm865' : 'cp865',
# cp866 codec
'866' : 'cp866',
'csibm866' : 'cp866',
'ibm866' : 'cp866',
# cp869 codec
'869' : 'cp869',
'cp_gr' : 'cp869',
'csibm869' : 'cp869',
'ibm869' : 'cp869',
# cp874 codec
'874' : 'cp874',
'ms874' : 'cp874',
'windows_874' : 'cp874',
# cp932 codec
'932' : 'cp932',
'ms932' : 'cp932',
'mskanji' : 'cp932',
'ms_kanji' : 'cp932',
'windows_31j' : 'cp932',
# cp949 codec
'949' : 'cp949',
'ms949' : 'cp949',
'uhc' : 'cp949',
# cp950 codec
'950' : 'cp950',
'ms950' : 'cp950',
# euc_jis_2004 codec
'jisx0213' : 'euc_jis_2004',
'eucjis2004' : 'euc_jis_2004',
'euc_jis2004' : 'euc_jis_2004',
# euc_jisx0213 codec
'eucjisx0213' : 'euc_jisx0213',
# euc_jp codec
'eucjp' : 'euc_jp',
'ujis' : 'euc_jp',
'u_jis' : 'euc_jp',
# euc_kr codec
'euckr' : 'euc_kr',
'korean' : 'euc_kr',
'ksc5601' : 'euc_kr',
'ks_c_5601' : 'euc_kr',
'ks_c_5601_1987' : 'euc_kr',
'ksx1001' : 'euc_kr',
'ks_x_1001' : 'euc_kr',
'cseuckr' : 'euc_kr',
# gb18030 codec
'gb18030_2000' : 'gb18030',
# gb2312 codec
'chinese' : 'gb2312',
'csiso58gb231280' : 'gb2312',
'euc_cn' : 'gb2312',
'euccn' : 'gb2312',
'eucgb2312_cn' : 'gb2312',
'gb2312_1980' : 'gb2312',
'gb2312_80' : 'gb2312',
'iso_ir_58' : 'gb2312',
# gbk codec
'936' : 'gbk',
'cp936' : 'gbk',
'ms936' : 'gbk',
# hex_codec codec
'hex' : 'hex_codec',
# hp_roman8 codec
'roman8' : 'hp_roman8',
'r8' : 'hp_roman8',
'csHPRoman8' : 'hp_roman8',
'cp1051' : 'hp_roman8',
'ibm1051' : 'hp_roman8',
# hz codec
'hzgb' : 'hz',
'hz_gb' : 'hz',
'hz_gb_2312' : 'hz',
# iso2022_jp codec
'csiso2022jp' : 'iso2022_jp',
'iso2022jp' : 'iso2022_jp',
'iso_2022_jp' : 'iso2022_jp',
# iso2022_jp_1 codec
'iso2022jp_1' : 'iso2022_jp_1',
'iso_2022_jp_1' : 'iso2022_jp_1',
# iso2022_jp_2 codec
'iso2022jp_2' : 'iso2022_jp_2',
'iso_2022_jp_2' : 'iso2022_jp_2',
# iso2022_jp_2004 codec
'iso_2022_jp_2004' : 'iso2022_jp_2004',
'iso2022jp_2004' : 'iso2022_jp_2004',
# iso2022_jp_3 codec
'iso2022jp_3' : 'iso2022_jp_3',
'iso_2022_jp_3' : 'iso2022_jp_3',
# iso2022_jp_ext codec
'iso2022jp_ext' : 'iso2022_jp_ext',
'iso_2022_jp_ext' : 'iso2022_jp_ext',
# iso2022_kr codec
'csiso2022kr' : 'iso2022_kr',
'iso2022kr' : 'iso2022_kr',
'iso_2022_kr' : 'iso2022_kr',
# iso8859_10 codec
'csisolatin6' : 'iso8859_10',
'iso_8859_10' : 'iso8859_10',
'iso_8859_10_1992' : 'iso8859_10',
'iso_ir_157' : 'iso8859_10',
'l6' : 'iso8859_10',
'latin6' : 'iso8859_10',
# iso8859_11 codec
'thai' : 'iso8859_11',
'iso_8859_11' : 'iso8859_11',
'iso_8859_11_2001' : 'iso8859_11',
# iso8859_13 codec
'iso_8859_13' : 'iso8859_13',
'l7' : 'iso8859_13',
'latin7' : 'iso8859_13',
# iso8859_14 codec
'iso_8859_14' : 'iso8859_14',
'iso_8859_14_1998' : 'iso8859_14',
'iso_celtic' : 'iso8859_14',
'iso_ir_199' : 'iso8859_14',
'l8' : 'iso8859_14',
'latin8' : 'iso8859_14',
# iso8859_15 codec
'iso_8859_15' : 'iso8859_15',
'l9' : 'iso8859_15',
'latin9' : 'iso8859_15',
# iso8859_16 codec
'iso_8859_16' : 'iso8859_16',
'iso_8859_16_2001' : 'iso8859_16',
'iso_ir_226' : 'iso8859_16',
'l10' : 'iso8859_16',
'latin10' : 'iso8859_16',
# iso8859_2 codec
'csisolatin2' : 'iso8859_2',
'iso_8859_2' : 'iso8859_2',
'iso_8859_2_1987' : 'iso8859_2',
'iso_ir_101' : 'iso8859_2',
'l2' : 'iso8859_2',
'latin2' : 'iso8859_2',
# iso8859_3 codec
'csisolatin3' : 'iso8859_3',
'iso_8859_3' : 'iso8859_3',
'iso_8859_3_1988' : 'iso8859_3',
'iso_ir_109' : 'iso8859_3',
'l3' : 'iso8859_3',
'latin3' : 'iso8859_3',
# iso8859_4 codec
'csisolatin4' : 'iso8859_4',
'iso_8859_4' : 'iso8859_4',
'iso_8859_4_1988' : 'iso8859_4',
'iso_ir_110' : 'iso8859_4',
'l4' : 'iso8859_4',
'latin4' : 'iso8859_4',
# iso8859_5 codec
'csisolatincyrillic' : 'iso8859_5',
'cyrillic' : 'iso8859_5',
'iso_8859_5' : 'iso8859_5',
'iso_8859_5_1988' : 'iso8859_5',
'iso_ir_144' : 'iso8859_5',
# iso8859_6 codec
'arabic' : 'iso8859_6',
'asmo_708' : 'iso8859_6',
'csisolatinarabic' : 'iso8859_6',
'ecma_114' : 'iso8859_6',
'iso_8859_6' : 'iso8859_6',
'iso_8859_6_1987' : 'iso8859_6',
'iso_ir_127' : 'iso8859_6',
# iso8859_7 codec
'csisolatingreek' : 'iso8859_7',
'ecma_118' : 'iso8859_7',
'elot_928' : 'iso8859_7',
'greek' : 'iso8859_7',
'greek8' : 'iso8859_7',
'iso_8859_7' : 'iso8859_7',
'iso_8859_7_1987' : 'iso8859_7',
'iso_ir_126' : 'iso8859_7',
# iso8859_8 codec
'csisolatinhebrew' : 'iso8859_8',
'hebrew' : 'iso8859_8',
'iso_8859_8' : 'iso8859_8',
'iso_8859_8_1988' : 'iso8859_8',
'iso_ir_138' : 'iso8859_8',
'iso_8859_8_i' : 'iso8859_8',
'iso_8859_8_e' : 'iso8859_8',
# iso8859_9 codec
'csisolatin5' : 'iso8859_9',
'iso_8859_9' : 'iso8859_9',
'iso_8859_9_1989' : 'iso8859_9',
'iso_ir_148' : 'iso8859_9',
'l5' : 'iso8859_9',
'latin5' : 'iso8859_9',
# johab codec
'cp1361' : 'johab',
'ms1361' : 'johab',
# koi8_r codec
'cskoi8r' : 'koi8_r',
# kz1048 codec
'kz_1048' : 'kz1048',
'rk1048' : 'kz1048',
'strk1048_2002' : 'kz1048',
# latin_1 codec
#
# Note that the latin_1 codec is implemented internally in C and a
# lot faster than the charmap codec iso8859_1 which uses the same
# encoding. This is why we discourage the use of the iso8859_1
# codec and alias it to latin_1 instead.
#
'8859' : 'latin_1',
'cp819' : 'latin_1',
'csisolatin1' : 'latin_1',
'ibm819' : 'latin_1',
'iso8859' : 'latin_1',
'iso8859_1' : 'latin_1',
'iso_8859_1' : 'latin_1',
'iso_8859_1_1987' : 'latin_1',
'iso_ir_100' : 'latin_1',
'l1' : 'latin_1',
'latin' : 'latin_1',
'latin1' : 'latin_1',
# mac_cyrillic codec
'maccyrillic' : 'mac_cyrillic',
# mac_greek codec
'macgreek' : 'mac_greek',
# mac_iceland codec
'maciceland' : 'mac_iceland',
# mac_latin2 codec
'maccentraleurope' : 'mac_latin2',
'mac_centeuro' : 'mac_latin2',
'maclatin2' : 'mac_latin2',
# mac_roman codec
'macintosh' : 'mac_roman',
'macroman' : 'mac_roman',
# mac_turkish codec
'macturkish' : 'mac_turkish',
# mbcs codec
'ansi' : 'mbcs',
'dbcs' : 'mbcs',
# ptcp154 codec
'csptcp154' : 'ptcp154',
'pt154' : 'ptcp154',
'cp154' : 'ptcp154',
'cyrillic_asian' : 'ptcp154',
# quopri_codec codec
'quopri' : 'quopri_codec',
'quoted_printable' : 'quopri_codec',
'quotedprintable' : 'quopri_codec',
# rot_13 codec
'rot13' : 'rot_13',
# shift_jis codec
'csshiftjis' : 'shift_jis',
'shiftjis' : 'shift_jis',
'sjis' : 'shift_jis',
's_jis' : 'shift_jis',
# shift_jis_2004 codec
'shiftjis2004' : 'shift_jis_2004',
'sjis_2004' : 'shift_jis_2004',
's_jis_2004' : 'shift_jis_2004',
# shift_jisx0213 codec
'shiftjisx0213' : 'shift_jisx0213',
'sjisx0213' : 'shift_jisx0213',
's_jisx0213' : 'shift_jisx0213',
# tis_620 codec
'tis620' : 'tis_620',
'tis_620_0' : 'tis_620',
'tis_620_2529_0' : 'tis_620',
'tis_620_2529_1' : 'tis_620',
'iso_ir_166' : 'tis_620',
# utf_16 codec
'u16' : 'utf_16',
'utf16' : 'utf_16',
# utf_16_be codec
'unicodebigunmarked' : 'utf_16_be',
'utf_16be' : 'utf_16_be',
# utf_16_le codec
'unicodelittleunmarked' : 'utf_16_le',
'utf_16le' : 'utf_16_le',
# utf_32 codec
'u32' : 'utf_32',
'utf32' : 'utf_32',
# utf_32_be codec
'utf_32be' : 'utf_32_be',
# utf_32_le codec
'utf_32le' : 'utf_32_le',
# utf_7 codec
'u7' : 'utf_7',
'utf7' : 'utf_7',
'unicode_1_1_utf_7' : 'utf_7',
# utf_8 codec
'u8' : 'utf_8',
'utf' : 'utf_8',
'utf8' : 'utf_8',
'utf8_ucs2' : 'utf_8',
'utf8_ucs4' : 'utf_8',
'cp65001' : 'utf_8',
# uu_codec codec
'uu' : 'uu_codec',
# zlib_codec codec
'zip' : 'zlib_codec',
'zlib' : 'zlib_codec',
# temporary mac CJK aliases, will be replaced by proper codecs in 3.1
'x_mac_japanese' : 'shift_jis',
'x_mac_korean' : 'euc_kr',
'x_mac_simp_chinese' : 'gb2312',
'x_mac_trad_chinese' : 'big5',
}

View file

@ -0,0 +1,50 @@
""" Python 'ascii' Codec
Written by Marc-Andre Lemburg (mal@lemburg.com).
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
"""
import codecs
### Codec APIs
class Codec(codecs.Codec):
# Note: Binding these as C functions will result in the class not
# converting them to methods. This is intended.
encode = codecs.ascii_encode
decode = codecs.ascii_decode
class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
return codecs.ascii_encode(input, self.errors)[0]
class IncrementalDecoder(codecs.IncrementalDecoder):
def decode(self, input, final=False):
return codecs.ascii_decode(input, self.errors)[0]
class StreamWriter(Codec,codecs.StreamWriter):
pass
class StreamReader(Codec,codecs.StreamReader):
pass
class StreamConverter(StreamWriter,StreamReader):
encode = codecs.ascii_decode
decode = codecs.ascii_encode
### encodings module API
def getregentry():
return codecs.CodecInfo(
name='ascii',
encode=Codec.encode,
decode=Codec.decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamwriter=StreamWriter,
streamreader=StreamReader,
)

View file

@ -0,0 +1,55 @@
"""Python 'base64_codec' Codec - base64 content transfer encoding.
This codec de/encodes from bytes to bytes.
Written by Marc-Andre Lemburg (mal@lemburg.com).
"""
import codecs
import base64
### Codec APIs
def base64_encode(input, errors='strict'):
assert errors == 'strict'
return (base64.encodebytes(input), len(input))
def base64_decode(input, errors='strict'):
assert errors == 'strict'
return (base64.decodebytes(input), len(input))
class Codec(codecs.Codec):
def encode(self, input, errors='strict'):
return base64_encode(input, errors)
def decode(self, input, errors='strict'):
return base64_decode(input, errors)
class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
assert self.errors == 'strict'
return base64.encodebytes(input)
class IncrementalDecoder(codecs.IncrementalDecoder):
def decode(self, input, final=False):
assert self.errors == 'strict'
return base64.decodebytes(input)
class StreamWriter(Codec, codecs.StreamWriter):
charbuffertype = bytes
class StreamReader(Codec, codecs.StreamReader):
charbuffertype = bytes
### encodings module API
def getregentry():
return codecs.CodecInfo(
name='base64',
encode=base64_encode,
decode=base64_decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamwriter=StreamWriter,
streamreader=StreamReader,
_is_text_encoding=False,
)

View file

@ -0,0 +1,39 @@
#
# big5.py: Python Unicode Codec for BIG5
#
# Written by Hye-Shik Chang <perky@FreeBSD.org>
#
import _codecs_tw, codecs
import _multibytecodec as mbc
codec = _codecs_tw.getcodec('big5')
class Codec(codecs.Codec):
encode = codec.encode
decode = codec.decode
class IncrementalEncoder(mbc.MultibyteIncrementalEncoder,
codecs.IncrementalEncoder):
codec = codec
class IncrementalDecoder(mbc.MultibyteIncrementalDecoder,
codecs.IncrementalDecoder):
codec = codec
class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader):
codec = codec
class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter):
codec = codec
def getregentry():
return codecs.CodecInfo(
name='big5',
encode=Codec().encode,
decode=Codec().decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamreader=StreamReader,
streamwriter=StreamWriter,
)

View file

@ -0,0 +1,39 @@
#
# big5hkscs.py: Python Unicode Codec for BIG5HKSCS
#
# Written by Hye-Shik Chang <perky@FreeBSD.org>
#
import _codecs_hk, codecs
import _multibytecodec as mbc
codec = _codecs_hk.getcodec('big5hkscs')
class Codec(codecs.Codec):
encode = codec.encode
decode = codec.decode
class IncrementalEncoder(mbc.MultibyteIncrementalEncoder,
codecs.IncrementalEncoder):
codec = codec
class IncrementalDecoder(mbc.MultibyteIncrementalDecoder,
codecs.IncrementalDecoder):
codec = codec
class StreamReader(Codec, mbc.MultibyteStreamReader, codecs.StreamReader):
codec = codec
class StreamWriter(Codec, mbc.MultibyteStreamWriter, codecs.StreamWriter):
codec = codec
def getregentry():
return codecs.CodecInfo(
name='big5hkscs',
encode=Codec().encode,
decode=Codec().decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamreader=StreamReader,
streamwriter=StreamWriter,
)

View file

@ -0,0 +1,78 @@
"""Python 'bz2_codec' Codec - bz2 compression encoding.
This codec de/encodes from bytes to bytes and is therefore usable with
bytes.transform() and bytes.untransform().
Adapted by Raymond Hettinger from zlib_codec.py which was written
by Marc-Andre Lemburg (mal@lemburg.com).
"""
import codecs
import bz2 # this codec needs the optional bz2 module !
### Codec APIs
def bz2_encode(input, errors='strict'):
assert errors == 'strict'
return (bz2.compress(input), len(input))
def bz2_decode(input, errors='strict'):
assert errors == 'strict'
return (bz2.decompress(input), len(input))
class Codec(codecs.Codec):
def encode(self, input, errors='strict'):
return bz2_encode(input, errors)
def decode(self, input, errors='strict'):
return bz2_decode(input, errors)
class IncrementalEncoder(codecs.IncrementalEncoder):
def __init__(self, errors='strict'):
assert errors == 'strict'
self.errors = errors
self.compressobj = bz2.BZ2Compressor()
def encode(self, input, final=False):
if final:
c = self.compressobj.compress(input)
return c + self.compressobj.flush()
else:
return self.compressobj.compress(input)
def reset(self):
self.compressobj = bz2.BZ2Compressor()
class IncrementalDecoder(codecs.IncrementalDecoder):
def __init__(self, errors='strict'):
assert errors == 'strict'
self.errors = errors
self.decompressobj = bz2.BZ2Decompressor()
def decode(self, input, final=False):
try:
return self.decompressobj.decompress(input)
except EOFError:
return ''
def reset(self):
self.decompressobj = bz2.BZ2Decompressor()
class StreamWriter(Codec, codecs.StreamWriter):
charbuffertype = bytes
class StreamReader(Codec, codecs.StreamReader):
charbuffertype = bytes
### encodings module API
def getregentry():
return codecs.CodecInfo(
name="bz2",
encode=bz2_encode,
decode=bz2_decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamwriter=StreamWriter,
streamreader=StreamReader,
_is_text_encoding=False,
)

View file

@ -0,0 +1,69 @@
""" Generic Python Character Mapping Codec.
Use this codec directly rather than through the automatic
conversion mechanisms supplied by unicode() and .encode().
Written by Marc-Andre Lemburg (mal@lemburg.com).
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
"""#"
import codecs
### Codec APIs
class Codec(codecs.Codec):
# Note: Binding these as C functions will result in the class not
# converting them to methods. This is intended.
encode = codecs.charmap_encode
decode = codecs.charmap_decode
class IncrementalEncoder(codecs.IncrementalEncoder):
def __init__(self, errors='strict', mapping=None):
codecs.IncrementalEncoder.__init__(self, errors)
self.mapping = mapping
def encode(self, input, final=False):
return codecs.charmap_encode(input, self.errors, self.mapping)[0]
class IncrementalDecoder(codecs.IncrementalDecoder):
def __init__(self, errors='strict', mapping=None):
codecs.IncrementalDecoder.__init__(self, errors)
self.mapping = mapping
def decode(self, input, final=False):
return codecs.charmap_decode(input, self.errors, self.mapping)[0]
class StreamWriter(Codec,codecs.StreamWriter):
def __init__(self,stream,errors='strict',mapping=None):
codecs.StreamWriter.__init__(self,stream,errors)
self.mapping = mapping
def encode(self,input,errors='strict'):
return Codec.encode(input,errors,self.mapping)
class StreamReader(Codec,codecs.StreamReader):
def __init__(self,stream,errors='strict',mapping=None):
codecs.StreamReader.__init__(self,stream,errors)
self.mapping = mapping
def decode(self,input,errors='strict'):
return Codec.decode(input,errors,self.mapping)
### encodings module API
def getregentry():
return codecs.CodecInfo(
name='charmap',
encode=Codec.encode,
decode=Codec.decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamwriter=StreamWriter,
streamreader=StreamReader,
)

View file

@ -0,0 +1,307 @@
""" Python Character Mapping Codec cp037 generated from 'MAPPINGS/VENDORS/MICSFT/EBCDIC/CP037.TXT' with gencodec.py.
"""#"
import codecs
### Codec APIs
class Codec(codecs.Codec):
def encode(self,input,errors='strict'):
return codecs.charmap_encode(input,errors,encoding_table)
def decode(self,input,errors='strict'):
return codecs.charmap_decode(input,errors,decoding_table)
class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
return codecs.charmap_encode(input,self.errors,encoding_table)[0]
class IncrementalDecoder(codecs.IncrementalDecoder):
def decode(self, input, final=False):
return codecs.charmap_decode(input,self.errors,decoding_table)[0]
class StreamWriter(Codec,codecs.StreamWriter):
pass
class StreamReader(Codec,codecs.StreamReader):
pass
### encodings module API
def getregentry():
return codecs.CodecInfo(
name='cp037',
encode=Codec().encode,
decode=Codec().decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamreader=StreamReader,
streamwriter=StreamWriter,
)
### Decoding Table
decoding_table = (
'\x00' # 0x00 -> NULL
'\x01' # 0x01 -> START OF HEADING
'\x02' # 0x02 -> START OF TEXT
'\x03' # 0x03 -> END OF TEXT
'\x9c' # 0x04 -> CONTROL
'\t' # 0x05 -> HORIZONTAL TABULATION
'\x86' # 0x06 -> CONTROL
'\x7f' # 0x07 -> DELETE
'\x97' # 0x08 -> CONTROL
'\x8d' # 0x09 -> CONTROL
'\x8e' # 0x0A -> CONTROL
'\x0b' # 0x0B -> VERTICAL TABULATION
'\x0c' # 0x0C -> FORM FEED
'\r' # 0x0D -> CARRIAGE RETURN
'\x0e' # 0x0E -> SHIFT OUT
'\x0f' # 0x0F -> SHIFT IN
'\x10' # 0x10 -> DATA LINK ESCAPE
'\x11' # 0x11 -> DEVICE CONTROL ONE
'\x12' # 0x12 -> DEVICE CONTROL TWO
'\x13' # 0x13 -> DEVICE CONTROL THREE
'\x9d' # 0x14 -> CONTROL
'\x85' # 0x15 -> CONTROL
'\x08' # 0x16 -> BACKSPACE
'\x87' # 0x17 -> CONTROL
'\x18' # 0x18 -> CANCEL
'\x19' # 0x19 -> END OF MEDIUM
'\x92' # 0x1A -> CONTROL
'\x8f' # 0x1B -> CONTROL
'\x1c' # 0x1C -> FILE SEPARATOR
'\x1d' # 0x1D -> GROUP SEPARATOR
'\x1e' # 0x1E -> RECORD SEPARATOR
'\x1f' # 0x1F -> UNIT SEPARATOR
'\x80' # 0x20 -> CONTROL
'\x81' # 0x21 -> CONTROL
'\x82' # 0x22 -> CONTROL
'\x83' # 0x23 -> CONTROL
'\x84' # 0x24 -> CONTROL
'\n' # 0x25 -> LINE FEED
'\x17' # 0x26 -> END OF TRANSMISSION BLOCK
'\x1b' # 0x27 -> ESCAPE
'\x88' # 0x28 -> CONTROL
'\x89' # 0x29 -> CONTROL
'\x8a' # 0x2A -> CONTROL
'\x8b' # 0x2B -> CONTROL
'\x8c' # 0x2C -> CONTROL
'\x05' # 0x2D -> ENQUIRY
'\x06' # 0x2E -> ACKNOWLEDGE
'\x07' # 0x2F -> BELL
'\x90' # 0x30 -> CONTROL
'\x91' # 0x31 -> CONTROL
'\x16' # 0x32 -> SYNCHRONOUS IDLE
'\x93' # 0x33 -> CONTROL
'\x94' # 0x34 -> CONTROL
'\x95' # 0x35 -> CONTROL
'\x96' # 0x36 -> CONTROL
'\x04' # 0x37 -> END OF TRANSMISSION
'\x98' # 0x38 -> CONTROL
'\x99' # 0x39 -> CONTROL
'\x9a' # 0x3A -> CONTROL
'\x9b' # 0x3B -> CONTROL
'\x14' # 0x3C -> DEVICE CONTROL FOUR
'\x15' # 0x3D -> NEGATIVE ACKNOWLEDGE
'\x9e' # 0x3E -> CONTROL
'\x1a' # 0x3F -> SUBSTITUTE
' ' # 0x40 -> SPACE
'\xa0' # 0x41 -> NO-BREAK SPACE
'\xe2' # 0x42 -> LATIN SMALL LETTER A WITH CIRCUMFLEX
'\xe4' # 0x43 -> LATIN SMALL LETTER A WITH DIAERESIS
'\xe0' # 0x44 -> LATIN SMALL LETTER A WITH GRAVE
'\xe1' # 0x45 -> LATIN SMALL LETTER A WITH ACUTE
'\xe3' # 0x46 -> LATIN SMALL LETTER A WITH TILDE
'\xe5' # 0x47 -> LATIN SMALL LETTER A WITH RING ABOVE
'\xe7' # 0x48 -> LATIN SMALL LETTER C WITH CEDILLA
'\xf1' # 0x49 -> LATIN SMALL LETTER N WITH TILDE
'\xa2' # 0x4A -> CENT SIGN
'.' # 0x4B -> FULL STOP
'<' # 0x4C -> LESS-THAN SIGN
'(' # 0x4D -> LEFT PARENTHESIS
'+' # 0x4E -> PLUS SIGN
'|' # 0x4F -> VERTICAL LINE
'&' # 0x50 -> AMPERSAND
'\xe9' # 0x51 -> LATIN SMALL LETTER E WITH ACUTE
'\xea' # 0x52 -> LATIN SMALL LETTER E WITH CIRCUMFLEX
'\xeb' # 0x53 -> LATIN SMALL LETTER E WITH DIAERESIS
'\xe8' # 0x54 -> LATIN SMALL LETTER E WITH GRAVE
'\xed' # 0x55 -> LATIN SMALL LETTER I WITH ACUTE
'\xee' # 0x56 -> LATIN SMALL LETTER I WITH CIRCUMFLEX
'\xef' # 0x57 -> LATIN SMALL LETTER I WITH DIAERESIS
'\xec' # 0x58 -> LATIN SMALL LETTER I WITH GRAVE
'\xdf' # 0x59 -> LATIN SMALL LETTER SHARP S (GERMAN)
'!' # 0x5A -> EXCLAMATION MARK
'$' # 0x5B -> DOLLAR SIGN
'*' # 0x5C -> ASTERISK
')' # 0x5D -> RIGHT PARENTHESIS
';' # 0x5E -> SEMICOLON
'\xac' # 0x5F -> NOT SIGN
'-' # 0x60 -> HYPHEN-MINUS
'/' # 0x61 -> SOLIDUS
'\xc2' # 0x62 -> LATIN CAPITAL LETTER A WITH CIRCUMFLEX
'\xc4' # 0x63 -> LATIN CAPITAL LETTER A WITH DIAERESIS
'\xc0' # 0x64 -> LATIN CAPITAL LETTER A WITH GRAVE
'\xc1' # 0x65 -> LATIN CAPITAL LETTER A WITH ACUTE
'\xc3' # 0x66 -> LATIN CAPITAL LETTER A WITH TILDE
'\xc5' # 0x67 -> LATIN CAPITAL LETTER A WITH RING ABOVE
'\xc7' # 0x68 -> LATIN CAPITAL LETTER C WITH CEDILLA
'\xd1' # 0x69 -> LATIN CAPITAL LETTER N WITH TILDE
'\xa6' # 0x6A -> BROKEN BAR
',' # 0x6B -> COMMA
'%' # 0x6C -> PERCENT SIGN
'_' # 0x6D -> LOW LINE
'>' # 0x6E -> GREATER-THAN SIGN
'?' # 0x6F -> QUESTION MARK
'\xf8' # 0x70 -> LATIN SMALL LETTER O WITH STROKE
'\xc9' # 0x71 -> LATIN CAPITAL LETTER E WITH ACUTE
'\xca' # 0x72 -> LATIN CAPITAL LETTER E WITH CIRCUMFLEX
'\xcb' # 0x73 -> LATIN CAPITAL LETTER E WITH DIAERESIS
'\xc8' # 0x74 -> LATIN CAPITAL LETTER E WITH GRAVE
'\xcd' # 0x75 -> LATIN CAPITAL LETTER I WITH ACUTE
'\xce' # 0x76 -> LATIN CAPITAL LETTER I WITH CIRCUMFLEX
'\xcf' # 0x77 -> LATIN CAPITAL LETTER I WITH DIAERESIS
'\xcc' # 0x78 -> LATIN CAPITAL LETTER I WITH GRAVE
'`' # 0x79 -> GRAVE ACCENT
':' # 0x7A -> COLON
'#' # 0x7B -> NUMBER SIGN
'@' # 0x7C -> COMMERCIAL AT
"'" # 0x7D -> APOSTROPHE
'=' # 0x7E -> EQUALS SIGN
'"' # 0x7F -> QUOTATION MARK
'\xd8' # 0x80 -> LATIN CAPITAL LETTER O WITH STROKE
'a' # 0x81 -> LATIN SMALL LETTER A
'b' # 0x82 -> LATIN SMALL LETTER B
'c' # 0x83 -> LATIN SMALL LETTER C
'd' # 0x84 -> LATIN SMALL LETTER D
'e' # 0x85 -> LATIN SMALL LETTER E
'f' # 0x86 -> LATIN SMALL LETTER F
'g' # 0x87 -> LATIN SMALL LETTER G
'h' # 0x88 -> LATIN SMALL LETTER H
'i' # 0x89 -> LATIN SMALL LETTER I
'\xab' # 0x8A -> LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
'\xbb' # 0x8B -> RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
'\xf0' # 0x8C -> LATIN SMALL LETTER ETH (ICELANDIC)
'\xfd' # 0x8D -> LATIN SMALL LETTER Y WITH ACUTE
'\xfe' # 0x8E -> LATIN SMALL LETTER THORN (ICELANDIC)
'\xb1' # 0x8F -> PLUS-MINUS SIGN
'\xb0' # 0x90 -> DEGREE SIGN
'j' # 0x91 -> LATIN SMALL LETTER J
'k' # 0x92 -> LATIN SMALL LETTER K
'l' # 0x93 -> LATIN SMALL LETTER L
'm' # 0x94 -> LATIN SMALL LETTER M
'n' # 0x95 -> LATIN SMALL LETTER N
'o' # 0x96 -> LATIN SMALL LETTER O
'p' # 0x97 -> LATIN SMALL LETTER P
'q' # 0x98 -> LATIN SMALL LETTER Q
'r' # 0x99 -> LATIN SMALL LETTER R
'\xaa' # 0x9A -> FEMININE ORDINAL INDICATOR
'\xba' # 0x9B -> MASCULINE ORDINAL INDICATOR
'\xe6' # 0x9C -> LATIN SMALL LIGATURE AE
'\xb8' # 0x9D -> CEDILLA
'\xc6' # 0x9E -> LATIN CAPITAL LIGATURE AE
'\xa4' # 0x9F -> CURRENCY SIGN
'\xb5' # 0xA0 -> MICRO SIGN
'~' # 0xA1 -> TILDE
's' # 0xA2 -> LATIN SMALL LETTER S
't' # 0xA3 -> LATIN SMALL LETTER T
'u' # 0xA4 -> LATIN SMALL LETTER U
'v' # 0xA5 -> LATIN SMALL LETTER V
'w' # 0xA6 -> LATIN SMALL LETTER W
'x' # 0xA7 -> LATIN SMALL LETTER X
'y' # 0xA8 -> LATIN SMALL LETTER Y
'z' # 0xA9 -> LATIN SMALL LETTER Z
'\xa1' # 0xAA -> INVERTED EXCLAMATION MARK
'\xbf' # 0xAB -> INVERTED QUESTION MARK
'\xd0' # 0xAC -> LATIN CAPITAL LETTER ETH (ICELANDIC)
'\xdd' # 0xAD -> LATIN CAPITAL LETTER Y WITH ACUTE
'\xde' # 0xAE -> LATIN CAPITAL LETTER THORN (ICELANDIC)
'\xae' # 0xAF -> REGISTERED SIGN
'^' # 0xB0 -> CIRCUMFLEX ACCENT
'\xa3' # 0xB1 -> POUND SIGN
'\xa5' # 0xB2 -> YEN SIGN
'\xb7' # 0xB3 -> MIDDLE DOT
'\xa9' # 0xB4 -> COPYRIGHT SIGN
'\xa7' # 0xB5 -> SECTION SIGN
'\xb6' # 0xB6 -> PILCROW SIGN
'\xbc' # 0xB7 -> VULGAR FRACTION ONE QUARTER
'\xbd' # 0xB8 -> VULGAR FRACTION ONE HALF
'\xbe' # 0xB9 -> VULGAR FRACTION THREE QUARTERS
'[' # 0xBA -> LEFT SQUARE BRACKET
']' # 0xBB -> RIGHT SQUARE BRACKET
'\xaf' # 0xBC -> MACRON
'\xa8' # 0xBD -> DIAERESIS
'\xb4' # 0xBE -> ACUTE ACCENT
'\xd7' # 0xBF -> MULTIPLICATION SIGN
'{' # 0xC0 -> LEFT CURLY BRACKET
'A' # 0xC1 -> LATIN CAPITAL LETTER A
'B' # 0xC2 -> LATIN CAPITAL LETTER B
'C' # 0xC3 -> LATIN CAPITAL LETTER C
'D' # 0xC4 -> LATIN CAPITAL LETTER D
'E' # 0xC5 -> LATIN CAPITAL LETTER E
'F' # 0xC6 -> LATIN CAPITAL LETTER F
'G' # 0xC7 -> LATIN CAPITAL LETTER G
'H' # 0xC8 -> LATIN CAPITAL LETTER H
'I' # 0xC9 -> LATIN CAPITAL LETTER I
'\xad' # 0xCA -> SOFT HYPHEN
'\xf4' # 0xCB -> LATIN SMALL LETTER O WITH CIRCUMFLEX
'\xf6' # 0xCC -> LATIN SMALL LETTER O WITH DIAERESIS
'\xf2' # 0xCD -> LATIN SMALL LETTER O WITH GRAVE
'\xf3' # 0xCE -> LATIN SMALL LETTER O WITH ACUTE
'\xf5' # 0xCF -> LATIN SMALL LETTER O WITH TILDE
'}' # 0xD0 -> RIGHT CURLY BRACKET
'J' # 0xD1 -> LATIN CAPITAL LETTER J
'K' # 0xD2 -> LATIN CAPITAL LETTER K
'L' # 0xD3 -> LATIN CAPITAL LETTER L
'M' # 0xD4 -> LATIN CAPITAL LETTER M
'N' # 0xD5 -> LATIN CAPITAL LETTER N
'O' # 0xD6 -> LATIN CAPITAL LETTER O
'P' # 0xD7 -> LATIN CAPITAL LETTER P
'Q' # 0xD8 -> LATIN CAPITAL LETTER Q
'R' # 0xD9 -> LATIN CAPITAL LETTER R
'\xb9' # 0xDA -> SUPERSCRIPT ONE
'\xfb' # 0xDB -> LATIN SMALL LETTER U WITH CIRCUMFLEX
'\xfc' # 0xDC -> LATIN SMALL LETTER U WITH DIAERESIS
'\xf9' # 0xDD -> LATIN SMALL LETTER U WITH GRAVE
'\xfa' # 0xDE -> LATIN SMALL LETTER U WITH ACUTE
'\xff' # 0xDF -> LATIN SMALL LETTER Y WITH DIAERESIS
'\\' # 0xE0 -> REVERSE SOLIDUS
'\xf7' # 0xE1 -> DIVISION SIGN
'S' # 0xE2 -> LATIN CAPITAL LETTER S
'T' # 0xE3 -> LATIN CAPITAL LETTER T
'U' # 0xE4 -> LATIN CAPITAL LETTER U
'V' # 0xE5 -> LATIN CAPITAL LETTER V
'W' # 0xE6 -> LATIN CAPITAL LETTER W
'X' # 0xE7 -> LATIN CAPITAL LETTER X
'Y' # 0xE8 -> LATIN CAPITAL LETTER Y
'Z' # 0xE9 -> LATIN CAPITAL LETTER Z
'\xb2' # 0xEA -> SUPERSCRIPT TWO
'\xd4' # 0xEB -> LATIN CAPITAL LETTER O WITH CIRCUMFLEX
'\xd6' # 0xEC -> LATIN CAPITAL LETTER O WITH DIAERESIS
'\xd2' # 0xED -> LATIN CAPITAL LETTER O WITH GRAVE
'\xd3' # 0xEE -> LATIN CAPITAL LETTER O WITH ACUTE
'\xd5' # 0xEF -> LATIN CAPITAL LETTER O WITH TILDE
'0' # 0xF0 -> DIGIT ZERO
'1' # 0xF1 -> DIGIT ONE
'2' # 0xF2 -> DIGIT TWO
'3' # 0xF3 -> DIGIT THREE
'4' # 0xF4 -> DIGIT FOUR
'5' # 0xF5 -> DIGIT FIVE
'6' # 0xF6 -> DIGIT SIX
'7' # 0xF7 -> DIGIT SEVEN
'8' # 0xF8 -> DIGIT EIGHT
'9' # 0xF9 -> DIGIT NINE
'\xb3' # 0xFA -> SUPERSCRIPT THREE
'\xdb' # 0xFB -> LATIN CAPITAL LETTER U WITH CIRCUMFLEX
'\xdc' # 0xFC -> LATIN CAPITAL LETTER U WITH DIAERESIS
'\xd9' # 0xFD -> LATIN CAPITAL LETTER U WITH GRAVE
'\xda' # 0xFE -> LATIN CAPITAL LETTER U WITH ACUTE
'\x9f' # 0xFF -> CONTROL
)
### Encoding table
encoding_table=codecs.charmap_build(decoding_table)

View file

@ -0,0 +1,307 @@
""" Python Character Mapping Codec cp1006 generated from 'MAPPINGS/VENDORS/MISC/CP1006.TXT' with gencodec.py.
"""#"
import codecs
### Codec APIs
class Codec(codecs.Codec):
def encode(self,input,errors='strict'):
return codecs.charmap_encode(input,errors,encoding_table)
def decode(self,input,errors='strict'):
return codecs.charmap_decode(input,errors,decoding_table)
class IncrementalEncoder(codecs.IncrementalEncoder):
def encode(self, input, final=False):
return codecs.charmap_encode(input,self.errors,encoding_table)[0]
class IncrementalDecoder(codecs.IncrementalDecoder):
def decode(self, input, final=False):
return codecs.charmap_decode(input,self.errors,decoding_table)[0]
class StreamWriter(Codec,codecs.StreamWriter):
pass
class StreamReader(Codec,codecs.StreamReader):
pass
### encodings module API
def getregentry():
return codecs.CodecInfo(
name='cp1006',
encode=Codec().encode,
decode=Codec().decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamreader=StreamReader,
streamwriter=StreamWriter,
)
### Decoding Table
decoding_table = (
'\x00' # 0x00 -> NULL
'\x01' # 0x01 -> START OF HEADING
'\x02' # 0x02 -> START OF TEXT
'\x03' # 0x03 -> END OF TEXT
'\x04' # 0x04 -> END OF TRANSMISSION
'\x05' # 0x05 -> ENQUIRY
'\x06' # 0x06 -> ACKNOWLEDGE
'\x07' # 0x07 -> BELL
'\x08' # 0x08 -> BACKSPACE
'\t' # 0x09 -> HORIZONTAL TABULATION
'\n' # 0x0A -> LINE FEED
'\x0b' # 0x0B -> VERTICAL TABULATION
'\x0c' # 0x0C -> FORM FEED
'\r' # 0x0D -> CARRIAGE RETURN
'\x0e' # 0x0E -> SHIFT OUT
'\x0f' # 0x0F -> SHIFT IN
'\x10' # 0x10 -> DATA LINK ESCAPE
'\x11' # 0x11 -> DEVICE CONTROL ONE
'\x12' # 0x12 -> DEVICE CONTROL TWO
'\x13' # 0x13 -> DEVICE CONTROL THREE
'\x14' # 0x14 -> DEVICE CONTROL FOUR
'\x15' # 0x15 -> NEGATIVE ACKNOWLEDGE
'\x16' # 0x16 -> SYNCHRONOUS IDLE
'\x17' # 0x17 -> END OF TRANSMISSION BLOCK
'\x18' # 0x18 -> CANCEL
'\x19' # 0x19 -> END OF MEDIUM
'\x1a' # 0x1A -> SUBSTITUTE
'\x1b' # 0x1B -> ESCAPE
'\x1c' # 0x1C -> FILE SEPARATOR
'\x1d' # 0x1D -> GROUP SEPARATOR
'\x1e' # 0x1E -> RECORD SEPARATOR
'\x1f' # 0x1F -> UNIT SEPARATOR
' ' # 0x20 -> SPACE
'!' # 0x21 -> EXCLAMATION MARK
'"' # 0x22 -> QUOTATION MARK
'#' # 0x23 -> NUMBER SIGN
'$' # 0x24 -> DOLLAR SIGN
'%' # 0x25 -> PERCENT SIGN
'&' # 0x26 -> AMPERSAND
"'" # 0x27 -> APOSTROPHE
'(' # 0x28 -> LEFT PARENTHESIS
')' # 0x29 -> RIGHT PARENTHESIS
'*' # 0x2A -> ASTERISK
'+' # 0x2B -> PLUS SIGN
',' # 0x2C -> COMMA
'-' # 0x2D -> HYPHEN-MINUS
'.' # 0x2E -> FULL STOP
'/' # 0x2F -> SOLIDUS
'0' # 0x30 -> DIGIT ZERO
'1' # 0x31 -> DIGIT ONE
'2' # 0x32 -> DIGIT TWO
'3' # 0x33 -> DIGIT THREE
'4' # 0x34 -> DIGIT FOUR
'5' # 0x35 -> DIGIT FIVE
'6' # 0x36 -> DIGIT SIX
'7' # 0x37 -> DIGIT SEVEN
'8' # 0x38 -> DIGIT EIGHT
'9' # 0x39 -> DIGIT NINE
':' # 0x3A -> COLON
';' # 0x3B -> SEMICOLON
'<' # 0x3C -> LESS-THAN SIGN
'=' # 0x3D -> EQUALS SIGN
'>' # 0x3E -> GREATER-THAN SIGN
'?' # 0x3F -> QUESTION MARK
'@' # 0x40 -> COMMERCIAL AT
'A' # 0x41 -> LATIN CAPITAL LETTER A
'B' # 0x42 -> LATIN CAPITAL LETTER B
'C' # 0x43 -> LATIN CAPITAL LETTER C
'D' # 0x44 -> LATIN CAPITAL LETTER D
'E' # 0x45 -> LATIN CAPITAL LETTER E
'F' # 0x46 -> LATIN CAPITAL LETTER F
'G' # 0x47 -> LATIN CAPITAL LETTER G
'H' # 0x48 -> LATIN CAPITAL LETTER H
'I' # 0x49 -> LATIN CAPITAL LETTER I
'J' # 0x4A -> LATIN CAPITAL LETTER J
'K' # 0x4B -> LATIN CAPITAL LETTER K
'L' # 0x4C -> LATIN CAPITAL LETTER L
'M' # 0x4D -> LATIN CAPITAL LETTER M
'N' # 0x4E -> LATIN CAPITAL LETTER N
'O' # 0x4F -> LATIN CAPITAL LETTER O
'P' # 0x50 -> LATIN CAPITAL LETTER P
'Q' # 0x51 -> LATIN CAPITAL LETTER Q
'R' # 0x52 -> LATIN CAPITAL LETTER R
'S' # 0x53 -> LATIN CAPITAL LETTER S
'T' # 0x54 -> LATIN CAPITAL LETTER T
'U' # 0x55 -> LATIN CAPITAL LETTER U
'V' # 0x56 -> LATIN CAPITAL LETTER V
'W' # 0x57 -> LATIN CAPITAL LETTER W
'X' # 0x58 -> LATIN CAPITAL LETTER X
'Y' # 0x59 -> LATIN CAPITAL LETTER Y
'Z' # 0x5A -> LATIN CAPITAL LETTER Z
'[' # 0x5B -> LEFT SQUARE BRACKET
'\\' # 0x5C -> REVERSE SOLIDUS
']' # 0x5D -> RIGHT SQUARE BRACKET
'^' # 0x5E -> CIRCUMFLEX ACCENT
'_' # 0x5F -> LOW LINE
'`' # 0x60 -> GRAVE ACCENT
'a' # 0x61 -> LATIN SMALL LETTER A
'b' # 0x62 -> LATIN SMALL LETTER B
'c' # 0x63 -> LATIN SMALL LETTER C
'd' # 0x64 -> LATIN SMALL LETTER D
'e' # 0x65 -> LATIN SMALL LETTER E
'f' # 0x66 -> LATIN SMALL LETTER F
'g' # 0x67 -> LATIN SMALL LETTER G
'h' # 0x68 -> LATIN SMALL LETTER H
'i' # 0x69 -> LATIN SMALL LETTER I
'j' # 0x6A -> LATIN SMALL LETTER J
'k' # 0x6B -> LATIN SMALL LETTER K
'l' # 0x6C -> LATIN SMALL LETTER L
'm' # 0x6D -> LATIN SMALL LETTER M
'n' # 0x6E -> LATIN SMALL LETTER N
'o' # 0x6F -> LATIN SMALL LETTER O
'p' # 0x70 -> LATIN SMALL LETTER P
'q' # 0x71 -> LATIN SMALL LETTER Q
'r' # 0x72 -> LATIN SMALL LETTER R
's' # 0x73 -> LATIN SMALL LETTER S
't' # 0x74 -> LATIN SMALL LETTER T
'u' # 0x75 -> LATIN SMALL LETTER U
'v' # 0x76 -> LATIN SMALL LETTER V
'w' # 0x77 -> LATIN SMALL LETTER W
'x' # 0x78 -> LATIN SMALL LETTER X
'y' # 0x79 -> LATIN SMALL LETTER Y
'z' # 0x7A -> LATIN SMALL LETTER Z
'{' # 0x7B -> LEFT CURLY BRACKET
'|' # 0x7C -> VERTICAL LINE
'}' # 0x7D -> RIGHT CURLY BRACKET
'~' # 0x7E -> TILDE
'\x7f' # 0x7F -> DELETE
'\x80' # 0x80 -> <control>
'\x81' # 0x81 -> <control>
'\x82' # 0x82 -> <control>
'\x83' # 0x83 -> <control>
'\x84' # 0x84 -> <control>
'\x85' # 0x85 -> <control>
'\x86' # 0x86 -> <control>
'\x87' # 0x87 -> <control>
'\x88' # 0x88 -> <control>
'\x89' # 0x89 -> <control>
'\x8a' # 0x8A -> <control>
'\x8b' # 0x8B -> <control>
'\x8c' # 0x8C -> <control>
'\x8d' # 0x8D -> <control>
'\x8e' # 0x8E -> <control>
'\x8f' # 0x8F -> <control>
'\x90' # 0x90 -> <control>
'\x91' # 0x91 -> <control>
'\x92' # 0x92 -> <control>
'\x93' # 0x93 -> <control>
'\x94' # 0x94 -> <control>
'\x95' # 0x95 -> <control>
'\x96' # 0x96 -> <control>
'\x97' # 0x97 -> <control>
'\x98' # 0x98 -> <control>
'\x99' # 0x99 -> <control>
'\x9a' # 0x9A -> <control>
'\x9b' # 0x9B -> <control>
'\x9c' # 0x9C -> <control>
'\x9d' # 0x9D -> <control>
'\x9e' # 0x9E -> <control>
'\x9f' # 0x9F -> <control>
'\xa0' # 0xA0 -> NO-BREAK SPACE
'\u06f0' # 0xA1 -> EXTENDED ARABIC-INDIC DIGIT ZERO
'\u06f1' # 0xA2 -> EXTENDED ARABIC-INDIC DIGIT ONE
'\u06f2' # 0xA3 -> EXTENDED ARABIC-INDIC DIGIT TWO
'\u06f3' # 0xA4 -> EXTENDED ARABIC-INDIC DIGIT THREE
'\u06f4' # 0xA5 -> EXTENDED ARABIC-INDIC DIGIT FOUR
'\u06f5' # 0xA6 -> EXTENDED ARABIC-INDIC DIGIT FIVE
'\u06f6' # 0xA7 -> EXTENDED ARABIC-INDIC DIGIT SIX
'\u06f7' # 0xA8 -> EXTENDED ARABIC-INDIC DIGIT SEVEN
'\u06f8' # 0xA9 -> EXTENDED ARABIC-INDIC DIGIT EIGHT
'\u06f9' # 0xAA -> EXTENDED ARABIC-INDIC DIGIT NINE
'\u060c' # 0xAB -> ARABIC COMMA
'\u061b' # 0xAC -> ARABIC SEMICOLON
'\xad' # 0xAD -> SOFT HYPHEN
'\u061f' # 0xAE -> ARABIC QUESTION MARK
'\ufe81' # 0xAF -> ARABIC LETTER ALEF WITH MADDA ABOVE ISOLATED FORM
'\ufe8d' # 0xB0 -> ARABIC LETTER ALEF ISOLATED FORM
'\ufe8e' # 0xB1 -> ARABIC LETTER ALEF FINAL FORM
'\ufe8e' # 0xB2 -> ARABIC LETTER ALEF FINAL FORM
'\ufe8f' # 0xB3 -> ARABIC LETTER BEH ISOLATED FORM
'\ufe91' # 0xB4 -> ARABIC LETTER BEH INITIAL FORM
'\ufb56' # 0xB5 -> ARABIC LETTER PEH ISOLATED FORM
'\ufb58' # 0xB6 -> ARABIC LETTER PEH INITIAL FORM
'\ufe93' # 0xB7 -> ARABIC LETTER TEH MARBUTA ISOLATED FORM
'\ufe95' # 0xB8 -> ARABIC LETTER TEH ISOLATED FORM
'\ufe97' # 0xB9 -> ARABIC LETTER TEH INITIAL FORM
'\ufb66' # 0xBA -> ARABIC LETTER TTEH ISOLATED FORM
'\ufb68' # 0xBB -> ARABIC LETTER TTEH INITIAL FORM
'\ufe99' # 0xBC -> ARABIC LETTER THEH ISOLATED FORM
'\ufe9b' # 0xBD -> ARABIC LETTER THEH INITIAL FORM
'\ufe9d' # 0xBE -> ARABIC LETTER JEEM ISOLATED FORM
'\ufe9f' # 0xBF -> ARABIC LETTER JEEM INITIAL FORM
'\ufb7a' # 0xC0 -> ARABIC LETTER TCHEH ISOLATED FORM
'\ufb7c' # 0xC1 -> ARABIC LETTER TCHEH INITIAL FORM
'\ufea1' # 0xC2 -> ARABIC LETTER HAH ISOLATED FORM
'\ufea3' # 0xC3 -> ARABIC LETTER HAH INITIAL FORM
'\ufea5' # 0xC4 -> ARABIC LETTER KHAH ISOLATED FORM
'\ufea7' # 0xC5 -> ARABIC LETTER KHAH INITIAL FORM
'\ufea9' # 0xC6 -> ARABIC LETTER DAL ISOLATED FORM
'\ufb84' # 0xC7 -> ARABIC LETTER DAHAL ISOLATED FORMN
'\ufeab' # 0xC8 -> ARABIC LETTER THAL ISOLATED FORM
'\ufead' # 0xC9 -> ARABIC LETTER REH ISOLATED FORM
'\ufb8c' # 0xCA -> ARABIC LETTER RREH ISOLATED FORM
'\ufeaf' # 0xCB -> ARABIC LETTER ZAIN ISOLATED FORM
'\ufb8a' # 0xCC -> ARABIC LETTER JEH ISOLATED FORM
'\ufeb1' # 0xCD -> ARABIC LETTER SEEN ISOLATED FORM
'\ufeb3' # 0xCE -> ARABIC LETTER SEEN INITIAL FORM
'\ufeb5' # 0xCF -> ARABIC LETTER SHEEN ISOLATED FORM
'\ufeb7' # 0xD0 -> ARABIC LETTER SHEEN INITIAL FORM
'\ufeb9' # 0xD1 -> ARABIC LETTER SAD ISOLATED FORM
'\ufebb' # 0xD2 -> ARABIC LETTER SAD INITIAL FORM
'\ufebd' # 0xD3 -> ARABIC LETTER DAD ISOLATED FORM
'\ufebf' # 0xD4 -> ARABIC LETTER DAD INITIAL FORM
'\ufec1' # 0xD5 -> ARABIC LETTER TAH ISOLATED FORM
'\ufec5' # 0xD6 -> ARABIC LETTER ZAH ISOLATED FORM
'\ufec9' # 0xD7 -> ARABIC LETTER AIN ISOLATED FORM
'\ufeca' # 0xD8 -> ARABIC LETTER AIN FINAL FORM
'\ufecb' # 0xD9 -> ARABIC LETTER AIN INITIAL FORM
'\ufecc' # 0xDA -> ARABIC LETTER AIN MEDIAL FORM
'\ufecd' # 0xDB -> ARABIC LETTER GHAIN ISOLATED FORM
'\ufece' # 0xDC -> ARABIC LETTER GHAIN FINAL FORM
'\ufecf' # 0xDD -> ARABIC LETTER GHAIN INITIAL FORM
'\ufed0' # 0xDE -> ARABIC LETTER GHAIN MEDIAL FORM
'\ufed1' # 0xDF -> ARABIC LETTER FEH ISOLATED FORM
'\ufed3' # 0xE0 -> ARABIC LETTER FEH INITIAL FORM
'\ufed5' # 0xE1 -> ARABIC LETTER QAF ISOLATED FORM
'\ufed7' # 0xE2 -> ARABIC LETTER QAF INITIAL FORM
'\ufed9' # 0xE3 -> ARABIC LETTER KAF ISOLATED FORM
'\ufedb' # 0xE4 -> ARABIC LETTER KAF INITIAL FORM
'\ufb92' # 0xE5 -> ARABIC LETTER GAF ISOLATED FORM
'\ufb94' # 0xE6 -> ARABIC LETTER GAF INITIAL FORM
'\ufedd' # 0xE7 -> ARABIC LETTER LAM ISOLATED FORM
'\ufedf' # 0xE8 -> ARABIC LETTER LAM INITIAL FORM
'\ufee0' # 0xE9 -> ARABIC LETTER LAM MEDIAL FORM
'\ufee1' # 0xEA -> ARABIC LETTER MEEM ISOLATED FORM
'\ufee3' # 0xEB -> ARABIC LETTER MEEM INITIAL FORM
'\ufb9e' # 0xEC -> ARABIC LETTER NOON GHUNNA ISOLATED FORM
'\ufee5' # 0xED -> ARABIC LETTER NOON ISOLATED FORM
'\ufee7' # 0xEE -> ARABIC LETTER NOON INITIAL FORM
'\ufe85' # 0xEF -> ARABIC LETTER WAW WITH HAMZA ABOVE ISOLATED FORM
'\ufeed' # 0xF0 -> ARABIC LETTER WAW ISOLATED FORM
'\ufba6' # 0xF1 -> ARABIC LETTER HEH GOAL ISOLATED FORM
'\ufba8' # 0xF2 -> ARABIC LETTER HEH GOAL INITIAL FORM
'\ufba9' # 0xF3 -> ARABIC LETTER HEH GOAL MEDIAL FORM
'\ufbaa' # 0xF4 -> ARABIC LETTER HEH DOACHASHMEE ISOLATED FORM
'\ufe80' # 0xF5 -> ARABIC LETTER HAMZA ISOLATED FORM
'\ufe89' # 0xF6 -> ARABIC LETTER YEH WITH HAMZA ABOVE ISOLATED FORM
'\ufe8a' # 0xF7 -> ARABIC LETTER YEH WITH HAMZA ABOVE FINAL FORM
'\ufe8b' # 0xF8 -> ARABIC LETTER YEH WITH HAMZA ABOVE INITIAL FORM
'\ufef1' # 0xF9 -> ARABIC LETTER YEH ISOLATED FORM
'\ufef2' # 0xFA -> ARABIC LETTER YEH FINAL FORM
'\ufef3' # 0xFB -> ARABIC LETTER YEH INITIAL FORM
'\ufbb0' # 0xFC -> ARABIC LETTER YEH BARREE WITH HAMZA ABOVE ISOLATED FORM
'\ufbae' # 0xFD -> ARABIC LETTER YEH BARREE ISOLATED FORM
'\ufe7c' # 0xFE -> ARABIC SHADDA ISOLATED FORM
'\ufe7d' # 0xFF -> ARABIC SHADDA MEDIAL FORM
)