Optimizing Python code

With Cython extensions

Andrew Svetlov

http://asvetlov.blogspot.com
andrew.svetlov@gmail.com
http://asvetlov.github.io/optimization-kaunas-2017/

Prerequisites

• There is Python code
• Profiler points on bottleneck
• The code is algorithmically perfect
  O(N) vs O(log(N))

1. Naive Cythonizing

def py(a, b):
    return a + b

import timeit
print(timeit.timeit('py(1, 2)', 'from __main__ import py'))
          

python

0.070 sec

def cy(a, b):
    return a + b
          

mod1.pyx

import pyximport; pyximport.install()
from mod1 import cy

print(timeit.timeit('cy(1, 2)', 'from __main__ import cy'))
          

cython without types

0.046 sec

was 0.070 sec

def cy2(int a, int b):
    return a + b
          

mod1.pyx with types

0.046 sec

was 0.046 and 0.070 sec

2. Better example

def pysum(start, step, count):
    ret = start
    for i in range(count):
        ret += step
    return ret


print(timeit.timeit('pysum(0, 1, 100)', 'from __main__ import pysum'))
          

python

2.772 sec

def cysum(start, step, count):
    ret = start
    for i in range(count):
        ret += step
    return ret
          

mod2.pyx

cython without types

1.177 sec

was 2.772

def cysum2(float start, float step, int count):
    cdef float ret
    ret = start
    for i in range(count):
        ret += step
    return ret
          

mod2.pyx

cython with types

0.222 sec

was 1.177 sec and 2.772

$ cython -a mod2.pyx
$ xdg-open mod2.html
          

Data structures

def pyappend(lst, item, count):
    for i in range(count):
        lst.append(item)


print(timeit.timeit('pyappend([], 1, 100)', 'from __main__ import pyappend'))
          

python

6.602 sec

def cyappend(lst, item, count):
    for i in range(count):
        lst.append(item)
          

mod3.pyx

cython without types

1.388 sec

was 6.602

def cyappend2(list lst, item, int count):
    for i in range(count):
        lst.append(item)
          

mod3.pyx

cython with types

0.799 sec

was 1.388 sec and 6.602

Abstract vs Concrete types

• PyObject_SetAttr
• PySequence_GetItem
• ... vs ...
• PyDict_GetItem
• PyList_GetItem
• PyTuple_GetItem

$ cython -a mod3.pyx
$ xdg-open mod3.html
          

Compiling

Project structure

.
├── prj
│   ├── __init__.py
│   ├── _mod.pyx
├── setup.py
└── tests
    └── test_prj.py
          

def _append(list lst, item, int count):
    for i in range(count):
        lst.append(item)
          

_mod.pyx

try:
    from ._mod import _append
except ImportError:
    _append = None

def _pyappend(lst, item, count):
    for i in range(count):
        lst.append(item)


if _append is not None:
    append = _append
else:
    append = _pyappend
          

__init__.py

from Cython.Build import cythonize
from setuptools import setup

setup(name='prj',
      version='0.0.1',
      packages=['prj'],
      ext_modules=cythonize('prj/_mod.pyx'))
          

setup.py

$ pip install -e .
Obtaining file:///home/andrew/projects/optimization-vilnius-2017/prj
Installing collected packages: prj
  Running setup.py develop for prj
Successfully installed prj
          

from unittest import TestCase
from prj import _pyappend
try:
    from prj import _append
except ImportError:
    _append = None

class BasetTestMixin:
    def func(self, *args):
        raise NotImplementedError

    def test_append(self):
        lst = [1, 2]
        self.func(lst, 3, 2)
        self.assertEqual([1, 2, 3, 3], lst)
          

tests/test_prj.py

class TestPython(TestCase, BasetTestMixin):
    def func(self, *args):
        _pyappend(*args)

if _append is not None:
    class TestCython(TestCase, BasetTestMixin):
        def func(self, *args):
            _append(*args)
          

tests/test_prj.py

# python -m unittest discover tests
..
----------------------------------------------------------------------
Ran 2 tests in 0.000s

OK
          

Questions?

Andrew Svetlov

http://asvetlov.blogspot.com
andrew.svetlov@gmail.com
http://asvetlov.github.io/optimization-kaunas-2017/