Problem:

I have a bunch python build scripts, all named bob.py. They are scattered all around a mono repo. I want to be able to import from one another and be treated as bobs package/namespace.

For example, I have the following 2 files:

 projects/a/bob.py
 projects/b/bob.py

From a/bob.py I want to be able to import b/bob.py like so:

# a/bob.py

import bobs.b.bob as b

# use b
...

Solution: 

# loader.py

def add_bob_scripts_to_python_path(search_path):

    root_dir = find_root_dir()
    bobs_dir = create_bobs_dir(root_dir)
    bob_files = find_bob_files(search_path)

    for bob in bob_files:
      create_symbolic_link(search_path, bobs_dir, bob)

    add_to_sys_path(root_dir)

def find_root_dir():
    path = '/tmp/_bobs'
    pathlib.Path(path).mkdir(exist_ok=True)
    return path

def create_bobs_dir(root):
    bobs_dir = os.path.join(root, 'bobs')
    pathlib.Path(bobs_dir).mkdir(exist_ok=True)
    return bobs_dir

def find_bob_files(search_path, name='bob.py'):
    bobs = []
    for path, dirs, files in os.walk(search_path, topdown=True):
        dirs[:] = [d for d in dirs]
        if name in files:
            bobs.append(os.path.join(path, name))

    return bobs

def create_symbolic_link(search_path, bobs_dir, bob_file):
    relative_path = os.path.relpath(bob_file, search_path)

    target = bob_file
    link_name = os.path.join(bobs_dir, relative_path)
    pathlib.Path(link_name).parent.mkdir(parents=True, exist_ok=True)

    if not os.path.exists(link_name):
        os.symlink(bob_file, link_name)


def add_to_sys_path(root_dir):
    sys.path.append(root_dir)

import os
import pathlib
import sys

Testing

Here is an end-to-end test that creates a/bob.py and b/bob.py and test the loader.py file.

# loader_e2e.py
def run_e2e(test_dir):
    script_A_path = os.path.join(test_dir, 'a/bob.py')
    script_B_path = os.path.join(test_dir, 'b/bob.py')

    create_python_file(path=script_A_path, content=a_bob())
    create_python_file(path=script_B_path, content=b_bob())

    result = execute_script(script_A_path)
    assert result == 'hello from B\n'
    print('success!')


def create_python_file(path, content):
    pathlib.Path(path).parent.mkdir(exist_ok=True)
    with open(path, 'w', encoding='utf8') as w:
        w.write(content)

def execute_script(path):
    return subprocess.check_output(['python3', path]).decode('utf8')


def a_bob():
    return f'''
import sys
sys.path.append('{os.getcwd()}')

import loader
import pathlib
import os
loader.add_bob_scripts_to_python_path(pathlib.Path(__file__).parent.parent)

import bobs.b.bob as b

print(b.hello())
'''

def b_bob():
    return '''
def hello():
    return 'hello from B'
'''

import os
import subprocess
import tempfile
import pathlib
import shutil

if __name__ == '__main__':
    try:
        with tempfile.TemporaryDirectory() as temp_dir:
            pathlib.Path(temp_dir).mkdir(exist_ok=True)
            run_e2e(temp_dir)

    finally:
        shutil.rmtree('/tmp/_bobs')

and to run it, place the two files in a directory and run:

$ python3 loader_e2e.py