python - Parsing the string-representation of a numpy array

Question

If I only have the string-representation of a numpy.array:

>>> import numpy as np
>>> arr = np.random.randint(0, 10, (10, 10))
>>> print(arr)  # this one!
[[9 4 7 3]
 [1 6 4 2]
 [6 7 6 0]
 [0 5 6 7]]

How can I convert this back to a numpy array? It's not complicated to actually insert the , manually but I'm looking for a programmatic approach.

A simple regex replacing whitespaces with , actually works for single-digit integers:

>>> import re
>>> sub = re.sub('s+', ',', """[[8 6 2 4 0 2]
...  [3 5 8 4 5 6]
...  [4 6 3 3 0 3]]
... """)
>>> sub
'[[8,6,2,4,0,2],[3,5,8,4,5,6],[4,6,3,3,0,3]],'  # the trailing "," is a bit annoying

It can be converted to an almost (dtype may be lost but that's okay) identical array:

>>> import ast
>>> np.array(ast.literal_eval(sub)[0])
array([[8, 6, 2, 4, 0, 2],
       [3, 5, 8, 4, 5, 6],
       [4, 6, 3, 3, 0, 3]])

But it fails for multidigit integers and floats:

>>> re.sub('s+', ',', """[[ 0.  1.  6.  9.  1.  4.]
... [ 4.  8.  2.  3.  6.  1.]]
... """)
'[[,0.,1.,6.,9.,1.,4.],[,4.,8.,2.,3.,6.,1.]],'

because these have an additional , at the beginning.

A solution doesn't necessarily need to be based on regex, any other approach that works for unabriged (not shortened with ...) bool/int/float/complex arrays with 1-4 dimensions would be ok.

Answer 1

Here's a pretty manual solution:

import re
import numpy

def parse_array_str(array_string):
    tokens = re.findall(r'''             # Find all...
                            [         | # opening brackets,
                            ]         | # closing brackets, or
                            [^[]s]+   # sequences of other non-whitespace characters''',
                        array_string,
                        flags = re.VERBOSE)
    tokens = iter(tokens)

    # Chomp first [, handle case where it's not a [
    first_token = next(tokens)
    if first_token != '[':
        # Input must represent a scalar
        if next(tokens, None) is not None:
            raise ValueError("Can't parse input.")
        return float(first_token)  # or int(token), but not bool(token) for bools

    list_form = []
    stack = [list_form]

    for token in tokens:
        if token == '[':
            # enter a new list
            stack.append([])
            stack[-2].append(stack[-1])
        elif token == ']':
            # close a list
            stack.pop()
        else:
            stack[-1].append(float(token))  # or int(token), but not bool(token) for bools

    if stack:
        raise ValueError("Can't parse input - it might be missing text at the end.")

    return numpy.array(list_form)

Or a less manual solution, based on detecting where to insert commas:

import re
import numpy

pattern = r'''# Match (mandatory) whitespace between...
              (?<=]) # ] and
              s+
              (?= [) # [, or
              |
              (?<=[^[]s]) 
              s+
              (?= [^[]s]) # two non-bracket non-whitespace characters
           '''

# Replace such whitespace with a comma
fixed_string = re.sub(pattern, ',', array_string, flags=re.VERBOSE)

output_array = numpy.array(ast.literal_eval(fixed_string))