Skip to content

Commit 2898ab2

Browse files
authored
Merge pull request #1022 from jmitrevs/sep_to_dw_point
Add an optimizer to replace SeparableConv by Depthwise + Conv (pointwise)
2 parents 5bcba68 + 9ab6a2e commit 2898ab2

15 files changed

+338
-46
lines changed

hls4ml/backends/fpga/fpga_backend.py

+10-2
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,6 @@ def __init__(self, name):
5555
Dense,
5656
Conv1D,
5757
Conv2D,
58-
SeparableConv1D,
59-
SeparableConv2D,
6058
Pooling1D,
6159
Pooling2D,
6260
GlobalPooling1D,
@@ -79,6 +77,16 @@ def __init__(self, name):
7977
attrs.append(ConfigurableAttribute('reuse_factor', default=1))
8078
self.attribute_map[layer] = attrs
8179

80+
# seperable is kind of special because it is effectively two layers that will be split
81+
for layer in (SeparableConv1D, SeparableConv2D):
82+
attrs = self.attribute_map.get(layer, [])
83+
attrs.append(TypeAttribute('depthwise_accum'))
84+
attrs.append(TypeAttribute('pointwise_accum'))
85+
attrs.append(TypeAttribute('depthwise_result'))
86+
attrs.append(ConfigurableAttribute('depthwise_reuse_factor', default=1))
87+
attrs.append(ConfigurableAttribute('pointwise_reuse_factor', default=1))
88+
self.attribute_map[layer] = attrs
89+
8290
act_attrs = self.attribute_map.get(Activation, [])
8391
act_attrs.append(ConfigurableAttribute('table_size', default=1024))
8492
act_attrs.append(TypeAttribute('table', default=FixedPrecisionType(18, 8)))

hls4ml/backends/fpga/passes/codegen.py

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
class GenerateConvIm2col(OptimizerPass):
77
'''Generates tcode for im2col step of 1D/2d convolution'''
88

9+
# Note, DepthwizeConv1D/2D also matches because it inherits from Conv1D/2D
910
def match(self, node):
1011
return (
1112
isinstance(node, (Conv1D, Conv2D, SeparableConv1D, SeparableConv2D))

hls4ml/backends/vivado/passes/convolution_templates.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ def format(self, node):
280280
# Override bias and bias_t since these are zeros in depthwise step of SepConv1D
281281
params['bias'] = params['zero_bias']
282282
params['bias_t'] = params['zero_bias_t']
283-
params['n_filt'] = params['n_chan'] # In depthwise step n_chan == n_filt
283+
params['n_filt'] = params['n_chan'] * node.get_attr('depth_multiplier') # In depthwise step n_chan == n_filt
284284
params['dilation'] = node.get_attr('dilation', 1)
285285
params['nzeros'] = node.get_weights('depthwise').nzeros
286286
params['index'] = str(node.index) + '_depthwise'

hls4ml/backends/vivado/vivado_backend.py

+26-6
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
Conv1D,
1616
Conv2D,
1717
Dense,
18+
DepthwiseConv1D,
1819
DepthwiseConv2D,
1920
Embedding,
2021
GarNet,
@@ -74,12 +75,6 @@ def _register_layer_attributes(self):
7475
attrs.append(ChoiceAttribute('conv_implementation', choices=['LineBuffer', 'Encoded'], default='LineBuffer'))
7576
self.attribute_map[layer] = attrs
7677

77-
sep_conv_layers = [SeparableConv1D, SeparableConv2D]
78-
for layer in sep_conv_layers:
79-
attrs = self.attribute_map.get(layer, [])
80-
attrs.append(TypeAttribute('dw_output', default=FixedPrecisionType(18, 8)))
81-
self.attribute_map[layer] = attrs
82-
8378
def _register_flows(self):
8479
initializers = self._get_layer_initializers()
8580
init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
@@ -359,6 +354,31 @@ def init_sepconv1d(self, layer):
359354
dw_output_t = NamedType(dw_out_name, dw_out_precision)
360355
layer.set_attr('dw_output_t', dw_output_t)
361356

357+
@layer_optimizer(DepthwiseConv1D)
358+
def init_depconv1d(self, layer):
359+
if layer.model.config.is_resource_strategy(layer):
360+
layer.set_attr('strategy', 'resource')
361+
n_in, n_out = self.get_layer_mult_size(layer)
362+
self.set_closest_reuse_factor(layer, n_in, n_out)
363+
else:
364+
layer.set_attr('strategy', 'latency')
365+
366+
out_width = layer.get_output_variable().shape[0]
367+
chosen_pf = layer.model.config.get_layer_config_value(layer, 'ParallelizationFactor', 1)
368+
valid_pf = self.get_valid_conv_partition_splits(1, out_width)
369+
if chosen_pf not in valid_pf:
370+
closest_pf = self.get_closest_reuse_factor(valid_pf, chosen_pf)
371+
valid_pf_str = ','.join(map(str, valid_pf))
372+
print(
373+
f'WARNING: Invalid ParallelizationFactor={chosen_pf} in layer "{layer.name}".'
374+
f'Using ParallelizationFactor={closest_pf} instead. Valid ParallelizationFactor(s): {valid_pf_str}.'
375+
)
376+
else:
377+
closest_pf = chosen_pf
378+
layer.set_attr('n_partitions', out_width // closest_pf)
379+
380+
layer.set_attr('implementation', layer.model.config.get_conv_implementation(layer).lower())
381+
362382
@layer_optimizer(Conv2D)
363383
def init_conv2d(self, layer):
364384
if len(layer.weights['weight'].data.shape) == 2: # This can happen if we assign weights of Dense layer to 1x1 Conv2D

hls4ml/converters/keras/convolution.py

+8-2
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,13 @@ def parse_conv1d_layer(keras_layer, input_names, input_shapes, data_reader):
2121

2222
layer['bias_data'] = get_weights_data(data_reader, layer['name'], 'bias')
2323

24+
if 'depth_multiplier' in keras_layer['config']:
25+
layer['depth_multiplier'] = keras_layer['config']['depth_multiplier']
26+
2427
if 'filters' in keras_layer['config']:
2528
layer['n_filt'] = keras_layer['config']['filters']
2629
else:
27-
layer['n_filt'] = layer['n_chan']
30+
layer['n_filt'] = layer['n_chan'] * layer.get('depth_multiplier')
2831
layer['filt_width'] = keras_layer['config']['kernel_size'][0]
2932
layer['stride_width'] = keras_layer['config']['strides'][0]
3033
layer['padding'] = keras_layer['config']['padding']
@@ -60,10 +63,13 @@ def parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader):
6063

6164
layer['bias_data'] = get_weights_data(data_reader, layer['name'], 'bias')
6265

66+
if 'depth_multiplier' in keras_layer['config']:
67+
layer['depth_multiplier'] = keras_layer['config']['depth_multiplier']
68+
6369
if 'filters' in keras_layer['config']:
6470
layer['n_filt'] = keras_layer['config']['filters']
6571
else:
66-
layer['n_filt'] = layer['n_chan']
72+
layer['n_filt'] = layer['n_chan'] * layer.get('depth_multiplier')
6773
layer['filt_height'] = keras_layer['config']['kernel_size'][0]
6874
layer['filt_width'] = keras_layer['config']['kernel_size'][1]
6975
layer['stride_height'] = keras_layer['config']['strides'][0]

hls4ml/model/graph.py

+74-26
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,12 @@ def get_layer_config(self, layer):
109109

110110
return layer_config
111111

112+
def set_name_config(self, name, config):
113+
"""sets hls_config["LayerName"][name] = config"""
114+
hls_config = self.config['HLSConfig']
115+
layer_config = hls_config.setdefault('LayerName', {})
116+
layer_config[name] = config
117+
112118
def get_precision(self, layer, var='default'):
113119
precision = self.layer_name_precision.get(layer.name.lower() + '_' + var)
114120
type_name = layer.name.lower() + '_' + var + '_t'
@@ -192,6 +198,35 @@ def get_compression(self, layer):
192198

193199
return compression
194200

201+
def parse_name_config(self, layer_name, layer_cfg):
202+
"""This is used by _parse_hls_config below, but also in optimizers when a new layer config is created"""
203+
precision_cfg = layer_cfg.get('Precision')
204+
if isinstance(precision_cfg, dict):
205+
for var, precision in precision_cfg.items():
206+
self.layer_name_precision[layer_name.lower() + '_' + var] = precision
207+
else:
208+
self.layer_name_precision[layer_name.lower() + '_default'] = precision_cfg
209+
210+
rf = layer_cfg.get('ReuseFactor')
211+
if rf is not None:
212+
self.layer_name_rf[layer_name.lower()] = rf
213+
214+
targ_cycles = layer_cfg.get('TargetCycles')
215+
if targ_cycles is not None:
216+
self.layer_name_targ_cycles[layer_name.lower()] = targ_cycles
217+
218+
strategy = layer_cfg.get('Strategy')
219+
if strategy is not None:
220+
self.layer_name_strategy[layer_name.lower()] = strategy
221+
222+
conv_implementation = layer_cfg.get('ConvImplementation')
223+
if conv_implementation is not None:
224+
self.layer_name_conv_implementation[layer_name.lower()] = conv_implementation
225+
226+
compression = layer_cfg.get('Compression')
227+
if compression is not None:
228+
self.layer_name_compression[layer_name.lower()] = bool(compression)
229+
195230
def get_writer_config(self):
196231
return self.writer_config
197232

@@ -267,32 +302,7 @@ def _parse_hls_config(self):
267302
layer_name_cfg = hls_config.get('LayerName')
268303
if layer_name_cfg is not None:
269304
for layer_name, layer_cfg in layer_name_cfg.items():
270-
precision_cfg = layer_cfg.get('Precision')
271-
if isinstance(precision_cfg, dict):
272-
for var, precision in precision_cfg.items():
273-
self.layer_name_precision[layer_name.lower() + '_' + var] = precision
274-
else:
275-
self.layer_name_precision[layer_name.lower() + '_default'] = precision_cfg
276-
277-
rf = layer_cfg.get('ReuseFactor')
278-
if rf is not None:
279-
self.layer_name_rf[layer_name.lower()] = rf
280-
281-
targ_cycles = layer_cfg.get('TargetCycles')
282-
if targ_cycles is not None:
283-
self.layer_name_targ_cycles[layer_name.lower()] = targ_cycles
284-
285-
strategy = layer_cfg.get('Strategy')
286-
if strategy is not None:
287-
self.layer_name_strategy[layer_name.lower()] = strategy
288-
289-
conv_implementation = layer_cfg.get('ConvImplementation')
290-
if conv_implementation is not None:
291-
self.layer_name_conv_implementation[layer_name.lower()] = conv_implementation
292-
293-
compression = layer_cfg.get('Compression')
294-
if compression is not None:
295-
self.layer_name_compression[layer_name.lower()] = bool(compression)
305+
self.parse_name_config(layer_name, layer_cfg)
296306

297307
def _validate_hls_config(self):
298308
use_dataflow = False
@@ -617,6 +627,44 @@ def replace_node(self, old_node, new_node):
617627
self.graph = OrderedDict((new_node.name, new_node) if k == old_node.name else (k, v) for k, v in self.graph.items())
618628
self._update_model_outputs()
619629

630+
def split_node(self, old_node, new_node1, new_node2):
631+
"""Replace an existing node in the graph with two nodes in sequence.
632+
633+
Args:
634+
old_node (Layer): The node to replace
635+
new_node1 (Layer): The first new node in sequence
636+
new_node2 (Layer): The second new node in sequence
637+
638+
"""
639+
640+
# fmt: off
641+
assert len(new_node1.inputs) == len(old_node.inputs), \
642+
f'{new_node1.name} and {old_node.name} have different number of inputs'
643+
assert len(new_node2.outputs) == len(old_node.outputs), \
644+
f'{new_node2.name} and {old_node.name} have different number of outputs'
645+
# fmt: on
646+
647+
repl = {old_name: new_name for old_name, new_name in zip(old_node.outputs, new_node2.outputs)}
648+
repl.update({old_name: new_name for old_name, new_name in zip(old_node.inputs, new_node1.inputs)})
649+
650+
for node in self.graph.values():
651+
for i, n in enumerate(node.inputs):
652+
if n in repl:
653+
node.inputs[i] = repl[n]
654+
for i, n in enumerate(node.outputs):
655+
if n in repl:
656+
node.outputs[i] = repl[n]
657+
658+
new_graph = OrderedDict()
659+
for key, value in self.graph.items():
660+
if key == old_node.name:
661+
new_graph[new_node1.name] = new_node1
662+
new_graph[new_node2.name] = new_node2
663+
else:
664+
new_graph[key] = value
665+
self.graph = new_graph
666+
self._update_model_outputs()
667+
620668
def _update_model_outputs(self):
621669
'''Update the model outputs
622670

hls4ml/model/layers.py

+67-4
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,12 @@ def _set_accum_t(self):
179179
accum_t = NamedType(*reversed(self.model.config.get_precision(self, 'accum')))
180180
self.set_attr('accum_t', accum_t)
181181

182+
def _set_type_t(self, name):
183+
has_type_t = any(a for a in self.expected_attributes if a.name == name + '_t' and isinstance(a, TypeAttribute))
184+
if has_type_t:
185+
type_t = NamedType(*reversed(self.model.config.get_precision(self, name)))
186+
self.set_attr(name + '_t', type_t)
187+
182188
def get_input_node(self, input_name=None):
183189
if input_name is None:
184190
if len(self.inputs) > 0:
@@ -446,6 +452,7 @@ class SeparableConv1D(Layer):
446452
Attribute('out_width'),
447453
Attribute('n_chan'),
448454
Attribute('n_filt'),
455+
Attribute('depth_multiplier', default=1),
449456
Attribute('filt_width'),
450457
Attribute('stride_width'),
451458
Attribute('pad_left'),
@@ -476,14 +483,35 @@ def initialize(self):
476483

477484
self.add_bias(quantizer=self.get_attr('bias_quantizer'))
478485

486+
# set the needed types if needed
487+
self._set_type_t('pointwise_accum')
488+
self._set_type_t('depthwise_accum')
489+
self._set_type_t('depthwise_result')
490+
479491

480492
class DepthwiseConv1D(Conv1D):
493+
_expected_attributes = [
494+
Attribute('in_width'),
495+
Attribute('out_width'),
496+
Attribute('n_chan'),
497+
Attribute('depth_multiplier', default=1),
498+
Attribute('n_filt'), # = n_chan * depth_multiplier
499+
Attribute('filt_width'),
500+
Attribute('stride_width'),
501+
Attribute('pad_left'),
502+
Attribute('pad_right'),
503+
WeightAttribute('weight'),
504+
WeightAttribute('bias'),
505+
TypeAttribute('weight'),
506+
TypeAttribute('bias'),
507+
]
508+
481509
def initialize(self):
482510
if self.get_attr('data_format') == 'channels_last':
483-
shape = [self.attributes['out_width'], self.attributes['n_chan']]
511+
shape = [self.attributes['out_width'], self.attributes['n_filt']]
484512
dims = [f'OUT_HEIGHT_{self.index}', f'N_CHAN_{self.index}']
485513
else:
486-
shape = [self.attributes['n_chan'], self.attributes['out_width']]
514+
shape = [self.attributes['n_filt'], self.attributes['out_width']]
487515
dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}']
488516
self.add_output_variable(shape, dims)
489517

@@ -588,6 +616,7 @@ class SeparableConv2D(Layer):
588616
Attribute('out_width'),
589617
Attribute('n_chan'),
590618
Attribute('n_filt'),
619+
Attribute('depth_multiplier', default=1),
591620
Attribute('filt_height'),
592621
Attribute('filt_width'),
593622
Attribute('stride_height'),
@@ -622,14 +651,48 @@ def initialize(self):
622651

623652
self.add_bias(quantizer=self.get_attr('bias_quantizer'))
624653

654+
self._set_type_t('pointwise_accum')
655+
self._set_type_t('depthwise_accum')
656+
self._set_type_t('depthwise_result')
657+
625658

626659
class DepthwiseConv2D(Conv2D):
660+
_expected_attributes = [
661+
Attribute('in_height'),
662+
Attribute('in_width'),
663+
Attribute('out_height'),
664+
Attribute('out_width'),
665+
Attribute('n_chan'),
666+
Attribute('depth_multiplier', default=1),
667+
Attribute('n_filt'), # = n_chan * depth_multiplier
668+
Attribute('filt_height'),
669+
Attribute('filt_width'),
670+
Attribute('stride_height'),
671+
Attribute('stride_width'),
672+
Attribute('pad_top'),
673+
Attribute('pad_bottom'),
674+
Attribute('pad_left'),
675+
Attribute('pad_right'),
676+
WeightAttribute('weight'),
677+
WeightAttribute('bias'),
678+
TypeAttribute('weight'),
679+
TypeAttribute('bias'),
680+
]
681+
627682
def initialize(self):
628683
if self.get_attr('data_format') == 'channels_last':
629-
shape = [self.attributes['out_height'], self.attributes['out_width'], self.attributes['n_chan']]
684+
shape = [
685+
self.attributes['out_height'],
686+
self.attributes['out_width'],
687+
self.attributes['n_filt'],
688+
]
630689
dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}']
631690
else:
632-
shape = [self.attributes['n_chan'], self.attributes['out_height'], self.attributes['out_width']]
691+
shape = [
692+
self.attributes['n_filt'],
693+
self.attributes['out_height'],
694+
self.attributes['out_width'],
695+
]
633696
dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}']
634697
self.add_output_variable(shape, dims)
635698

hls4ml/model/optimizer/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
register_flow(
3434
'convert',
3535
[
36+
'seperable_to_depthwise_and_conv', # has to be before precision inference
3637
'infer_precision_types',
3738
'channels_last_converter',
3839
'remove_transpose_before_flatten',

0 commit comments

Comments
 (0)