Index

A

allreduce() (parallelformers.utils.dist_utils.ParallelModule method) allreduce_linear_layer() (parallelformers.utils.dist_utils.AllReduceQuantLinear method) AllReduceConv1D (class in parallelformers.utils.dist_utils) AllReduceLinear (class in parallelformers.utils.dist_utils) AllReduceQuantLinear (class in parallelformers.utils.dist_utils)

attn_out() (parallelformers.policies.base.policy.Policy static method) attn_qkv() (parallelformers.policies.base.policy.Policy static method) auto_policy() (parallelformers.parallel.replacing.TensorReplacer method) AutoPolicy (class in parallelformers.policies.base.auto) available() (parallelformers.policies.base.auto.AutoPolicy static method)

B

bias (parallelformers.policies.base.policy.Layer attribute)

C

check_picklable() (parallelformers.parallel.process.ParallelProcess method)

column_slice() (parallelformers.parallel.slicing.TensorSlicer method) create_process_group() (parallelformers.parallel.engine.ParallelEngine method)

D

deparallelize() (parallelformers.parallelize.parallelize method)

destroy() (parallelformers.parallel.process.ParallelProcess method) dumps() (parallelformers.parallel.process.ForkingPickler class method)

F

ForkingPickler (class in parallelformers.parallel.process) forward() (parallelformers.utils.dist_utils.AllReduceConv1D method) (parallelformers.utils.dist_utils.AllReduceLinear method) (parallelformers.utils.dist_utils.AllReduceQuantLinear method)

G

get_policy() (parallelformers.policies.base.auto.AutoPolicy method)

H

hijack() (parallelformers.parallelize.parallelize method)

I

igetattr() (in module parallelformers.utils.attr_utils) ignore_checker (parallelformers.policies.base.policy.Layer attribute), [1] in_features (parallelformers.utils.dist_utils.AllReduceLinear attribute)

inference() (parallelformers.parallel.process.ParallelProcess method) init_environments() (parallelformers.parallelize.parallelize method) isetattr() (in module parallelformers.utils.attr_utils)

L

Layer (class in parallelformers.policies.base.policy)

loads() (parallelformers.parallel.process.ForkingPickler method)

M

make_megatron_layer() (parallelformers.parallel.replacing.TensorReplacer method) mlp_in() (parallelformers.policies.base.policy.Policy static method) mlp_out() (parallelformers.policies.base.policy.Policy static method) module parallelformers.parallel.engine parallelformers.parallel.process parallelformers.parallel.replacing parallelformers.parallel.slicing parallelformers.parallelize parallelformers.policies.base.auto parallelformers.policies.base.policy parallelformers.utils.attr_utils parallelformers.utils.dist_utils

N

n_fused (parallelformers.policies.base.policy.Layer attribute), [1]

O

original_layer_class() (parallelformers.policies.base.policy.Policy static method)

out_features (parallelformers.utils.dist_utils.AllReduceLinear attribute)

P

ParallelEngine (class in parallelformers.parallel.engine) parallelformers.parallel.engine module parallelformers.parallel.process module parallelformers.parallel.replacing module parallelformers.parallel.slicing module parallelformers.parallelize module parallelformers.policies.base.auto module

parallelformers.policies.base.policy module parallelformers.utils.attr_utils module parallelformers.utils.dist_utils module parallelize (class in parallelformers.parallelize) parallelize() (parallelformers.parallel.engine.ParallelEngine method) (parallelformers.parallelize.parallelize method) ParallelModule (class in parallelformers.utils.dist_utils) ParallelProcess (class in parallelformers.parallel.process) Policy (class in parallelformers.policies.base.policy) preprocess() (parallelformers.parallel.replacing.TensorReplacer method) preprocess_for_wav2vec() (parallelformers.parallelize.parallelize method)

R

register() (parallelformers.parallel.process.ForkingPickler class method) register_hijack_methods() (parallelformers.parallelize.parallelize method) register_memory_methods() (parallelformers.parallelize.parallelize method) replace (parallelformers.policies.base.policy.Layer attribute), [1] replace_arguments() (parallelformers.policies.base.policy.Policy static method) replace_modules() (parallelformers.parallel.replacing.TensorReplacer method) (parallelformers.policies.base.policy.Policy static method)

replace_orig_to_megatron_modules() (parallelformers.parallel.replacing.TensorReplacer method) replace_user_define_modules() (parallelformers.parallel.replacing.TensorReplacer method) reversed (parallelformers.policies.base.policy.Layer attribute), [1] rgetattr() (in module parallelformers.utils.attr_utils) rhasattr() (in module parallelformers.utils.attr_utils) row_slice() (parallelformers.parallel.slicing.TensorSlicer method) rsetattr() (in module parallelformers.utils.attr_utils) run() (parallelformers.parallel.process.ParallelProcess method)

S

set_environ() (parallelformers.parallel.process.ParallelProcess method) set_layer_size() (parallelformers.parallel.replacing.TensorReplacer static method)

set_parameters() (parallelformers.parallel.replacing.TensorReplacer method) slice_tensor() (parallelformers.parallel.slicing.TensorSlicer method) slice_weight_and_bias() (parallelformers.parallel.slicing.TensorSlicer method)

T

TensorReplacer (class in parallelformers.parallel.replacing) TensorSlicer (class in parallelformers.parallel.slicing)

training (parallelformers.utils.dist_utils.AllReduceConv1D attribute) (parallelformers.utils.dist_utils.AllReduceQuantLinear attribute) (parallelformers.utils.dist_utils.ParallelModule attribute)

W

weight (parallelformers.policies.base.policy.Layer attribute) (parallelformers.utils.dist_utils.AllReduceLinear attribute)