Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[RFC] [PoC] btf: LoadSpecFromReader: add filter param to optimize memory #1589

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

[RFC] [PoC] btf: LoadSpecFromReader: add filter param to optimize memory #1589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 17 additions & 13 deletions btf/btf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,24 +187,24 @@ func LoadSpec(file string) (*Spec, error) {
}
defer fh.Close()

return LoadSpecFromReader(fh)
return LoadSpecFromReader(fh, nil)
}

// LoadSpecFromReader reads from an ELF or a raw BTF blob.
//
// Returns ErrNotFound if reading from an ELF which contains no BTF. ExtInfos
// may be nil.
func LoadSpecFromReader(rd io.ReaderAt) (*Spec, error) {
func LoadSpecFromReader(rd io.ReaderAt, filter *TypeFilter) (*Spec, error) {
file, err := internal.NewSafeELFFile(rd)
if err != nil {
if bo := guessRawBTFByteOrder(rd); bo != nil {
return loadRawSpec(io.NewSectionReader(rd, 0, math.MaxInt64), bo, nil)
return loadRawSpec(io.NewSectionReader(rd, 0, math.MaxInt64), bo, nil, nil)
}

return nil, err
}

return loadSpecFromELF(file)
return loadSpecFromELF(file, filter)
}

// LoadSpecAndExtInfosFromReader reads from an ELF.
Expand All @@ -217,7 +217,7 @@ func LoadSpecAndExtInfosFromReader(rd io.ReaderAt) (*Spec, *ExtInfos, error) {
return nil, nil, err
}

spec, err := loadSpecFromELF(file)
spec, err := loadSpecFromELF(file, nil)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -266,7 +266,7 @@ func symbolOffsets(file *internal.SafeELFFile) (map[symbol]uint32, error) {
return offsets, nil
}

func loadSpecFromELF(file *internal.SafeELFFile) (*Spec, error) {
func loadSpecFromELF(file *internal.SafeELFFile, filter *TypeFilter) (*Spec, error) {
var (
btfSection *elf.Section
sectionSizes = make(map[string]uint32)
Expand Down Expand Up @@ -302,7 +302,7 @@ func loadSpecFromELF(file *internal.SafeELFFile) (*Spec, error) {
return nil, fmt.Errorf("compressed BTF is not supported")
}

spec, err := loadRawSpec(btfSection.ReaderAt, file.ByteOrder, nil)
spec, err := loadRawSpec(btfSection.ReaderAt, file.ByteOrder, nil, filter)
if err != nil {
return nil, err
}
Expand All @@ -315,7 +315,7 @@ func loadSpecFromELF(file *internal.SafeELFFile) (*Spec, error) {
return spec, nil
}

func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec) (*Spec, error) {
func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec, filter *TypeFilter) (*Spec, error) {
var (
baseStrings *stringTable
firstTypeID TypeID
Expand All @@ -335,7 +335,7 @@ func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec) (*Spec, error
}
}

types, rawStrings, err := parseBTF(btf, bo, baseStrings, base)
types, rawStrings, err := parseBTF(btf, bo, baseStrings, base, filter)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -402,7 +402,7 @@ func guessRawBTFByteOrder(r io.ReaderAt) binary.ByteOrder {

// parseBTF reads a .BTF section into memory and parses it into a list of
// raw types and a string table.
func parseBTF(btf io.ReaderAt, bo binary.ByteOrder, baseStrings *stringTable, base *Spec) ([]Type, *stringTable, error) {
func parseBTF(btf io.ReaderAt, bo binary.ByteOrder, baseStrings *stringTable, base *Spec, filter *TypeFilter) ([]Type, *stringTable, error) {
buf := internal.NewBufferedSectionReader(btf, 0, math.MaxInt64)
header, err := parseBTFHeader(buf, bo)
if err != nil {
Expand All @@ -415,8 +415,11 @@ func parseBTF(btf io.ReaderAt, bo binary.ByteOrder, baseStrings *stringTable, ba
return nil, nil, fmt.Errorf("can't read type names: %w", err)
}

buf.Reset(io.NewSectionReader(btf, header.typeStart(), int64(header.TypeLen)))
types, err := readAndInflateTypes(buf, bo, header.TypeLen, rawStrings, base)
reset := func(offset int64) {
buf.Reset(io.NewSectionReader(btf, header.typeStart()+offset, int64(header.TypeLen)))
}
reset(0)
types, err := readAndInflateTypes(buf, reset, bo, header.TypeLen, rawStrings, base, filter)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -488,6 +491,7 @@ func fixupDatasec(types []Type, sectionSizes map[string]uint32, offsets map[symb
symName := ds.Vars[i].Type.TypeName()
ds.Vars[i].Offset, ok = offsets[symbol{name, symName}]
if !ok {
//fmt.Printf("Error: %+v not found: offsets=%+v\n", symbol{name, symName}, offsets)
return fmt.Errorf("data section %s: missing offset for symbol %s", name, symName)
}
}
Expand Down Expand Up @@ -674,7 +678,7 @@ func (s *Spec) TypeByName(name string, typ interface{}) error {
// Types from base are used to resolve references in the split BTF.
// The returned Spec only contains types from the split BTF, not from the base.
func LoadSplitSpecFromReader(r io.ReaderAt, base *Spec) (*Spec, error) {
return loadRawSpec(r, internal.NativeEndian, base)
return loadRawSpec(r, internal.NativeEndian, base, nil)
}

// TypesIterator iterates over types of a given spec.
Expand Down
4 changes: 4 additions & 0 deletions btf/ext_info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,10 @@ func (cr *CORERelocation) String() string {
return fmt.Sprintf("CORERelocation(%s, %s[%s], local_id=%d)", cr.kind, cr.typ, cr.accessor, cr.id)
}

func (cr *CORERelocation) TypeName() string {
return cr.typ.TypeName()
}

func CORERelocationMetadata(ins *asm.Instruction) *CORERelocation {
relo, _ := ins.Metadata.Get(coreRelocationMeta{}).(*CORERelocation)
return relo
Expand Down
2 changes: 1 addition & 1 deletion btf/handle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ func (h *Handle) Spec(base *Spec) (*Spec, error) {
return nil, fmt.Errorf("missing base types")
}

return loadRawSpec(bytes.NewReader(btfBuffer), internal.NativeEndian, base)
return loadRawSpec(bytes.NewReader(btfBuffer), internal.NativeEndian, base, nil)
}

// Close destroys the handle.
Expand Down
16 changes: 11 additions & 5 deletions btf/kernel.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ func FlushKernelSpec() {
kernelBTF.modules = make(map[string]*Spec)
}

func LoadKernelSpecWithFilter(filter *TypeFilter) (*Spec, error) {
spec, _, err := loadKernelSpec(filter)
// without copy
return spec, err
}

// LoadKernelSpec returns the current kernel's BTF information.
//
// Defaults to /sys/kernel/btf/vmlinux and falls back to scanning the file system
Expand All @@ -51,7 +57,7 @@ func LoadKernelSpec() (*Spec, error) {
return spec.Copy(), nil
}

spec, _, err := loadKernelSpec()
spec, _, err := loadKernelSpec(nil)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -95,12 +101,12 @@ func LoadKernelModuleSpec(module string) (*Spec, error) {
return spec.Copy(), nil
}

func loadKernelSpec() (_ *Spec, fallback bool, _ error) {
func loadKernelSpec(filter *TypeFilter) (_ *Spec, fallback bool, _ error) {
fh, err := os.Open("/sys/kernel/btf/vmlinux")
if err == nil {
defer fh.Close()

spec, err := loadRawSpec(fh, internal.NativeEndian, nil)
spec, err := loadRawSpec(fh, internal.NativeEndian, nil, filter)
return spec, false, err
}

Expand All @@ -110,7 +116,7 @@ func loadKernelSpec() (_ *Spec, fallback bool, _ error) {
}
defer file.Close()

spec, err := LoadSpecFromReader(file)
spec, err := LoadSpecFromReader(file, filter)
return spec, true, err
}

Expand All @@ -126,7 +132,7 @@ func loadKernelModuleSpec(module string, base *Spec) (*Spec, error) {
}
defer fh.Close()

return loadRawSpec(fh, internal.NativeEndian, base)
return loadRawSpec(fh, internal.NativeEndian, base, nil)
}

// findVMLinux scans multiple well-known paths for vmlinux kernel images.
Expand Down
Loading
Loading