import ContentParser from 'node-warc/lib/warcRecord/warcContentParsers.js'

public class | source

ContentParser

Utility class for parsing parts of WARC records

Static Method Summary

Static Public Methods
public static	bufEndPosNoCRLF(buf: Buffer, bufLen: number): number Returns the index of the end of the supplied buffer that does not include `\r\n`
public static	parseHTTPPortion(bufs: Buffer[], req: boolean): RequestHTTP \| ResponseHTTP Parses the HTTP information of WARC request and response records
public static	parseReqHTTP(headerBuffs: Buffer[]): RequestHTTP Parses the request HTTP headers
public static	parseResHTTP(headerBuffs: Buffer[]): ResponseHTTP Parses the response HTTP headers
public static	parseWarcInfoMetaDataContent(bufs: Buffer[]): Object Parse a WARC Metadata records metadata content
public static	parseWarcRecordHeader(bufs: Buffer[]): Object Parse a WARC Records headers not HTTP Header parser
public static	utf8BufferSlice(buf: Buffer, start: number, end: number): string Slices the supplied buffer returning a UTF-8 string

Static Private Methods
private static	_parseHeaders(headerBuffs: Buffer[]): Object Parses an array of buffers containing HTTP headers

Static Public Methods

public static bufEndPosNoCRLF(buf: Buffer, bufLen: number): number source

Returns the index of the end of the supplied buffer that does not include \r\n

Params:

Name	Type	Attribute	Description
buf	Buffer		The buffer to receive the correct end index for
bufLen	number		The full length of the buffer

Return:

number

public static parseHTTPPortion(bufs: Buffer[], req: boolean): RequestHTTP | ResponseHTTP source

Parses the HTTP information of WARC request and response records

Params:

Name	Type	Attribute	Description
bufs	Buffer[]		Buffers containing the HTTP header information
req	boolean		Should the buffers be parsed as request or response

Return:

RequestHTTP | ResponseHTTP

public static parseReqHTTP(headerBuffs: Buffer[]): RequestHTTP source

Parses the request HTTP headers

Params:

Name	Type	Attribute	Description
headerBuffs	Buffer[]		the request HTTP headers

Return:

RequestHTTP

public static parseResHTTP(headerBuffs: Buffer[]): ResponseHTTP source

Parses the response HTTP headers

Params:

Name	Type	Attribute	Description
headerBuffs	Buffer[]		the response HTTP headers

Return:

ResponseHTTP

public static parseWarcInfoMetaDataContent(bufs: Buffer[]): Object source

Parse a WARC Metadata records metadata content

Params:

Name	Type	Attribute	Description
bufs	Buffer[]		the WARC Metadata records content lines

Return:

Object

public static parseWarcRecordHeader(bufs: Buffer[]): Object source

Parse a WARC Records headers not HTTP Header parser

Params:

Name	Type	Attribute	Description
bufs	Buffer[]		the WARC Records header lines

Return:

Object

public static utf8BufferSlice(buf: Buffer, start: number, end: number): string source

Slices the supplied buffer returning a UTF-8 string

Params:

Name	Type	Attribute	Description
buf	Buffer		The buffer to slice
start	number		The start position of the slice
end	number		The end position of the slice

Return:

string

Static Private Methods

private static _parseHeaders(headerBuffs: Buffer[]): Object source

Parses an array of buffers containing HTTP headers

Params:

Name	Type	Attribute	Description
headerBuffs	Buffer[]		The array of buffers representing HTTP headers

Return:

Object