Skip to content

Commit

Permalink
Add ScraperInterface and DataProcessorInterface
Browse files Browse the repository at this point in the history
  • Loading branch information
lqmanh committed Jul 19, 2020
1 parent 638a42d commit dcc4432
Show file tree
Hide file tree
Showing 7 changed files with 36 additions and 10 deletions.
2 changes: 2 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
export { DataProcessor } from './modules/data-processor/data-processor.class'
export { DataProcessorInterface } from './modules/data-processor/data-processor.interface'
export { DataEntity } from './modules/scheduler/data-entity.class'
export { Scheduler } from './modules/scheduler/scheduler.class'
export { UrlEntity } from './modules/scheduler/url-entity.class'
export { Scraper } from './modules/scraper/scraper.class'
export { ScraperInterface } from './modules/scraper/scraper.interface'
5 changes: 3 additions & 2 deletions src/modules/data-processor/data-processor.class.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ import pino, { Logger } from 'pino'
import { DataProcessingResult } from './data-processing-result.interface'
import { DataProcessorOptions } from './data-processor-options.class'
import { DataProcessorOptionsInterface } from './data-processor-options.interface'
import { DataProcessorInterface } from './data-processor.interface'

/**
* Data processor
* Data processor class
*/
export abstract class DataProcessor {
export abstract class DataProcessor implements DataProcessorInterface {
private readonly logger: Logger

constructor(options: DataProcessorOptionsInterface = {}) {
Expand Down
11 changes: 11 additions & 0 deletions src/modules/data-processor/data-processor.interface.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { DataProcessingResult } from './data-processing-result.interface'

/**
* Data processor interface
*/
export interface DataProcessorInterface {
/**
* Run
*/
run(data: object): Promise<DataProcessingResult>
}
6 changes: 3 additions & 3 deletions src/modules/scheduler/data-entity.class.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import { DataProcessor } from '../data-processor/data-processor.class'
import { DataProcessorInterface } from '../data-processor/data-processor.interface'

/** Data entity */
export class DataEntity {
data: object
dataProcessor: DataProcessor
dataProcessor: DataProcessorInterface
retryCount = -1

constructor(data: object, dataProcessor: DataProcessor) {
constructor(data: object, dataProcessor: DataProcessorInterface) {
this.data = data
this.dataProcessor = dataProcessor
}
Expand Down
6 changes: 3 additions & 3 deletions src/modules/scheduler/url-entity.class.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import { Scraper } from '../scraper/scraper.class'
import { ScraperInterface } from '../scraper/scraper.interface'

/** URL entity */
export class UrlEntity {
url: string
scraper: Scraper
scraper: ScraperInterface
retryCount = -1

constructor(url: string, scraper: Scraper) {
constructor(url: string, scraper: ScraperInterface) {
this.url = url
this.scraper = scraper
}
Expand Down
5 changes: 3 additions & 2 deletions src/modules/scraper/scraper.class.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ import { ParsingMeta } from './parsing-meta.interface'
import { ParsingResult } from './parsing-result.interface'
import { ScraperOptions } from './scraper-options.class'
import { ScraperOptionsInterface } from './scraper-options.interface'
import { ScraperInterface } from './scraper.interface'
import { ScrapingResult } from './scraping-result.interface'

/**
* Scraper
* Scraper class
*/
export abstract class Scraper {
export abstract class Scraper implements ScraperInterface {
private readonly axios: AxiosInstance
protected readonly userAgents: string[]
protected readonly proxies: HttpProxy[]
Expand Down
11 changes: 11 additions & 0 deletions src/modules/scraper/scraper.interface.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import { ScrapingResult } from './scraping-result.interface'

/**
* Scraper interface
*/
export interface ScraperInterface {
/**
* Run
*/
run(url: string): Promise<ScrapingResult>
}

0 comments on commit dcc4432

Please sign in to comment.